Initial commit

src/augmentationFactory.ts

@@ -0,0 +1,628 @@
+/**
+ * Augmentation Factory
+ *
+ * This module provides a simplified factory for creating augmentations with minimal boilerplate.
+ * It reduces the complexity of creating and using augmentations by providing a fluent API
+ * and handling common patterns automatically.
+ */
+
+import {
+  IAugmentation,
+  AugmentationType,
+  AugmentationResponse,
+  ISenseAugmentation,
+  IConduitAugmentation,
+  ICognitionAugmentation,
+  IMemoryAugmentation,
+  IPerceptionAugmentation,
+  IDialogAugmentation,
+  IActivationAugmentation,
+  IWebSocketSupport,
+  WebSocketConnection
+} from './types/augmentations.js'
+import { registerAugmentation } from './augmentationRegistry.js'
+
+/**
+ * Options for creating an augmentation
+ */
+export interface AugmentationOptions {
+  name: string
+  description?: string
+  enabled?: boolean
+  autoRegister?: boolean
+  autoInitialize?: boolean
+}
+
+/**
+ * Base class for all augmentations created with the factory
+ * Handles common functionality like initialization, shutdown, and status
+ */
+class BaseAugmentation implements IAugmentation {
+  readonly name: string
+  readonly description: string
+  enabled: boolean = true
+  protected isInitialized: boolean = false
+
+  constructor(options: AugmentationOptions) {
+    this.name = options.name
+    this.description = options.description || `${options.name} augmentation`
+    this.enabled = options.enabled !== false
+  }
+
+  async initialize(): Promise<void> {
+    if (this.isInitialized) return
+    this.isInitialized = true
+  }
+
+  async shutDown(): Promise<void> {
+    this.isInitialized = false
+  }
+
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return this.isInitialized ? 'active' : 'inactive'
+  }
+
+  protected async ensureInitialized(): Promise<void> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+  }
+}
+
+/**
+ * Factory for creating sense augmentations
+ */
+export function createSenseAugmentation(
+  options: AugmentationOptions & {
+    processRawData?: (
+      rawData: Buffer | string,
+      dataType: string
+    ) =>
+      | Promise<AugmentationResponse<{ nouns: string[]; verbs: string[] }>>
+      | AugmentationResponse<{
+          nouns: string[]
+          verbs: string[]
+        }>
+    listenToFeed?: (
+      feedUrl: string,
+      callback: (data: { nouns: string[]; verbs: string[] }) => void
+    ) => Promise<void>
+  }
+): ISenseAugmentation {
+  const augmentation = new BaseAugmentation(options) as unknown as ISenseAugmentation
+
+  // Implement the sense augmentation methods
+  augmentation.processRawData = async (
+    rawData: Buffer | string,
+    dataType: string
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.processRawData) {
+      const result = options.processRawData(rawData, dataType)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: { nouns: [], verbs: [] },
+      error: 'processRawData not implemented'
+    }
+  }
+
+  augmentation.listenToFeed = async (
+    feedUrl: string,
+    callback: (data: { nouns: string[]; verbs: string[] }) => void
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.listenToFeed) {
+      return options.listenToFeed(feedUrl, callback)
+    }
+
+    throw new Error('listenToFeed not implemented')
+  }
+
+  // Auto-register if requested
+  if (options.autoRegister) {
+    registerAugmentation(augmentation)
+
+    // Auto-initialize if requested
+    if (options.autoInitialize) {
+      augmentation.initialize().catch((error) => {
+        console.error(
+          `Failed to initialize augmentation ${augmentation.name}:`,
+          error
+        )
+      })
+    }
+  }
+
+  return augmentation
+}
+
+/**
+ * Factory for creating conduit augmentations
+ */
+export function createConduitAugmentation(
+  options: AugmentationOptions & {
+    establishConnection?: (
+      targetSystemId: string,
+      config: Record<string, unknown>
+    ) =>
+      | Promise<AugmentationResponse<WebSocketConnection>>
+      | AugmentationResponse<WebSocketConnection>
+    readData?: (
+      query: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<unknown>> | AugmentationResponse<unknown>
+    writeData?: (
+      data: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<unknown>> | AugmentationResponse<unknown>
+    monitorStream?: (
+      streamId: string,
+      callback: (data: unknown) => void
+    ) => Promise<void>
+  }
+): IConduitAugmentation {
+  const augmentation = new BaseAugmentation(options) as unknown as IConduitAugmentation
+
+  // Implement the conduit augmentation methods
+  augmentation.establishConnection = async (
+    targetSystemId: string,
+    config: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.establishConnection) {
+      const result = options.establishConnection(targetSystemId, config)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null as any,
+      error: 'establishConnection not implemented'
+    }
+  }
+
+  augmentation.readData = async (
+    query: Record<string, unknown>,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.readData) {
+      const result = options.readData(query, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null,
+      error: 'readData not implemented'
+    }
+  }
+
+  augmentation.writeData = async (
+    data: Record<string, unknown>,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.writeData) {
+      const result = options.writeData(data, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null,
+      error: 'writeData not implemented'
+    }
+  }
+
+  augmentation.monitorStream = async (
+    streamId: string,
+    callback: (data: unknown) => void
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.monitorStream) {
+      return options.monitorStream(streamId, callback)
+    }
+
+    throw new Error('monitorStream not implemented')
+  }
+
+  // Auto-register if requested
+  if (options.autoRegister) {
+    registerAugmentation(augmentation)
+
+    // Auto-initialize if requested
+    if (options.autoInitialize) {
+      augmentation.initialize().catch((error) => {
+        console.error(
+          `Failed to initialize augmentation ${augmentation.name}:`,
+          error
+        )
+      })
+    }
+  }
+
+  return augmentation
+}
+
+/**
+ * Factory for creating memory augmentations
+ */
+export function createMemoryAugmentation(
+  options: AugmentationOptions & {
+    storeData?: (
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<boolean>> | AugmentationResponse<boolean>
+    retrieveData?: (
+      key: string,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<unknown>> | AugmentationResponse<unknown>
+    updateData?: (
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<boolean>> | AugmentationResponse<boolean>
+    deleteData?: (
+      key: string,
+      options?: Record<string, unknown>
+    ) => Promise<AugmentationResponse<boolean>> | AugmentationResponse<boolean>
+    listDataKeys?: (
+      pattern?: string,
+      options?: Record<string, unknown>
+    ) =>
+      | Promise<AugmentationResponse<string[]>>
+      | AugmentationResponse<string[]>
+    search?: (
+      query: unknown,
+      k?: number,
+      options?: Record<string, unknown>
+    ) =>
+      | Promise<
+          AugmentationResponse<
+            Array<{ id: string; score: number; data: unknown }>
+          >
+        >
+      | AugmentationResponse<
+          Array<{ id: string; score: number; data: unknown }>
+        >
+  }
+): IMemoryAugmentation {
+  const augmentation = new BaseAugmentation(options) as unknown as IMemoryAugmentation
+
+  // Implement the memory augmentation methods
+  augmentation.storeData = async (
+    key: string,
+    data: unknown,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.storeData) {
+      const result = options.storeData(key, data, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: false,
+      error: 'storeData not implemented'
+    }
+  }
+
+  augmentation.retrieveData = async (
+    key: string,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.retrieveData) {
+      const result = options.retrieveData(key, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: null,
+      error: 'retrieveData not implemented'
+    }
+  }
+
+  augmentation.updateData = async (
+    key: string,
+    data: unknown,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.updateData) {
+      const result = options.updateData(key, data, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: false,
+      error: 'updateData not implemented'
+    }
+  }
+
+  augmentation.deleteData = async (
+    key: string,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.deleteData) {
+      const result = options.deleteData(key, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: false,
+      error: 'deleteData not implemented'
+    }
+  }
+
+  augmentation.listDataKeys = async (
+    pattern?: string,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.listDataKeys) {
+      const result = options.listDataKeys(pattern, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: [],
+      error: 'listDataKeys not implemented'
+    }
+  }
+
+  augmentation.search = async (
+    query: unknown,
+    k?: number,
+    opts?: Record<string, unknown>
+  ) => {
+    await augmentation.ensureInitialized()
+
+    if (options.search) {
+      const result = options.search(query, k, opts)
+      return result instanceof Promise ? await result : result
+    }
+
+    return {
+      success: false,
+      data: [],
+      error: 'search not implemented'
+    }
+  }
+
+  // Auto-register if requested
+  if (options.autoRegister) {
+    registerAugmentation(augmentation)
+
+    // Auto-initialize if requested
+    if (options.autoInitialize) {
+      augmentation.initialize().catch((error) => {
+        console.error(
+          `Failed to initialize augmentation ${augmentation.name}:`,
+          error
+        )
+      })
+    }
+  }
+
+  return augmentation
+}
+
+/**
+ * Factory for creating WebSocket-enabled augmentations
+ * This can be combined with other augmentation factories to create WebSocket-enabled versions
+ */
+export function addWebSocketSupport<T extends IAugmentation>(
+  augmentation: T,
+  options: {
+    connectWebSocket?: (
+      url: string,
+      protocols?: string | string[]
+    ) => Promise<WebSocketConnection>
+    sendWebSocketMessage?: (
+      connectionId: string,
+      data: unknown
+    ) => Promise<void>
+    onWebSocketMessage?: (
+      connectionId: string,
+      callback: (data: unknown) => void
+    ) => Promise<void>
+    offWebSocketMessage?: (
+      connectionId: string,
+      callback: (data: unknown) => void
+    ) => Promise<void>
+    closeWebSocket?: (
+      connectionId: string,
+      code?: number,
+      reason?: string
+    ) => Promise<void>
+  }
+): T & IWebSocketSupport {
+  const wsAugmentation = augmentation as T & IWebSocketSupport
+
+  // Add WebSocket methods
+  wsAugmentation.connectWebSocket = async (
+    url: string,
+    protocols?: string | string[]
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.connectWebSocket) {
+      return options.connectWebSocket(url, protocols)
+    }
+
+    throw new Error('connectWebSocket not implemented')
+  }
+
+  wsAugmentation.sendWebSocketMessage = async (
+    connectionId: string,
+    data: unknown
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.sendWebSocketMessage) {
+      return options.sendWebSocketMessage(connectionId, data)
+    }
+
+    throw new Error('sendWebSocketMessage not implemented')
+  }
+
+  wsAugmentation.onWebSocketMessage = async (
+    connectionId: string,
+    callback: (data: unknown) => void
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.onWebSocketMessage) {
+      return options.onWebSocketMessage(connectionId, callback)
+    }
+
+    throw new Error('onWebSocketMessage not implemented')
+  }
+
+  wsAugmentation.offWebSocketMessage = async (
+    connectionId: string,
+    callback: (data: unknown) => void
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.offWebSocketMessage) {
+      return options.offWebSocketMessage(connectionId, callback)
+    }
+
+    throw new Error('offWebSocketMessage not implemented')
+  }
+
+  wsAugmentation.closeWebSocket = async (
+    connectionId: string,
+    code?: number,
+    reason?: string
+  ) => {
+    await (augmentation as any).ensureInitialized?.()
+
+    if (options.closeWebSocket) {
+      return options.closeWebSocket(connectionId, code, reason)
+    }
+
+    throw new Error('closeWebSocket not implemented')
+  }
+
+  return wsAugmentation
+}
+
+/**
+ * Simplified function to execute an augmentation method with automatic error handling
+ * This provides a more concise way to execute augmentation methods compared to the full pipeline
+ */
+export async function executeAugmentation<T, R>(
+  augmentation: IAugmentation,
+  method: string,
+  ...args: any[]
+): Promise<AugmentationResponse<R>> {
+  try {
+    if (!augmentation.enabled) {
+      return {
+        success: false,
+        data: null as any,
+        error: `Augmentation ${augmentation.name} is disabled`
+      }
+    }
+
+    if (typeof (augmentation as any)[method] !== 'function') {
+      return {
+        success: false,
+        data: null as any,
+        error: `Method ${method} not found on augmentation ${augmentation.name}`
+      }
+    }
+
+    const result = await (augmentation as any)[method](...args)
+    return result
+  } catch (error) {
+    console.error(`Error executing ${method} on ${augmentation.name}:`, error)
+    return {
+      success: false,
+      data: null as any,
+      error: error instanceof Error ? error.message : String(error)
+    }
+  }
+}
+
+/**
+ * Dynamically load augmentations from a module at runtime
+ * This allows for lazy-loading augmentations when needed instead of at build time
+ */
+export async function loadAugmentationModule(
+  modulePromise: Promise<any>,
+  options: {
+    autoRegister?: boolean
+    autoInitialize?: boolean
+  } = {}
+): Promise<IAugmentation[]> {
+  try {
+    const module = await modulePromise
+    const augmentations: IAugmentation[] = []
+
+    // Extract augmentations from the module
+    for (const key in module) {
+      const exported = module[key]
+
+      // Skip non-objects and null
+      if (!exported || typeof exported !== 'object') {
+        continue
+      }
+
+      // Check if it's an augmentation
+      if (
+        typeof exported.name === 'string' &&
+        typeof exported.initialize === 'function' &&
+        typeof exported.shutDown === 'function' &&
+        typeof exported.getStatus === 'function'
+      ) {
+        augmentations.push(exported)
+
+        // Auto-register if requested
+        if (options.autoRegister) {
+          registerAugmentation(exported)
+
+          // Auto-initialize if requested
+          if (options.autoInitialize) {
+            exported.initialize().catch((error: Error) => {
+              console.error(
+                `Failed to initialize augmentation ${exported.name}:`,
+                error
+              )
+            })
+          }
+        }
+      }
+    }
+
+    return augmentations
+  } catch (error) {
+    console.error('Error loading augmentation module:', error)
+    return []
+  }
+}

src/augmentationPipeline.ts

@@ -0,0 +1,709 @@
+/**
+ * Augmentation Event Pipeline
+ *
+ * This module provides a pipeline for managing and executing multiple augmentations
+ * of each type. It allows registering multiple augmentations and executing them
+ * in sequence or in parallel.
+ */
+
+import {
+  BrainyAugmentations,
+  IAugmentation,
+  IWebSocketSupport,
+  AugmentationResponse,
+  AugmentationType
+} from './types/augmentations.js'
+import { isThreadingAvailable, isBrowser, isNode } from './utils/environment.js'
+import { executeInThread } from './utils/workerUtils.js'
+
+/**
+ * Type definitions for the augmentation registry
+ */
+type AugmentationRegistry = {
+  sense: BrainyAugmentations.ISenseAugmentation[];
+  conduit: BrainyAugmentations.IConduitAugmentation[];
+  cognition: BrainyAugmentations.ICognitionAugmentation[];
+  memory: BrainyAugmentations.IMemoryAugmentation[];
+  perception: BrainyAugmentations.IPerceptionAugmentation[];
+  dialog: BrainyAugmentations.IDialogAugmentation[];
+  activation: BrainyAugmentations.IActivationAugmentation[];
+  webSocket: IWebSocketSupport[];
+}
+
+/**
+ * Execution mode for the pipeline
+ */
+export enum ExecutionMode {
+  SEQUENTIAL = 'sequential',
+  PARALLEL = 'parallel',
+  FIRST_SUCCESS = 'firstSuccess',
+  FIRST_RESULT = 'firstResult',
+  THREADED = 'threaded'  // Execute in separate threads when available
+}
+
+/**
+ * Options for pipeline execution
+ */
+export interface PipelineOptions {
+  mode?: ExecutionMode;
+  timeout?: number;
+  stopOnError?: boolean;
+  forceThreading?: boolean;  // Force threading even if not in THREADED mode
+  disableThreading?: boolean;  // Disable threading even if in THREADED mode
+}
+
+/**
+ * Default pipeline options
+ */
+const DEFAULT_PIPELINE_OPTIONS: PipelineOptions = {
+  mode: ExecutionMode.SEQUENTIAL,
+  timeout: 30000,
+  stopOnError: false,
+  forceThreading: false,
+  disableThreading: false
+}
+
+/**
+ * AugmentationPipeline class
+ *
+ * Manages multiple augmentations of each type and provides methods to execute them.
+ */
+export class AugmentationPipeline {
+  private registry: AugmentationRegistry = {
+    sense: [],
+    conduit: [],
+    cognition: [],
+    memory: [],
+    perception: [],
+    dialog: [],
+    activation: [],
+    webSocket: []
+  }
+
+  /**
+   * Register an augmentation with the pipeline
+   *
+   * @param augmentation The augmentation to register
+   * @returns The pipeline instance for chaining
+   */
+  public register<T extends IAugmentation>(augmentation: T): AugmentationPipeline {
+    let registered = false
+
+    // Check for specific augmentation types
+    if (this.isAugmentationType<BrainyAugmentations.ISenseAugmentation>(
+      augmentation,
+      'processRawData',
+      'listenToFeed'
+    )) {
+      this.registry.sense.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IConduitAugmentation>(
+      augmentation,
+      'establishConnection',
+      'readData',
+      'writeData',
+      'monitorStream'
+    )) {
+      this.registry.conduit.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.ICognitionAugmentation>(
+      augmentation,
+      'reason',
+      'infer',
+      'executeLogic'
+    )) {
+      this.registry.cognition.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IMemoryAugmentation>(
+      augmentation,
+      'storeData',
+      'retrieveData',
+      'updateData',
+      'deleteData',
+      'listDataKeys'
+    )) {
+      this.registry.memory.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IPerceptionAugmentation>(
+      augmentation,
+      'interpret',
+      'organize',
+      'generateVisualization'
+    )) {
+      this.registry.perception.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IDialogAugmentation>(
+      augmentation,
+      'processUserInput',
+      'generateResponse',
+      'manageContext'
+    )) {
+      this.registry.dialog.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IActivationAugmentation>(
+      augmentation,
+      'triggerAction',
+      'generateOutput',
+      'interactExternal'
+    )) {
+      this.registry.activation.push(augmentation)
+      registered = true
+    }
+
+    // Check if the augmentation supports WebSocket
+    if (this.isAugmentationType<IWebSocketSupport>(
+      augmentation,
+      'connectWebSocket',
+      'sendWebSocketMessage',
+      'onWebSocketMessage',
+      'closeWebSocket'
+    )) {
+      this.registry.webSocket.push(augmentation as IWebSocketSupport)
+      registered = true
+    }
+
+    // If the augmentation wasn't registered as any known type, throw an error
+    if (!registered) {
+      throw new Error(`Unknown augmentation type: ${augmentation.name}`)
+    }
+
+    return this
+  }
+
+  /**
+   * Unregister an augmentation from the pipeline
+   *
+   * @param augmentationName The name of the augmentation to unregister
+   * @returns The pipeline instance for chaining
+   */
+  public unregister(augmentationName: string): AugmentationPipeline {
+    let found = false
+
+    // Remove from all registries
+    for (const type in this.registry) {
+      const typedRegistry = this.registry[type as keyof AugmentationRegistry]
+      const index = typedRegistry.findIndex(aug => aug.name === augmentationName)
+
+      if (index !== -1) {
+        typedRegistry.splice(index, 1)
+        found = true
+      }
+    }
+
+    return this
+  }
+
+  /**
+   * Initialize all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are initialized
+   */
+  public async initialize(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.initialize().catch(error => {
+          console.error(`Failed to initialize augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Shut down all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are shut down
+   */
+  public async shutDown(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.shutDown().catch(error => {
+          console.error(`Failed to shut down augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Execute a sense pipeline
+   *
+   * @param method The method to execute on each sense augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeSensePipeline<
+    M extends keyof BrainyAugmentations.ISenseAugmentation & string,
+    R extends BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ISenseAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.ISenseAugmentation, M, R>(
+      this.registry.sense,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a conduit pipeline
+   *
+   * @param method The method to execute on each conduit augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeConduitPipeline<
+    M extends keyof BrainyAugmentations.IConduitAugmentation & string,
+    R extends BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IConduitAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IConduitAugmentation, M, R>(
+      this.registry.conduit,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a cognition pipeline
+   *
+   * @param method The method to execute on each cognition augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeCognitionPipeline<
+    M extends keyof BrainyAugmentations.ICognitionAugmentation & string,
+    R extends BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ICognitionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.ICognitionAugmentation, M, R>(
+      this.registry.cognition,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a memory pipeline
+   *
+   * @param method The method to execute on each memory augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeMemoryPipeline<
+    M extends keyof BrainyAugmentations.IMemoryAugmentation & string,
+    R extends BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IMemoryAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IMemoryAugmentation, M, R>(
+      this.registry.memory,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a perception pipeline
+   *
+   * @param method The method to execute on each perception augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executePerceptionPipeline<
+    M extends keyof BrainyAugmentations.IPerceptionAugmentation & string,
+    R extends BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IPerceptionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IPerceptionAugmentation, M, R>(
+      this.registry.perception,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute a dialog pipeline
+   *
+   * @param method The method to execute on each dialog augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeDialogPipeline<
+    M extends keyof BrainyAugmentations.IDialogAugmentation & string,
+    R extends BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IDialogAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IDialogAugmentation, M, R>(
+      this.registry.dialog,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Execute an activation pipeline
+   *
+   * @param method The method to execute on each activation augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  public async executeActivationPipeline<
+    M extends keyof BrainyAugmentations.IActivationAugmentation & string,
+    R extends BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IActivationAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    return this.executeTypedPipeline<BrainyAugmentations.IActivationAugmentation, M, R>(
+      this.registry.activation,
+      method,
+      args,
+      opts
+    )
+  }
+
+  /**
+   * Get all registered augmentations
+   *
+   * @returns An array of all registered augmentations
+   */
+  public getAllAugmentations(): IAugmentation[] {
+    // Create a Set to avoid duplicates (an augmentation might be in multiple registries)
+    const allAugmentations = new Set<IAugmentation>([
+      ...this.registry.sense,
+      ...this.registry.conduit,
+      ...this.registry.cognition,
+      ...this.registry.memory,
+      ...this.registry.perception,
+      ...this.registry.dialog,
+      ...this.registry.activation,
+      ...this.registry.webSocket
+    ])
+
+    // Convert back to array
+    return Array.from(allAugmentations)
+  }
+
+  /**
+   * Get all augmentations of a specific type
+   *
+   * @param type The type of augmentation to get
+   * @returns An array of all augmentations of the specified type
+   */
+  public getAugmentationsByType(type: AugmentationType): IAugmentation[] {
+    switch (type) {
+      case AugmentationType.SENSE:
+        return [...this.registry.sense]
+      case AugmentationType.CONDUIT:
+        return [...this.registry.conduit]
+      case AugmentationType.COGNITION:
+        return [...this.registry.cognition]
+      case AugmentationType.MEMORY:
+        return [...this.registry.memory]
+      case AugmentationType.PERCEPTION:
+        return [...this.registry.perception]
+      case AugmentationType.DIALOG:
+        return [...this.registry.dialog]
+      case AugmentationType.ACTIVATION:
+        return [...this.registry.activation]
+      case AugmentationType.WEBSOCKET:
+        return [...this.registry.webSocket]
+      default:
+        return []
+    }
+  }
+
+  /**
+   * Get all available augmentation types
+   *
+   * @returns An array of all augmentation types that have at least one registered augmentation
+   */
+  public getAvailableAugmentationTypes(): AugmentationType[] {
+    const availableTypes: AugmentationType[] = []
+
+    if (this.registry.sense.length > 0) availableTypes.push(AugmentationType.SENSE)
+    if (this.registry.conduit.length > 0) availableTypes.push(AugmentationType.CONDUIT)
+    if (this.registry.cognition.length > 0) availableTypes.push(AugmentationType.COGNITION)
+    if (this.registry.memory.length > 0) availableTypes.push(AugmentationType.MEMORY)
+    if (this.registry.perception.length > 0) availableTypes.push(AugmentationType.PERCEPTION)
+    if (this.registry.dialog.length > 0) availableTypes.push(AugmentationType.DIALOG)
+    if (this.registry.activation.length > 0) availableTypes.push(AugmentationType.ACTIVATION)
+    if (this.registry.webSocket.length > 0) availableTypes.push(AugmentationType.WEBSOCKET)
+
+    return availableTypes
+  }
+
+  /**
+   * Get all WebSocket-supporting augmentations
+   *
+   * @returns An array of all augmentations that support WebSocket connections
+   */
+  public getWebSocketAugmentations(): IWebSocketSupport[] {
+    return [...this.registry.webSocket]
+  }
+
+  /**
+   * Check if an augmentation is of a specific type
+   *
+   * @param augmentation The augmentation to check
+   * @param methods The methods that should be present on the augmentation
+   * @returns True if the augmentation is of the specified type
+   */
+  private isAugmentationType<T extends IAugmentation>(
+    augmentation: IAugmentation,
+    ...methods: (keyof T)[]
+  ): augmentation is T {
+    // First check that the augmentation has all the required base methods
+    const baseMethodsExist = [
+      'initialize',
+      'shutDown',
+      'getStatus'
+    ].every(method => typeof (augmentation as any)[method] === 'function')
+
+    if (!baseMethodsExist) {
+      return false
+    }
+
+    // Then check that it has all the specific methods for this type
+    return methods.every(method => typeof (augmentation as any)[method] === 'function')
+  }
+
+  /**
+   * Determines if threading should be used based on options and environment
+   * 
+   * @param options The pipeline options
+   * @returns True if threading should be used, false otherwise
+   */
+  private shouldUseThreading(options: PipelineOptions): boolean {
+    // If threading is explicitly disabled, don't use it
+    if (options.disableThreading) {
+      return false;
+    }
+
+    // If threading is explicitly forced, use it if available
+    if (options.forceThreading) {
+      return isThreadingAvailable();
+    }
+
+    // If in THREADED mode, use threading if available
+    if (options.mode === ExecutionMode.THREADED) {
+      return isThreadingAvailable();
+    }
+
+    // Otherwise, don't use threading
+    return false;
+  }
+
+  /**
+   * Execute a pipeline for a specific augmentation type
+   *
+   * @param augmentations The augmentations to execute
+   * @param method The method to execute on each augmentation
+   * @param args The arguments to pass to the method
+   * @param options The pipeline execution options
+   * @returns A promise that resolves with the results from all augmentations
+   */
+  private async executeTypedPipeline<
+    T extends IAugmentation,
+    M extends keyof T & string,
+    R extends T[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    augmentations: T[],
+    method: M & (T[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<T[M], (...args: any[]) => any>>,
+    options: PipelineOptions
+  ): Promise<Promise<{
+    success: boolean
+    data: R
+    error?: string
+  }>[]> {
+    // Filter out disabled augmentations
+    const enabledAugmentations = augmentations.filter(aug => aug.enabled !== false)
+
+    if (enabledAugmentations.length === 0) {
+      return []
+    }
+
+    // Create a function to execute the method on an augmentation
+    const executeMethod = async (augmentation: T): Promise<{
+      success: boolean
+      data: R
+      error?: string
+    }> => {
+      try {
+        // Create a timeout promise if a timeout is specified
+        const timeoutPromise = options.timeout
+          ? new Promise<{
+            success: boolean
+            data: R
+            error?: string
+          }>((_, reject) => {
+            setTimeout(() => {
+              reject(new Error(`Timeout executing ${String(method)} on ${augmentation.name}`))
+            }, options.timeout)
+          })
+          : null
+
+        // Check if threading should be used
+        const useThreading = this.shouldUseThreading(options);
+
+        // Execute the method on the augmentation, using threading if appropriate
+        let methodPromise: Promise<AugmentationResponse<R>>;
+
+        if (useThreading) {
+          // Execute in a separate thread
+          try {
+            // Create a function that can be serialized and executed in a worker
+            const workerFn = (...workerArgs: any[]) => {
+              // This function will be stringified and executed in the worker
+              // It needs to be self-contained
+              const augFn = augmentation[method as string] as Function;
+              return augFn.apply(augmentation, workerArgs);
+            };
+
+            methodPromise = executeInThread<AugmentationResponse<R>>(workerFn.toString(), args);
+          } catch (threadError) {
+            console.warn(`Failed to execute in thread, falling back to main thread: ${threadError}`);
+            // Fall back to executing in the main thread
+            methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<R>);
+          }
+        } else {
+          // Execute in the main thread
+          methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<R>);
+        }
+
+        // Race the method promise against the timeout promise if a timeout is specified
+        const result = timeoutPromise
+          ? await Promise.race([methodPromise, timeoutPromise])
+          : await methodPromise
+
+        return result
+      } catch (error) {
+        console.error(`Error executing ${String(method)} on ${augmentation.name}:`, error)
+        return {
+          success: false,
+          data: null as unknown as R,
+          error: error instanceof Error ? error.message : String(error)
+        }
+      }
+    }
+
+    // Execute the pipeline based on the specified mode
+    switch (options.mode) {
+      case ExecutionMode.PARALLEL:
+        // Execute all augmentations in parallel
+        return enabledAugmentations.map(executeMethod)
+
+      case ExecutionMode.THREADED:
+        // Execute all augmentations in parallel with threading enabled
+        // Force threading for this mode
+        const threadedOptions = { ...options, forceThreading: true };
+
+        // Create a new executeMethod function that uses the threaded options
+        const executeMethodThreaded = async (augmentation: T) => {
+          // Save the original options
+          const originalOptions = options;
+
+          // Set the options to the threaded options
+          options = threadedOptions;
+
+          // Execute the method
+          const result = await executeMethod(augmentation);
+
+          // Restore the original options
+          options = originalOptions;
+
+          return result;
+        };
+
+        return enabledAugmentations.map(executeMethodThreaded);
+
+      case ExecutionMode.FIRST_SUCCESS:
+        // Execute augmentations sequentially until one succeeds
+        for (const augmentation of enabledAugmentations) {
+          const resultPromise = executeMethod(augmentation)
+          const result = await resultPromise
+          if (result.success) {
+            return [resultPromise]
+          }
+        }
+        return []
+
+      case ExecutionMode.FIRST_RESULT:
+        // Execute augmentations sequentially until one returns a result
+        for (const augmentation of enabledAugmentations) {
+          const resultPromise = executeMethod(augmentation)
+          const result = await resultPromise
+          if (result.success && result.data) {
+            return [resultPromise]
+          }
+        }
+        return []
+
+      case ExecutionMode.SEQUENTIAL:
+      default:
+        // Execute augmentations sequentially
+        const results: Promise<{
+          success: boolean
+          data: R
+          error?: string
+        }>[] = []
+        for (const augmentation of enabledAugmentations) {
+          const resultPromise = executeMethod(augmentation)
+          results.push(resultPromise)
+
+          // Check if we need to stop on error
+          if (options.stopOnError) {
+            const result = await resultPromise
+            if (!result.success) {
+              break
+            }
+          }
+        }
+        return results
+    }
+  }
+}
+
+// Create and export a default instance of the pipeline
+export const augmentationPipeline = new AugmentationPipeline()

src/augmentationRegistry.ts

@@ -0,0 +1,122 @@
+/**
+ * Augmentation Registry
+ *
+ * This module provides a registry for augmentations that are loaded at build time.
+ * It replaces the dynamic loading mechanism in pluginLoader.ts.
+ */
+
+import { IPipeline } from './types/pipelineTypes.js'
+import { AugmentationType, IAugmentation } from './types/augmentations.js'
+
+// Forward declaration of the pipeline instance to avoid circular dependency
+// The actual pipeline will be provided when initializeAugmentationPipeline is called
+let defaultPipeline: IPipeline | null = null
+
+/**
+ * Sets the default pipeline instance
+ * This function should be called from pipeline.ts after the pipeline is created
+ */
+export function setDefaultPipeline(pipeline: IPipeline): void {
+  defaultPipeline = pipeline
+}
+
+/**
+ * Registry of all available augmentations
+ */
+export const availableAugmentations: IAugmentation[] = []
+
+/**
+ * Registers an augmentation with the registry
+ *
+ * @param augmentation The augmentation to register
+ * @returns The augmentation that was registered
+ */
+export function registerAugmentation<T extends IAugmentation>(augmentation: T): T {
+  // Set enabled to true by default if not specified
+  if (augmentation.enabled === undefined) {
+    augmentation.enabled = true
+  }
+
+  // Add to the registry
+  availableAugmentations.push(augmentation)
+
+  return augmentation
+}
+
+/**
+ * Initializes the augmentation pipeline with all registered augmentations
+ *
+ * @param pipeline Optional custom pipeline to use instead of the default
+ * @returns The pipeline that was initialized
+ * @throws Error if no pipeline is provided and the default pipeline hasn't been set
+ */
+export function initializeAugmentationPipeline(
+  pipelineInstance?: IPipeline
+): IPipeline {
+  // Use the provided pipeline or fall back to the default
+  const pipeline = pipelineInstance || defaultPipeline
+
+  if (!pipeline) {
+    throw new Error('No pipeline provided and default pipeline not set. Call setDefaultPipeline first.')
+  }
+
+  // Register all augmentations with the pipeline
+  for (const augmentation of availableAugmentations) {
+    if (augmentation.enabled) {
+      pipeline.register(augmentation)
+    }
+  }
+
+  return pipeline
+}
+
+/**
+ * Enables or disables an augmentation by name
+ *
+ * @param name The name of the augmentation to enable/disable
+ * @param enabled Whether to enable or disable the augmentation
+ * @returns True if the augmentation was found and updated, false otherwise
+ */
+export function setAugmentationEnabled(name: string, enabled: boolean): boolean {
+  const augmentation = availableAugmentations.find(aug => aug.name === name)
+
+  if (augmentation) {
+    augmentation.enabled = enabled
+    return true
+  }
+
+  return false
+}
+
+/**
+ * Gets all augmentations of a specific type
+ *
+ * @param type The type of augmentation to get
+ * @returns An array of all augmentations of the specified type
+ */
+export function getAugmentationsByType(type: AugmentationType): IAugmentation[] {
+  return availableAugmentations.filter(aug => {
+    // Check if the augmentation is of the specified type
+    // This is a simplified check and may need to be updated based on how types are determined
+    switch (type) {
+      case AugmentationType.SENSE:
+        return 'processRawData' in aug && 'listenToFeed' in aug
+      case AugmentationType.CONDUIT:
+        return 'establishConnection' in aug && 'readData' in aug && 'writeData' in aug
+      case AugmentationType.COGNITION:
+        return 'reason' in aug && 'infer' in aug && 'executeLogic' in aug
+      case AugmentationType.MEMORY:
+        return 'storeData' in aug && 'retrieveData' in aug && 'updateData' in aug
+      case AugmentationType.PERCEPTION:
+        return 'interpret' in aug && 'organize' in aug && 'generateVisualization' in aug
+      case AugmentationType.DIALOG:
+        return 'processUserInput' in aug && 'generateResponse' in aug && 'manageContext' in aug
+      case AugmentationType.ACTIVATION:
+        return 'triggerAction' in aug && 'generateOutput' in aug && 'interactExternal' in aug
+      case AugmentationType.WEBSOCKET:
+        return 'connectWebSocket' in aug && 'sendWebSocketMessage' in aug && 'onWebSocketMessage' in aug
+      default:
+        return false
+    }
+  })
+}

src/augmentationRegistryLoader.ts

@@ -0,0 +1,291 @@
+/**
+ * Augmentation Registry Loader
+ *
+ * This module provides functionality for loading augmentation registrations
+ * at build time. It's designed to be used with build tools like webpack or rollup
+ * to automatically discover and register augmentations.
+ */
+
+import { IAugmentation } from './types/augmentations.js'
+import { registerAugmentation } from './augmentationRegistry.js'
+
+/**
+ * Options for the augmentation registry loader
+ */
+export interface AugmentationRegistryLoaderOptions {
+  /**
+   * Whether to automatically initialize the augmentations after loading
+   * @default false
+   */
+  autoInitialize?: boolean;
+
+  /**
+   * Whether to log debug information during loading
+   * @default false
+   */
+  debug?: boolean;
+}
+
+/**
+ * Default options for the augmentation registry loader
+ */
+const DEFAULT_OPTIONS: AugmentationRegistryLoaderOptions = {
+  autoInitialize: false,
+  debug: false
+}
+
+/**
+ * Result of loading augmentations
+ */
+export interface AugmentationLoadResult {
+  /**
+   * The augmentations that were loaded
+   */
+  augmentations: IAugmentation[];
+
+  /**
+   * Any errors that occurred during loading
+   */
+  errors: Error[];
+}
+
+/**
+ * Loads augmentations from the specified modules
+ *
+ * This function is designed to be used with build tools like webpack or rollup
+ * to automatically discover and register augmentations.
+ *
+ * @param modules An object containing modules with augmentations to register
+ * @param options Options for the loader
+ * @returns A promise that resolves with the result of loading the augmentations
+ *
+ * @example
+ * ```typescript
+ * // webpack.config.js
+ * const { AugmentationRegistryPlugin } = require('brainy/dist/webpack');
+ *
+ * module.exports = {
+ *   // ... other webpack config
+ *   plugins: [
+ *     new AugmentationRegistryPlugin({
+ *       // Pattern to match files containing augmentations
+ *       pattern: /augmentation\.js$/,
+ *       // Options for the loader
+ *       options: {
+ *         autoInitialize: true,
+ *         debug: true
+ *       }
+ *     })
+ *   ]
+ * };
+ * ```
+ */
+export async function loadAugmentationsFromModules(
+  modules: Record<string, any>,
+  options: AugmentationRegistryLoaderOptions = {}
+): Promise<AugmentationLoadResult> {
+  const opts = { ...DEFAULT_OPTIONS, ...options }
+  const result: AugmentationLoadResult = {
+    augmentations: [],
+    errors: []
+  }
+
+  if (opts.debug) {
+    console.log(`[AugmentationRegistryLoader] Loading augmentations from ${Object.keys(modules).length} modules`)
+  }
+
+  // Process each module
+  for (const [modulePath, module] of Object.entries(modules)) {
+    try {
+      if (opts.debug) {
+        console.log(`[AugmentationRegistryLoader] Processing module: ${modulePath}`)
+      }
+
+      // Extract augmentations from the module
+      const augmentations = extractAugmentationsFromModule(module)
+
+      if (augmentations.length === 0) {
+        if (opts.debug) {
+          console.log(`[AugmentationRegistryLoader] No augmentations found in module: ${modulePath}`)
+        }
+        continue
+      }
+
+      // Register each augmentation
+      for (const augmentation of augmentations) {
+        try {
+          const registered = registerAugmentation(augmentation)
+          result.augmentations.push(registered)
+
+          if (opts.debug) {
+            console.log(`[AugmentationRegistryLoader] Registered augmentation: ${registered.name}`)
+          }
+        } catch (error) {
+          const err = error instanceof Error ? error : new Error(String(error))
+          result.errors.push(err)
+
+          if (opts.debug) {
+            console.error(`[AugmentationRegistryLoader] Failed to register augmentation: ${err.message}`)
+          }
+        }
+      }
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error))
+      result.errors.push(err)
+
+      if (opts.debug) {
+        console.error(`[AugmentationRegistryLoader] Error processing module ${modulePath}: ${err.message}`)
+      }
+    }
+  }
+
+  if (opts.debug) {
+    console.log(`[AugmentationRegistryLoader] Loaded ${result.augmentations.length} augmentations with ${result.errors.length} errors`)
+  }
+
+  return result
+}
+
+/**
+ * Extracts augmentations from a module
+ *
+ * @param module The module to extract augmentations from
+ * @returns An array of augmentations found in the module
+ */
+function extractAugmentationsFromModule(module: any): IAugmentation[] {
+  const augmentations: IAugmentation[] = []
+
+  // If the module itself is an augmentation, add it
+  if (isAugmentation(module)) {
+    augmentations.push(module)
+  }
+
+  // Check for exported augmentations
+  if (module && typeof module === 'object') {
+    for (const key of Object.keys(module)) {
+      const exported = module[key]
+
+      // Skip non-objects and null
+      if (!exported || typeof exported !== 'object') {
+        continue
+      }
+
+      // If the exported value is an augmentation, add it
+      if (isAugmentation(exported)) {
+        augmentations.push(exported)
+      }
+
+      // If the exported value is an array of augmentations, add them
+      if (Array.isArray(exported) && exported.every(isAugmentation)) {
+        augmentations.push(...exported)
+      }
+    }
+  }
+
+  return augmentations
+}
+
+/**
+ * Checks if an object is an augmentation
+ *
+ * @param obj The object to check
+ * @returns True if the object is an augmentation
+ */
+function isAugmentation(obj: any): obj is IAugmentation {
+  return (
+    obj &&
+    typeof obj === 'object' &&
+    typeof obj.name === 'string' &&
+    typeof obj.initialize === 'function' &&
+    typeof obj.shutDown === 'function' &&
+    typeof obj.getStatus === 'function'
+  )
+}
+
+/**
+ * Creates a webpack plugin for automatically loading augmentations
+ *
+ * @param options Options for the plugin
+ * @returns A webpack plugin
+ *
+ * @example
+ * ```typescript
+ * // webpack.config.js
+ * const { createAugmentationRegistryPlugin } = require('brainy/dist/webpack');
+ *
+ * module.exports = {
+ *   // ... other webpack config
+ *   plugins: [
+ *     createAugmentationRegistryPlugin({
+ *       pattern: /augmentation\.js$/,
+ *       options: {
+ *         autoInitialize: true,
+ *         debug: true
+ *       }
+ *     })
+ *   ]
+ * };
+ * ```
+ */
+export function createAugmentationRegistryPlugin(options: {
+  /**
+   * Pattern to match files containing augmentations
+   */
+  pattern: RegExp;
+
+  /**
+   * Options for the loader
+   */
+  options?: AugmentationRegistryLoaderOptions;
+}) {
+  // This is just a placeholder - the actual implementation would depend on the build tool
+  return {
+    name: 'AugmentationRegistryPlugin',
+    pattern: options.pattern,
+    options: options.options || {}
+  }
+}
+
+/**
+ * Creates a rollup plugin for automatically loading augmentations
+ *
+ * @param options Options for the plugin
+ * @returns A rollup plugin
+ *
+ * @example
+ * ```typescript
+ * // rollup.config.js
+ * import { createAugmentationRegistryRollupPlugin } from 'brainy/dist/rollup';
+ *
+ * export default {
+ *   // ... other rollup config
+ *   plugins: [
+ *     createAugmentationRegistryRollupPlugin({
+ *       pattern: /augmentation\.js$/,
+ *       options: {
+ *         autoInitialize: true,
+ *         debug: true
+ *       }
+ *     })
+ *   ]
+ * };
+ * ```
+ */
+export function createAugmentationRegistryRollupPlugin(options: {
+  /**
+   * Pattern to match files containing augmentations
+   */
+  pattern: RegExp;
+
+  /**
+   * Options for the loader
+   */
+  options?: AugmentationRegistryLoaderOptions;
+}) {
+  // This is just a placeholder - the actual implementation would depend on the build tool
+  return {
+    name: 'augmentation-registry-rollup-plugin',
+    pattern: options.pattern,
+    options: options.options || {}
+  }
+}

src/augmentations/README.md

@@ -0,0 +1,251 @@
+<div align="center">
+<img src="../../brainy.png" alt="Brainy Logo" width="200"/>
+
+# Brainy Augmentations
+
+</div>
+
+This directory contains the augmentation implementations for Brainy. Augmentations are pluggable components that extend
+Brainy's functionality in various ways.
+
+## Available Augmentations
+
+### Conduit Augmentations
+
+Conduit augmentations provide data synchronization between Brainy instances.
+
+#### WebSocketConduitAugmentation
+
+A conduit augmentation that syncs Brainy instances using WebSockets. This is used for syncing between browsers and
+servers, or between servers.
+
+```javascript
+import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
+
+// Create a WebSocket conduit augmentation
+const wsConduit = await createConduitAugmentation('websocket', 'my-websocket-sync')
+
+// Register the augmentation with the pipeline
+augmentationPipeline.register(wsConduit)
+
+// Connect to another Brainy instance
+const connectionResult = await wsConduit.establishConnection(
+  'wss://your-websocket-server.com/brainy-sync',
+  { protocols: 'brainy-sync' }
+)
+```
+
+#### WebRTCConduitAugmentation
+
+A conduit augmentation that syncs Brainy instances using WebRTC. This is used for direct peer-to-peer syncing between
+browsers.
+
+```javascript
+import { createConduitAugmentation, augmentationPipeline } from '@soulcraft/brainy'
+
+// Create a WebRTC conduit augmentation
+const webrtcConduit = await createConduitAugmentation('webrtc', 'my-webrtc-sync')
+
+// Register the augmentation with the pipeline
+augmentationPipeline.register(webrtcConduit)
+
+// Connect to a peer
+const connectionResult = await webrtcConduit.establishConnection(
+  'peer-id-to-connect-to',
+  {
+    signalServerUrl: 'wss://your-signal-server.com',
+    localPeerId: 'my-peer-id',
+    iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
+  }
+)
+```
+
+#### ServerSearchConduitAugmentation
+
+A specialized conduit augmentation that provides functionality for searching a server-hosted Brainy instance and storing
+results locally. This allows you to:
+
+- Search a server-hosted Brainy instance from a browser
+- Store the search results in a local Brainy instance
+- Perform further searches against the local instance without needing to query the server again
+- Add data to both local and server instances
+
+```javascript
+import {
+  ServerSearchConduitAugmentation,
+  createServerSearchAugmentations,
+  augmentationPipeline
+} from '@soulcraft/brainy'
+
+// Using the factory function (recommended)
+const { conduit, activation, connection } = await createServerSearchAugmentations(
+  'wss://your-brainy-server.com/ws',
+  { protocols: 'brainy-sync' }
+)
+
+// Register the augmentations with the pipeline
+augmentationPipeline.register(conduit)
+augmentationPipeline.register(activation)
+
+// Search the server and store results locally
+const serverSearchResult = await conduit.searchServer(
+  connection.connectionId,
+  'your search query',
+  5 // limit
+)
+
+// Search the local instance
+const localSearchResult = await conduit.searchLocal('your search query', 5)
+
+// Perform a combined search (local first, then server if needed)
+const combinedSearchResult = await conduit.searchCombined(
+  connection.connectionId,
+  'your search query',
+  5
+)
+
+// Add data to both local and server
+const addResult = await conduit.addToBoth(
+  connection.connectionId,
+  'Text to add',
+  { /* metadata */ }
+)
+```
+
+### Activation Augmentations
+
+Activation augmentations dictate how Brainy initiates actions, responses, or data manipulations.
+
+#### ServerSearchActivationAugmentation
+
+An activation augmentation that provides actions for server search functionality. This works in conjunction with the
+ServerSearchConduitAugmentation to provide a complete solution for browser-server search.
+
+```javascript
+import {
+  ServerSearchActivationAugmentation,
+  createServerSearchAugmentations,
+  augmentationPipeline
+} from '@soulcraft/brainy'
+
+// Using the factory function (recommended)
+const { conduit, activation, connection } = await createServerSearchAugmentations(
+  'wss://your-brainy-server.com/ws',
+  { protocols: 'brainy-sync' }
+)
+
+// Register the augmentations with the pipeline
+augmentationPipeline.register(conduit)
+augmentationPipeline.register(activation)
+
+// Use the activation augmentation to search the server
+const serverSearchAction = activation.triggerAction('searchServer', {
+  connectionId: connection.connectionId,
+  query: 'your search query',
+  limit: 5
+})
+
+if (serverSearchAction.success) {
+  // The data property contains a promise that will resolve to the search results
+  const serverSearchResult = await serverSearchAction.data
+  console.log('Server search results:', serverSearchResult)
+}
+
+// Other available actions:
+// - 'connectToServer': Connect to a server
+// - 'searchLocal': Search the local instance
+// - 'searchCombined': Search both local and server
+// - 'addToBoth': Add data to both local and server
+```
+
+## Using the Augmentation Pipeline
+
+The augmentation pipeline provides a way to execute augmentations based on their type.
+
+```javascript
+import { augmentationPipeline } from '@soulcraft/brainy'
+
+// Execute a conduit augmentation
+const conduitResults = await augmentationPipeline.executeConduitPipeline(
+  'methodName',
+  [arg1, arg2, ...],
+  { /* options */ }
+)
+
+// Execute an activation augmentation
+const activationResults = await augmentationPipeline.executeActivationPipeline(
+  'methodName',
+  [arg1, arg2, ...],
+  { /* options */ }
+)
+```
+
+## Creating Custom Augmentations
+
+To create a custom augmentation, implement one of the augmentation interfaces:
+
+- `ISenseAugmentation`: For processing raw data
+- `IConduitAugmentation`: For data synchronization
+- `ICognitionAugmentation`: For reasoning and inference
+- `IMemoryAugmentation`: For data storage
+- `IPerceptionAugmentation`: For data interpretation and visualization
+- `IDialogAugmentation`: For natural language processing
+- `IActivationAugmentation`: For triggering actions
+
+Example:
+
+```javascript
+import { AugmentationType, IActivationAugmentation } from '@soulcraft/brainy'
+
+class MyCustomActivation implements IActivationAugmentation {
+  readonly
+  name = 'my-custom-activation'
+  readonly
+  description = 'My custom activation augmentation'
+  enabled = true
+
+  getType(): AugmentationType {
+    return AugmentationType.ACTIVATION
+  }
+
+  async initialize(): Promise<void> {
+    // Initialization code
+  }
+
+  async shutDown(): Promise<void> {
+    // Cleanup code
+  }
+
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return 'active'
+  }
+
+  triggerAction(actionName: string, parameters
+
+?:
+
+  Record<string, unknown>
+
+):
+
+  AugmentationResponse<unknown> {
+    // Implementation
+  }
+
+  generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown
+
+>> {
+  // Implementation
+}
+
+interactExternal(systemId
+:
+string, payload
+:
+Record < string, unknown >
+):
+AugmentationResponse < unknown > {
+  // Implementation
+}
+}
+```

src/augmentations/memoryAugmentations.ts

@@ -0,0 +1,333 @@
+import {
+  AugmentationType,
+  IMemoryAugmentation,
+  AugmentationResponse
+} from '../types/augmentations.js'
+import { StorageAdapter, Vector } from '../coreTypes.js'
+import { MemoryStorage } from '../storage/opfsStorage.js'
+import { FileSystemStorage } from '../storage/fileSystemStorage.js'
+import { OPFSStorage } from '../storage/opfsStorage.js'
+import { cosineDistance } from '../utils/distance.js'
+
+/**
+ * Base class for memory augmentations that wrap a StorageAdapter
+ */
+abstract class BaseMemoryAugmentation implements IMemoryAugmentation {
+  readonly name: string
+  readonly description: string = 'Base memory augmentation'
+  enabled: boolean = true
+  protected storage: StorageAdapter
+  protected isInitialized = false
+
+  constructor(name: string, storage: StorageAdapter) {
+    this.name = name
+    this.storage = storage
+  }
+
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      await this.storage.init()
+      this.isInitialized = true
+    } catch (error) {
+      console.error(`Failed to initialize ${this.name}:`, error)
+      throw new Error(`Failed to initialize ${this.name}: ${error}`)
+    }
+  }
+
+  async shutDown(): Promise<void> {
+    this.isInitialized = false
+  }
+
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return this.isInitialized ? 'active' : 'inactive'
+  }
+
+  async storeData(
+    key: string,
+    data: unknown,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<boolean>> {
+    await this.ensureInitialized()
+
+    try {
+      await this.storage.saveMetadata(key, data)
+      return { success: true, data: true }
+    } catch (error) {
+      console.error(`Failed to store data for key ${key}:`, error)
+      return {
+        success: false,
+        data: false,
+        error: `Failed to store data: ${error}`
+      }
+    }
+  }
+
+  async retrieveData(
+    key: string,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      const data = await this.storage.getMetadata(key)
+      return {
+        success: true,
+        data
+      }
+    } catch (error) {
+      console.error(`Failed to retrieve data for key ${key}:`, error)
+      return {
+        success: false,
+        data: null,
+        error: `Failed to retrieve data: ${error}`
+      }
+    }
+  }
+
+  async updateData(
+    key: string,
+    data: unknown,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<boolean>> {
+    await this.ensureInitialized()
+
+    try {
+      await this.storage.saveMetadata(key, data)
+      return { success: true, data: true }
+    } catch (error) {
+      console.error(`Failed to update data for key ${key}:`, error)
+      return {
+        success: false,
+        data: false,
+        error: `Failed to update data: ${error}`
+      }
+    }
+  }
+
+  async deleteData(
+    key: string,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<boolean>> {
+    await this.ensureInitialized()
+
+    try {
+      // There's no direct deleteMetadata method, so we save null
+      await this.storage.saveMetadata(key, null)
+      return { success: true, data: true }
+    } catch (error) {
+      console.error(`Failed to delete data for key ${key}:`, error)
+      return {
+        success: false,
+        data: false,
+        error: `Failed to delete data: ${error}`
+      }
+    }
+  }
+
+  async listDataKeys(
+    pattern?: string,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<string[]>> {
+    // This is a limitation of the current StorageAdapter interface
+    // It doesn't provide a way to list all metadata keys
+    // We could implement this in the future by extending the StorageAdapter interface
+    return {
+      success: false,
+      data: [],
+      error: 'listDataKeys is not supported by this storage adapter'
+    }
+  }
+
+  /**
+   * Searches for data in the storage using vector similarity.
+   * Implements the findNearest functionality by calculating distances client-side.
+   * @param query The query vector or data to search for
+   * @param k Number of results to return (default: 10)
+   * @param options Optional search options
+   */
+  async search(
+    query: unknown,
+    k: number = 10,
+    options?: Record<string, unknown>
+  ): Promise<AugmentationResponse<Array<{
+    id: string;
+    score: number;
+    data: unknown;
+  }>>> {
+    await this.ensureInitialized()
+
+    try {
+      // Check if query is a vector
+      let queryVector: Vector
+
+      if (Array.isArray(query) && query.every(item => typeof item === 'number')) {
+        queryVector = query as Vector
+      } else {
+        // If query is not a vector, we can't perform vector search
+        return {
+          success: false,
+          data: [],
+          error: 'Query must be a vector (array of numbers) for vector search'
+        }
+      }
+
+      // Get all nodes from storage
+      const nodes = await this.storage.getAllNouns()
+
+      // Calculate distances and prepare results
+      const results: Array<{
+        id: string;
+        score: number;
+        data: unknown;
+      }> = []
+
+      for (const node of nodes) {
+        // Skip nodes that don't have a vector
+        if (!node.vector || !Array.isArray(node.vector)) {
+          continue
+        }
+
+        // Get metadata for the node
+        const metadata = await this.storage.getMetadata(node.id)
+
+        // Calculate distance between query vector and node vector
+        const distance = cosineDistance(queryVector, node.vector)
+
+        // Convert distance to similarity score (1 - distance for cosine)
+        // This way higher scores are better (more similar)
+        const score = 1 - distance
+
+        results.push({
+          id: node.id,
+          score,
+          data: metadata
+        })
+      }
+
+      // Sort results by score (descending) and take top k
+      results.sort((a, b) => b.score - a.score)
+      const topResults = results.slice(0, k)
+
+      return {
+        success: true,
+        data: topResults
+      }
+    } catch (error) {
+      console.error(`Failed to search in storage:`, error)
+      return {
+        success: false,
+        data: [],
+        error: `Failed to search in storage: ${error}`
+      }
+    }
+  }
+
+  protected async ensureInitialized(): Promise<void> {
+    if (!this.isInitialized) {
+      await this.initialize()
+    }
+  }
+}
+
+/**
+ * Memory augmentation that uses in-memory storage
+ */
+export class MemoryStorageAugmentation extends BaseMemoryAugmentation {
+  readonly description = 'Memory augmentation that stores data in memory'
+  enabled = true
+
+  constructor(name: string) {
+    super(name, new MemoryStorage())
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.MEMORY
+  }
+}
+
+/**
+ * Memory augmentation that uses file system storage
+ */
+export class FileSystemStorageAugmentation extends BaseMemoryAugmentation {
+  readonly description = 'Memory augmentation that stores data in the file system'
+  enabled = true
+
+  constructor(name: string, rootDirectory?: string) {
+    super(name, new FileSystemStorage(rootDirectory))
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.MEMORY
+  }
+}
+
+/**
+ * Memory augmentation that uses OPFS (Origin Private File System) storage
+ */
+export class OPFSStorageAugmentation extends BaseMemoryAugmentation {
+  readonly description = 'Memory augmentation that stores data in the Origin Private File System'
+  enabled = true
+
+  constructor(name: string) {
+    super(name, new OPFSStorage())
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.MEMORY
+  }
+}
+
+/**
+ * Factory function to create the appropriate memory augmentation based on the environment
+ */
+export async function createMemoryAugmentation(
+  name: string,
+  options: {
+    storageType?: 'memory' | 'filesystem' | 'opfs'
+    rootDirectory?: string
+    requestPersistentStorage?: boolean
+  } = {}
+): Promise<IMemoryAugmentation> {
+  // If a specific storage type is requested, use that
+  if (options.storageType) {
+    switch (options.storageType) {
+      case 'memory':
+        return new MemoryStorageAugmentation(name)
+      case 'filesystem':
+        return new FileSystemStorageAugmentation(name, options.rootDirectory)
+      case 'opfs':
+        return new OPFSStorageAugmentation(name)
+    }
+  }
+
+  // Otherwise, select based on environment
+  // Use the global isNode variable from the environment detection
+  const isNodeEnv = globalThis.__ENV__?.isNode || (
+    typeof process !== 'undefined' &&
+    process.versions != null &&
+    process.versions.node != null
+  )
+
+  if (isNodeEnv) {
+    // In Node.js, use FileSystemStorage
+    return new FileSystemStorageAugmentation(name, options.rootDirectory)
+  } else {
+    // In browser, try OPFS first
+    const opfsStorage = new OPFSStorage()
+
+    if (opfsStorage.isOPFSAvailable()) {
+      // Request persistent storage if specified
+      if (options.requestPersistentStorage) {
+        await opfsStorage.requestPersistentStorage()
+      }
+      return new OPFSStorageAugmentation(name)
+    } else {
+      // Fall back to memory storage
+      return new MemoryStorageAugmentation(name)
+    }
+  }
+}

src/augmentations/serverSearchAugmentations.ts

@@ -0,0 +1,665 @@
+/**
+ * Server Search Augmentations
+ *
+ * This file implements conduit and activation augmentations for browser-server search functionality.
+ * It allows Brainy to search a server-hosted instance and store results locally.
+ */
+
+import {
+  AugmentationType,
+  IConduitAugmentation,
+  IActivationAugmentation,
+  IWebSocketSupport,
+  AugmentationResponse,
+  WebSocketConnection
+} from '../types/augmentations.js'
+import { WebSocketConduitAugmentation } from './conduitAugmentations.js'
+import { v4 as uuidv4 } from 'uuid'
+import { BrainyDataInterface } from '../types/brainyDataInterface.js'
+
+/**
+ * ServerSearchConduitAugmentation
+ *
+ * A specialized conduit augmentation that provides functionality for searching
+ * a server-hosted Brainy instance and storing results locally.
+ */
+export class ServerSearchConduitAugmentation extends WebSocketConduitAugmentation {
+  private localDb: BrainyDataInterface | null = null
+
+  constructor(name: string = 'server-search-conduit') {
+    super(name)
+    // this.description = 'Conduit augmentation for server-hosted Brainy search'
+  }
+
+  /**
+   * Initialize the augmentation
+   */
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      // Initialize the base conduit
+      await super.initialize()
+
+      // Local DB must be set before initialization
+      if (!this.localDb) {
+        throw new Error('Local database not set. Call setLocalDb before initializing.')
+      }
+
+      this.isInitialized = true
+    } catch (error) {
+      console.error(`Failed to initialize ${this.name}:`, error)
+      throw new Error(`Failed to initialize ${this.name}: ${error}`)
+    }
+  }
+
+  /**
+   * Set the local Brainy instance
+   * @param db The Brainy instance to use for local storage
+   */
+  setLocalDb(db: BrainyDataInterface): void {
+    this.localDb = db
+  }
+
+  /**
+   * Get the local Brainy instance
+   * @returns The local Brainy instance
+   */
+  getLocalDb(): BrainyDataInterface | null {
+    return this.localDb
+  }
+
+  /**
+   * Search the server-hosted Brainy instance and store results locally
+   * @param connectionId The ID of the established connection
+   * @param query The search query
+   * @param limit Maximum number of results to return
+   * @returns Search results
+   */
+  async searchServer(
+    connectionId: string,
+    query: string,
+    limit: number = 10
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      // Create a search request
+      const readResult = await this.readData({
+        connectionId,
+        query: {
+          type: 'search',
+          query,
+          limit
+        }
+      })
+
+      if (readResult.success && readResult.data) {
+        const searchResults = readResult.data as any[]
+
+        // Store the results in the local Brainy instance
+        if (this.localDb) {
+          for (const result of searchResults) {
+            // Check if the noun already exists in the local database
+            const existingNoun = await this.localDb.get(result.id)
+
+            if (!existingNoun) {
+              // Add the noun to the local database
+              await this.localDb.add(result.vector, result.metadata)
+            }
+          }
+        }
+
+        return {
+          success: true,
+          data: searchResults
+        }
+      } else {
+        return {
+          success: false,
+          data: null,
+          error: readResult.error || 'Unknown error searching server'
+        }
+      }
+    } catch (error) {
+      console.error('Error searching server:', error)
+      return {
+        success: false,
+        data: null,
+        error: `Error searching server: ${error}`
+      }
+    }
+  }
+
+  /**
+   * Search the local Brainy instance
+   * @param query The search query
+   * @param limit Maximum number of results to return
+   * @returns Search results
+   */
+  async searchLocal(
+    query: string,
+    limit: number = 10
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      if (!this.localDb) {
+        return {
+          success: false,
+          data: null,
+          error: 'Local database not initialized'
+        }
+      }
+
+      const results = await this.localDb.searchText(query, limit)
+
+      return {
+        success: true,
+        data: results
+      }
+    } catch (error) {
+      console.error('Error searching local database:', error)
+      return {
+        success: false,
+        data: null,
+        error: `Error searching local database: ${error}`
+      }
+    }
+  }
+
+  /**
+   * Search both server and local instances, combine results, and store server results locally
+   * @param connectionId The ID of the established connection
+   * @param query The search query
+   * @param limit Maximum number of results to return
+   * @returns Combined search results
+   */
+  async searchCombined(
+    connectionId: string,
+    query: string,
+    limit: number = 10
+  ): Promise<AugmentationResponse<unknown>> {
+    await this.ensureInitialized()
+
+    try {
+      // Search local first
+      const localSearchResult = await this.searchLocal(query, limit)
+
+      if (!localSearchResult.success) {
+        return localSearchResult
+      }
+
+      const localResults = localSearchResult.data as any[]
+
+      // If we have enough local results, return them
+      if (localResults.length >= limit) {
+        return localSearchResult
+      }
+
+      // Otherwise, search server for additional results
+      const serverSearchResult = await this.searchServer(
+        connectionId,
+        query,
+        limit - localResults.length
+      )
+
+      if (!serverSearchResult.success) {
+        // If server search fails, return local results
+        return localSearchResult
+      }
+
+      const serverResults = serverSearchResult.data as any[]
+
+      // Combine results, removing duplicates
+      const combinedResults = [...localResults]
+      const localIds = new Set(localResults.map(r => r.id))
+
+      for (const result of serverResults) {
+        if (!localIds.has(result.id)) {
+          combinedResults.push(result)
+        }
+      }
+
+      return {
+        success: true,
+        data: combinedResults
+      }
+    } catch (error) {
+      console.error('Error performing combined search:', error)
+      return {
+        success: false,
+        data: null,
+        error: `Error performing combined search: ${error}`
+      }
+    }
+  }
+
+  /**
+   * Add data to both local and server instances
+   * @param connectionId The ID of the established connection
+   * @param data Text or vector to add
+   * @param metadata Metadata for the data
+   * @returns ID of the added data
+   */
+  async addToBoth(
+    connectionId: string,
+    data: string | any[],
+    metadata: any = {}
+  ): Promise<AugmentationResponse<string>> {
+    await this.ensureInitialized()
+
+    try {
+      if (!this.localDb) {
+        return {
+          success: false,
+          data: '',
+          error: 'Local database not initialized'
+        }
+      }
+
+      // Add to local first
+      const id = await this.localDb.add(data, metadata)
+
+      // Get the vector and metadata
+      const noun = await this.localDb.get(id) as import('../coreTypes.js').VectorDocument<unknown>
+
+      if (!noun) {
+        return {
+          success: false,
+          data: '',
+          error: 'Failed to retrieve newly created noun'
+        }
+      }
+
+      // Add to server
+      const writeResult = await this.writeData({
+        connectionId,
+        data: {
+          type: 'addNoun',
+          vector: noun.vector,
+          metadata: noun.metadata
+        }
+      })
+
+      if (!writeResult.success) {
+        return {
+          success: true,
+          data: id,
+          error: `Added locally but failed to add to server: ${writeResult.error}`
+        }
+      }
+
+      return {
+        success: true,
+        data: id
+      }
+    } catch (error) {
+      console.error('Error adding data to both:', error)
+      return {
+        success: false,
+        data: '',
+        error: `Error adding data to both: ${error}`
+      }
+    }
+  }
+}
+
+/**
+ * ServerSearchActivationAugmentation
+ *
+ * An activation augmentation that provides actions for server search functionality.
+ */
+export class ServerSearchActivationAugmentation implements IActivationAugmentation {
+  readonly name: string
+  readonly description: string
+  enabled: boolean = true
+  private isInitialized = false
+  private conduitAugmentation: ServerSearchConduitAugmentation | null = null
+  private connections: Map<string, WebSocketConnection> = new Map()
+
+  constructor(name: string = 'server-search-activation') {
+    this.name = name
+    this.description = 'Activation augmentation for server-hosted Brainy search'
+  }
+
+  getType(): AugmentationType {
+    return AugmentationType.ACTIVATION
+  }
+
+  /**
+   * Initialize the augmentation
+   */
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    this.isInitialized = true
+  }
+
+  /**
+   * Shut down the augmentation
+   */
+  async shutDown(): Promise<void> {
+    this.isInitialized = false
+  }
+
+  /**
+   * Get the status of the augmentation
+   */
+  async getStatus(): Promise<'active' | 'inactive' | 'error'> {
+    return this.isInitialized ? 'active' : 'inactive'
+  }
+
+  /**
+   * Set the conduit augmentation to use for server search
+   * @param conduit The ServerSearchConduitAugmentation to use
+   */
+  setConduitAugmentation(conduit: ServerSearchConduitAugmentation): void {
+    this.conduitAugmentation = conduit
+  }
+
+  /**
+   * Store a connection for later use
+   * @param connectionId The ID to use for the connection
+   * @param connection The WebSocket connection
+   */
+  storeConnection(connectionId: string, connection: WebSocketConnection): void {
+    this.connections.set(connectionId, connection)
+  }
+
+  /**
+   * Get a stored connection
+   * @param connectionId The ID of the connection to retrieve
+   * @returns The WebSocket connection
+   */
+  getConnection(connectionId: string): WebSocketConnection | undefined {
+    return this.connections.get(connectionId)
+  }
+
+  /**
+   * Trigger an action based on a processed command or internal state
+   * @param actionName The name of the action to trigger
+   * @param parameters Optional parameters for the action
+   */
+  triggerAction(
+    actionName: string,
+    parameters?: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    if (!this.conduitAugmentation) {
+      return {
+        success: false,
+        data: null,
+        error: 'Conduit augmentation not set'
+      }
+    }
+
+    // Handle different actions
+    switch (actionName) {
+      case 'connectToServer':
+        return this.handleConnectToServer(parameters || {})
+      case 'searchServer':
+        return this.handleSearchServer(parameters || {})
+      case 'searchLocal':
+        return this.handleSearchLocal(parameters || {})
+      case 'searchCombined':
+        return this.handleSearchCombined(parameters || {})
+      case 'addToBoth':
+        return this.handleAddToBoth(parameters || {})
+      default:
+        return {
+          success: false,
+          data: null,
+          error: `Unknown action: ${actionName}`
+        }
+    }
+  }
+
+  /**
+   * Handle the connectToServer action
+   * @param parameters Action parameters
+   */
+  private handleConnectToServer(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const serverUrl = parameters.serverUrl as string
+    const protocols = parameters.protocols as string | string[] | undefined
+
+    if (!serverUrl) {
+      return {
+        success: false,
+        data: null,
+        error: 'serverUrl parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the connection is established
+    return {
+      success: true,
+      data: this.conduitAugmentation!.establishConnection(serverUrl, {
+        protocols
+      })
+    }
+  }
+
+  /**
+   * Handle the searchServer action
+   * @param parameters Action parameters
+   */
+  private handleSearchServer(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const connectionId = parameters.connectionId as string
+    const query = parameters.query as string
+    const limit = parameters.limit as number || 10
+
+    if (!connectionId) {
+      return {
+        success: false,
+        data: null,
+        error: 'connectionId parameter is required'
+      }
+    }
+
+    if (!query) {
+      return {
+        success: false,
+        data: null,
+        error: 'query parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the search is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.searchServer(connectionId, query, limit)
+    }
+  }
+
+  /**
+   * Handle the searchLocal action
+   * @param parameters Action parameters
+   */
+  private handleSearchLocal(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const query = parameters.query as string
+    const limit = parameters.limit as number || 10
+
+    if (!query) {
+      return {
+        success: false,
+        data: null,
+        error: 'query parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the search is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.searchLocal(query, limit)
+    }
+  }
+
+  /**
+   * Handle the searchCombined action
+   * @param parameters Action parameters
+   */
+  private handleSearchCombined(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const connectionId = parameters.connectionId as string
+    const query = parameters.query as string
+    const limit = parameters.limit as number || 10
+
+    if (!connectionId) {
+      return {
+        success: false,
+        data: null,
+        error: 'connectionId parameter is required'
+      }
+    }
+
+    if (!query) {
+      return {
+        success: false,
+        data: null,
+        error: 'query parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the search is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.searchCombined(connectionId, query, limit)
+    }
+  }
+
+  /**
+   * Handle the addToBoth action
+   * @param parameters Action parameters
+   */
+  private handleAddToBoth(
+    parameters: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    const connectionId = parameters.connectionId as string
+    const data = parameters.data
+    const metadata = parameters.metadata || {}
+
+    if (!connectionId) {
+      return {
+        success: false,
+        data: null,
+        error: 'connectionId parameter is required'
+      }
+    }
+
+    if (!data) {
+      return {
+        success: false,
+        data: null,
+        error: 'data parameter is required'
+      }
+    }
+
+    // Return a promise that will be resolved when the add is complete
+    return {
+      success: true,
+      data: this.conduitAugmentation!.addToBoth(connectionId, data as any, metadata as any)
+    }
+  }
+
+  /**
+   * Generates an expressive output or response from Brainy
+   * @param knowledgeId The identifier of the knowledge to express
+   * @param format The desired output format (e.g., 'text', 'json')
+   */
+  generateOutput(
+    knowledgeId: string,
+    format: string
+  ): AugmentationResponse<string | Record<string, unknown>> {
+    // This method is not used for server search functionality
+    return {
+      success: false,
+      data: '',
+      error: 'generateOutput is not implemented for ServerSearchActivationAugmentation'
+    }
+  }
+
+  /**
+   * Interacts with an external system or API
+   * @param systemId The identifier of the external system
+   * @param payload The data to send to the external system
+   */
+  interactExternal(
+    systemId: string,
+    payload: Record<string, unknown>
+  ): AugmentationResponse<unknown> {
+    // This method is not used for server search functionality
+    return {
+      success: false,
+      data: null,
+      error: 'interactExternal is not implemented for ServerSearchActivationAugmentation'
+    }
+  }
+}
+
+/**
+ * Factory function to create server search augmentations
+ * @param serverUrl The URL of the server to connect to
+ * @param options Additional options
+ * @returns An object containing the created augmentations
+ */
+export async function createServerSearchAugmentations(
+  serverUrl: string,
+  options: {
+    conduitName?: string,
+    activationName?: string,
+    protocols?: string | string[],
+    localDb?: BrainyDataInterface
+  } = {}
+): Promise<{
+  conduit: ServerSearchConduitAugmentation,
+  activation: ServerSearchActivationAugmentation,
+  connection: WebSocketConnection
+}> {
+  // Create the conduit augmentation
+  const conduit = new ServerSearchConduitAugmentation(options.conduitName)
+  await conduit.initialize()
+
+  // Set the local database if provided
+  if (options.localDb) {
+    conduit.setLocalDb(options.localDb)
+  }
+
+  // Create the activation augmentation
+  const activation = new ServerSearchActivationAugmentation(options.activationName)
+  await activation.initialize()
+
+  // Link the augmentations
+  activation.setConduitAugmentation(conduit)
+
+  // Connect to the server
+  const connectionResult = await conduit.establishConnection(
+    serverUrl,
+    { protocols: options.protocols }
+  )
+
+  if (!connectionResult.success || !connectionResult.data) {
+    throw new Error(`Failed to connect to server: ${connectionResult.error}`)
+  }
+
+  const connection = connectionResult.data
+
+  // Store the connection in the activation augmentation
+  activation.storeConnection(connection.connectionId, connection)
+
+  return {
+    conduit,
+    activation,
+    connection
+  }
+}

src/coreTypes.ts

@@ -0,0 +1,163 @@
+/**
+ * Type definitions for the Soulcraft Brainy
+ */
+
+/**
+ * Vector representation - an array of numbers
+ */
+export type Vector = number[]
+
+/**
+ * A document with a vector embedding and optional metadata
+ */
+export interface VectorDocument<T = any> {
+  id: string
+  vector: Vector
+  metadata?: T
+}
+
+/**
+ * Search result with similarity score
+ */
+export interface SearchResult<T = any> {
+  id: string
+  score: number
+  vector: Vector
+  metadata?: T
+}
+
+/**
+ * Distance function for comparing vectors
+ */
+export type DistanceFunction = (a: Vector, b: Vector) => number
+
+/**
+ * Embedding function for converting data to vectors
+ */
+export type EmbeddingFunction = (data: any) => Promise<Vector>
+
+/**
+ * Embedding model interface
+ */
+export interface EmbeddingModel {
+  /**
+   * Initialize the embedding model
+   */
+  init(): Promise<void>
+
+  /**
+   * Embed data into a vector
+   */
+  embed(data: any): Promise<Vector>
+
+  /**
+   * Dispose of the model resources
+   */
+  dispose(): Promise<void>
+}
+
+/**
+ * HNSW graph noun
+ */
+export interface HNSWNoun {
+  id: string
+  vector: Vector
+  connections: Map<number, Set<string>> // level -> set of connected noun ids
+}
+
+/**
+ * Verb representing a relationship between nouns
+ * Extends HNSWNoun to allow verbs to be first-class entities in the data model
+ */
+export interface GraphVerb extends HNSWNoun {
+  sourceId: string // ID of the source noun
+  targetId: string // ID of the target noun
+  type?: string // Optional type of the relationship
+  weight?: number // Optional weight of the relationship
+  metadata?: any // Optional metadata for the verb
+
+  // Additional properties used in the codebase
+  source?: string // Alias for sourceId
+  target?: string // Alias for targetId
+  verb?: string // Alias for type
+  data?: Record<string, any> // Additional flexible data storage
+  embedding?: Vector // Vector representation of the relationship
+}
+
+/**
+ * HNSW index configuration
+ */
+export interface HNSWConfig {
+  M: number // Maximum number of connections per noun
+  efConstruction: number // Size of the dynamic candidate list during construction
+  efSearch: number // Size of the dynamic candidate list during search
+  ml: number // Maximum level
+}
+
+/**
+ * Storage interface for persistence
+ */
+export interface StorageAdapter {
+  init(): Promise<void>
+
+  saveNoun(noun: HNSWNoun): Promise<void>
+
+  getNoun(id: string): Promise<HNSWNoun | null>
+
+  getAllNouns(): Promise<HNSWNoun[]>
+
+  /**
+   * Get nouns by noun type
+   * @param nounType The noun type to filter by
+   * @returns Promise that resolves to an array of nouns of the specified noun type
+   */
+  getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
+
+  deleteNoun(id: string): Promise<void>
+
+  saveVerb(verb: GraphVerb): Promise<void>
+
+  getVerb(id: string): Promise<GraphVerb | null>
+
+  getAllVerbs(): Promise<GraphVerb[]>
+
+  getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
+
+  getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
+
+  getVerbsByType(type: string): Promise<GraphVerb[]>
+
+  deleteVerb(id: string): Promise<void>
+
+  saveMetadata(id: string, metadata: any): Promise<void>
+
+  getMetadata(id: string): Promise<any | null>
+
+  clear(): Promise<void>
+
+  /**
+   * Get information about storage usage and capacity
+   * @returns Promise that resolves to an object containing storage status information
+   */
+  getStorageStatus(): Promise<{
+    /**
+     * The type of storage being used (e.g., 'filesystem', 'opfs', 'memory')
+     */
+    type: string
+
+    /**
+     * The amount of storage being used in bytes
+     */
+    used: number
+
+    /**
+     * The total amount of storage available in bytes, or null if unknown
+     */
+    quota: number | null
+
+    /**
+     * Additional storage-specific information
+     */
+    details?: Record<string, any>
+  }>
+}

src/examples/basicUsage.ts

@@ -0,0 +1,163 @@
+/**
+ * Basic usage example for the Soulcraft Brainy database
+ */
+
+import { BrainyData } from '../brainyData.js'
+
+// Example data - word embeddings
+const wordEmbeddings = {
+  cat: [0.2, 0.3, 0.4, 0.1],
+  dog: [0.3, 0.2, 0.4, 0.2],
+  fish: [0.1, 0.1, 0.8, 0.2],
+  bird: [0.1, 0.4, 0.2, 0.5],
+  tiger: [0.3, 0.4, 0.3, 0.1],
+  lion: [0.4, 0.3, 0.2, 0.1],
+  shark: [0.2, 0.1, 0.7, 0.3],
+  eagle: [0.2, 0.5, 0.1, 0.4]
+}
+
+// Example metadata
+const metadata = {
+  cat: { type: 'mammal', domesticated: true },
+  dog: { type: 'mammal', domesticated: true },
+  fish: { type: 'fish', domesticated: false },
+  bird: { type: 'bird', domesticated: false },
+  tiger: { type: 'mammal', domesticated: false },
+  lion: { type: 'mammal', domesticated: false },
+  shark: { type: 'fish', domesticated: false },
+  eagle: { type: 'bird', domesticated: false }
+}
+
+/**
+ * Run the example
+ */
+async function runExample() {
+  console.log('Initializing vector database...')
+
+  // Create a new vector database
+  const db = new BrainyData()
+  await db.init()
+
+  console.log('Adding vectors to the database...')
+
+  // Add vectors to the database
+  const ids: Record<string, string> = {}
+  const metadata: Record<string, { type: string; domesticated: boolean }> = {
+    cat: { type: 'mammal', domesticated: true },
+    dog: { type: 'mammal', domesticated: true },
+    fish: { type: 'fish', domesticated: false },
+    bird: { type: 'bird', domesticated: false },
+    tiger: { type: 'mammal', domesticated: false },
+    lion: { type: 'mammal', domesticated: false },
+    shark: { type: 'fish', domesticated: false },
+    eagle: { type: 'bird', domesticated: false }
+  }
+  for (const [word, vector] of Object.entries(wordEmbeddings)) {
+    ids[word] = await db.add(vector, metadata[word])
+
+    console.log(`Added "${word}" with ID: ${ids[word]}`)
+  }
+
+  console.log('\nDatabase size:', db.size())
+
+  // Search for similar vectors
+  console.log('\nSearching for vectors similar to "cat"...')
+  const catResults = await db.search(wordEmbeddings['cat'], 3)
+  console.log('Results:')
+  for (const result of catResults) {
+    const word =
+      Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
+    console.log(
+      `- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
+      result.metadata,
+      ')'
+    )
+  }
+
+  // Search for similar vectors
+  console.log('\nSearching for vectors similar to "fish"...')
+  const fishResults = await db.search(wordEmbeddings['fish'], 3)
+  console.log('Results:')
+  for (const result of fishResults) {
+    const word =
+      Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
+    console.log(
+      `- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
+      result.metadata,
+      ')'
+    )
+  }
+
+  // Update metadata
+  console.log('\nUpdating metadata for "bird"...')
+  await db.updateMetadata(ids['bird'], {
+    ...metadata['bird'],
+    notes: 'Can fly'
+  })
+
+  // Get the updated document
+  const birdDoc = await db.get(ids['bird'])
+  console.log('Updated bird document:', birdDoc)
+
+  // Delete a vector
+  console.log('\nDeleting "shark"...')
+  await db.delete(ids['shark'])
+  console.log('Database size after deletion:', db.size())
+
+  // Search again to verify shark is gone
+  console.log('\nSearching for vectors similar to "fish" after deletion...')
+  const fishResultsAfterDeletion = await db.search(wordEmbeddings['fish'], 3)
+  console.log('Results:')
+  for (const result of fishResultsAfterDeletion) {
+    const word =
+      Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
+    console.log(
+      `- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
+      result.metadata,
+      ')'
+    )
+  }
+
+  console.log('\nExample completed successfully!')
+}
+
+// Check if we're in a browser or Node.js environment
+if (typeof window !== 'undefined') {
+  // Browser environment
+  document.addEventListener('DOMContentLoaded', () => {
+    const button = document.createElement('button')
+    button.textContent = 'Run BrainyData Example'
+    button.addEventListener('click', async () => {
+      const output = document.createElement('pre')
+      document.body.appendChild(output)
+
+      // Redirect console.log to the output element
+      const originalLog = console.log
+      console.log = (...args) => {
+        originalLog(...args)
+        output.textContent +=
+          args
+            .map((arg) =>
+              typeof arg === 'object' ? JSON.stringify(arg, null, 2) : arg
+            )
+            .join(' ') + '\n'
+      }
+
+      try {
+        await runExample()
+      } catch (error) {
+        console.error('Error running example:', error)
+      }
+
+      // Restore console.log
+      console.log = originalLog
+    })
+
+    document.body.appendChild(button)
+  })
+} else {
+  // Node.js environment
+  runExample().catch((error) => {
+    console.error('Error running example:', error)
+  })
+}

src/global.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Global type declarations for Brainy
+ */
+
+// Extend the globalThis interface to include the __ENV__ property
+declare global {
+  var __ENV__: {
+    isBrowser: boolean;
+    isNode: string | false;
+    isServerless: boolean;
+  };
+}
+
+// This export is needed to make this file a module
+export {};

src/hnsw/hnswIndex.ts

@@ -0,0 +1,704 @@
+/**
+ * HNSW (Hierarchical Navigable Small World) Index implementation
+ * Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
+ */
+
+import { DistanceFunction, HNSWConfig, HNSWNoun, Vector, VectorDocument } from '../coreTypes.js'
+import { euclideanDistance } from '../utils/index.js'
+import { executeInThread } from '../utils/workerUtils.js'
+
+// Default HNSW parameters
+const DEFAULT_CONFIG: HNSWConfig = {
+  M: 16, // Max number of connections per noun
+  efConstruction: 200, // Size of a dynamic candidate list during construction
+  efSearch: 50, // Size of a dynamic candidate list during search
+  ml: 16 // Max level
+}
+
+export class HNSWIndex {
+  private nouns: Map<string, HNSWNoun> = new Map()
+  private entryPointId: string | null = null
+  private maxLevel = 0
+  private config: HNSWConfig
+  private distanceFunction: DistanceFunction
+  private dimension: number | null = null
+  private useParallelization: boolean = true // Whether to use parallelization for performance-critical operations
+
+  constructor(
+    config: Partial<HNSWConfig> = {},
+    distanceFunction: DistanceFunction = euclideanDistance,
+    options: { useParallelization?: boolean } = {}
+  ) {
+    this.config = { ...DEFAULT_CONFIG, ...config }
+    this.distanceFunction = distanceFunction
+    this.useParallelization = options.useParallelization !== undefined ? options.useParallelization : true
+  }
+
+  /**
+   * Set whether to use parallelization for performance-critical operations
+   */
+  public setUseParallelization(useParallelization: boolean): void {
+    this.useParallelization = useParallelization
+  }
+
+  /**
+   * Get whether parallelization is enabled
+   */
+  public getUseParallelization(): boolean {
+    return this.useParallelization
+  }
+
+  /**
+   * Calculate distances between a query vector and multiple vectors in parallel
+   * This is used to optimize performance for search operations
+   * @param queryVector The query vector
+   * @param vectors Array of vectors to compare against
+   * @returns Array of distances
+   */
+  private async calculateDistancesInParallel(
+    queryVector: Vector,
+    vectors: Array<{ id: string; vector: Vector }>
+  ): Promise<Array<{ id: string; distance: number }>> {
+    // If parallelization is disabled or there are very few vectors, use sequential processing
+    if (!this.useParallelization || vectors.length < 10) {
+      return vectors.map(item => ({
+        id: item.id,
+        distance: this.distanceFunction(queryVector, item.vector)
+      }))
+    }
+
+    // Function to be executed in a worker thread
+    const distanceCalculator = (
+      args: {
+        queryVector: Vector,
+        vectors: Array<{ id: string; vector: Vector }>,
+        distanceFnString: string
+      }
+    ) => {
+      const { queryVector, vectors, distanceFnString } = args;
+
+      // Recreate the distance function from its string representation
+      const distanceFunction = new Function('return ' + distanceFnString)() as DistanceFunction
+
+      // Calculate distances for all items
+      return vectors.map(item => ({
+        id: item.id,
+        distance: distanceFunction(queryVector, item.vector)
+      }))
+    }
+
+    try {
+      // Convert the distance function to a string for serialization
+      const distanceFnString = this.distanceFunction.toString()
+
+      // Execute the distance calculation in a separate thread
+      return await executeInThread<Array<{ id: string; distance: number }>>(
+        distanceCalculator.toString(),
+        { queryVector, vectors, distanceFnString }
+      )
+    } catch (error) {
+      console.error('Error in parallel distance calculation, falling back to sequential:', error)
+      // Fall back to sequential processing if parallel execution fails
+      return vectors.map(item => ({
+        id: item.id,
+        distance: this.distanceFunction(queryVector, item.vector)
+      }))
+    }
+  }
+
+  /**
+   * Add a vector to the index
+   */
+  public async addItem(item: VectorDocument): Promise<string> {
+    // Check if item is defined
+    if (!item) {
+      throw new Error('Item is undefined or null')
+    }
+
+    const { id, vector } = item
+
+    // Check if vector is defined
+    if (!vector) {
+      throw new Error('Vector is undefined or null')
+    }
+
+    // Set dimension on first insert
+    if (this.dimension === null) {
+      this.dimension = vector.length
+    } else if (vector.length !== this.dimension) {
+      throw new Error(
+        `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`
+      )
+    }
+
+    // Generate random level for this noun
+    const nounLevel = this.getRandomLevel()
+
+    // Create new noun
+    const noun: HNSWNoun = {
+      id,
+      vector,
+      connections: new Map()
+    }
+
+    // Initialize empty connection sets for each level
+    for (let level = 0; level <= nounLevel; level++) {
+      noun.connections.set(level, new Set<string>())
+    }
+
+    // If this is the first noun, make it the entry point
+    if (this.nouns.size === 0) {
+      this.entryPointId = id
+      this.maxLevel = nounLevel
+      this.nouns.set(id, noun)
+      return id
+    }
+
+    // Find entry point
+    if (!this.entryPointId) {
+      console.error('Entry point ID is null')
+      // If there's no entry point, this is the first noun, so we should have returned earlier
+      // This is a safety check
+      this.entryPointId = id
+      this.maxLevel = nounLevel
+      this.nouns.set(id, noun)
+      return id
+    }
+
+    const entryPoint = this.nouns.get(this.entryPointId)
+    if (!entryPoint) {
+      console.error(`Entry point with ID ${this.entryPointId} not found`)
+      // If the entry point doesn't exist, treat this as the first noun
+      this.entryPointId = id
+      this.maxLevel = nounLevel
+      this.nouns.set(id, noun)
+      return id
+    }
+
+    let currObj = entryPoint
+    let currDist = this.distanceFunction(vector, entryPoint.vector)
+
+    // Traverse the graph from top to bottom to find the closest noun
+    for (let level = this.maxLevel; level > nounLevel; level--) {
+      let changed = true
+      while (changed) {
+        changed = false
+
+        // Check all neighbors at current level
+        const connections = currObj.connections.get(level) || new Set<string>()
+
+        for (const neighborId of connections) {
+          const neighbor = this.nouns.get(neighborId)
+          if (!neighbor) {
+            // Skip neighbors that don't exist (expected during rapid additions/deletions)
+            continue
+          }
+          const distToNeighbor = this.distanceFunction(vector, neighbor.vector)
+
+          if (distToNeighbor < currDist) {
+            currDist = distToNeighbor
+            currObj = neighbor
+            changed = true
+          }
+        }
+      }
+    }
+
+    // For each level from nounLevel down to 0
+    for (let level = Math.min(nounLevel, this.maxLevel); level >= 0; level--) {
+      // Find ef nearest elements using greedy search
+      const nearestNouns = await this.searchLayer(
+        vector,
+        currObj,
+        this.config.efConstruction,
+        level
+      )
+
+      // Select M nearest neighbors
+      const neighbors = this.selectNeighbors(
+        vector,
+        nearestNouns,
+        this.config.M
+      )
+
+      // Add bidirectional connections
+      for (const [neighborId, _] of neighbors) {
+        const neighbor = this.nouns.get(neighborId)
+        if (!neighbor) {
+          // Skip neighbors that don't exist (expected during rapid additions/deletions)
+          continue
+        }
+
+        noun.connections.get(level)!.add(neighborId)
+
+        // Add reverse connection
+        if (!neighbor.connections.has(level)) {
+          neighbor.connections.set(level, new Set<string>())
+        }
+        neighbor.connections.get(level)!.add(id)
+
+        // Ensure neighbor doesn't have too many connections
+        if (neighbor.connections.get(level)!.size > this.config.M) {
+          this.pruneConnections(neighbor, level)
+        }
+      }
+
+      // Update entry point for the next level
+      if (nearestNouns.size > 0) {
+        const [nearestId, nearestDist] = [...nearestNouns][0]
+        if (nearestDist < currDist) {
+          currDist = nearestDist
+          const nearestNoun = this.nouns.get(nearestId)
+          if (!nearestNoun) {
+            console.error(`Nearest noun with ID ${nearestId} not found in addItem`)
+            // Keep the current object as is
+          } else {
+            currObj = nearestNoun
+          }
+        }
+      }
+    }
+
+    // Update max level and entry point if needed
+    if (nounLevel > this.maxLevel) {
+      this.maxLevel = nounLevel
+      this.entryPointId = id
+    }
+
+    // Add noun to the index
+    this.nouns.set(id, noun)
+    return id
+  }
+
+  /**
+   * Search for nearest neighbors
+   */
+  public async search(queryVector: Vector, k: number = 10): Promise<Array<[string, number]>> {
+    if (this.nouns.size === 0) {
+      return []
+    }
+
+    // Check if query vector is defined
+    if (!queryVector) {
+      throw new Error('Query vector is undefined or null')
+    }
+
+    if (this.dimension !== null && queryVector.length !== this.dimension) {
+      throw new Error(
+        `Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`
+      )
+    }
+
+    // Start from the entry point
+    if (!this.entryPointId) {
+      console.error('Entry point ID is null')
+      return []
+    }
+
+    const entryPoint = this.nouns.get(this.entryPointId)
+    if (!entryPoint) {
+      console.error(`Entry point with ID ${this.entryPointId} not found`)
+      return []
+    }
+
+    let currObj = entryPoint
+    let currDist = this.distanceFunction(queryVector, currObj.vector)
+
+    // Traverse the graph from top to bottom to find the closest noun
+    for (let level = this.maxLevel; level > 0; level--) {
+      let changed = true
+      while (changed) {
+        changed = false
+
+        // Check all neighbors at current level
+        const connections = currObj.connections.get(level) || new Set<string>()
+
+        // If we have enough connections, use parallel distance calculation
+        if (this.useParallelization && connections.size >= 10) {
+          // Prepare vectors for parallel calculation
+          const vectors: Array<{ id: string; vector: Vector }> = []
+          for (const neighborId of connections) {
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) continue
+            vectors.push({ id: neighborId, vector: neighbor.vector })
+          }
+
+          // Calculate distances in parallel
+          const distances = await this.calculateDistancesInParallel(queryVector, vectors)
+
+          // Find the closest neighbor
+          for (const { id, distance } of distances) {
+            if (distance < currDist) {
+              currDist = distance
+              const neighbor = this.nouns.get(id)
+              if (neighbor) {
+                currObj = neighbor
+                changed = true
+              }
+            }
+          }
+        } else {
+          // Use sequential processing for small number of connections
+          for (const neighborId of connections) {
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) {
+              // Skip neighbors that don't exist (expected during rapid additions/deletions)
+              continue
+            }
+            const distToNeighbor = this.distanceFunction(
+              queryVector,
+              neighbor.vector
+            )
+
+            if (distToNeighbor < currDist) {
+              currDist = distToNeighbor
+              currObj = neighbor
+              changed = true
+            }
+          }
+        }
+      }
+    }
+
+    // Search at level 0 with ef = k
+    const nearestNouns = await this.searchLayer(
+      queryVector,
+      currObj,
+      Math.max(this.config.efSearch, k),
+      0
+    )
+
+    // Convert to array and sort by distance
+    return [...nearestNouns].slice(0, k)
+  }
+
+  /**
+   * Remove an item from the index
+   */
+  public removeItem(id: string): boolean {
+    if (!this.nouns.has(id)) {
+      return false
+    }
+
+    const noun = this.nouns.get(id)!
+
+    // Remove connections to this noun from all neighbors
+    for (const [level, connections] of noun.connections.entries()) {
+      for (const neighborId of connections) {
+        const neighbor = this.nouns.get(neighborId)
+        if (!neighbor) {
+          // Skip neighbors that don't exist (expected during rapid additions/deletions)
+          continue
+        }
+        if (neighbor.connections.has(level)) {
+          neighbor.connections.get(level)!.delete(id)
+
+          // Prune connections after removing this noun to ensure consistency
+          this.pruneConnections(neighbor, level)
+        }
+      }
+    }
+
+    // Also check all other nouns for references to this noun and remove them
+    for (const [nounId, otherNoun] of this.nouns.entries()) {
+      if (nounId === id) continue // Skip the noun being removed
+
+      for (const [level, connections] of otherNoun.connections.entries()) {
+        if (connections.has(id)) {
+          connections.delete(id)
+
+          // Prune connections after removing this reference
+          this.pruneConnections(otherNoun, level)
+        }
+      }
+    }
+
+    // Remove the noun
+    this.nouns.delete(id)
+
+    // If we removed the entry point, find a new one
+    if (this.entryPointId === id) {
+      if (this.nouns.size === 0) {
+        this.entryPointId = null
+        this.maxLevel = 0
+      } else {
+        // Find the noun with the highest level
+        let maxLevel = 0
+        let newEntryPointId = null
+
+        for (const [nounId, noun] of this.nouns.entries()) {
+          if (noun.connections.size === 0) continue // Skip nouns with no connections
+
+          const nounLevel = Math.max(...noun.connections.keys())
+          if (nounLevel >= maxLevel) {
+            maxLevel = nounLevel
+            newEntryPointId = nounId
+          }
+        }
+
+        this.entryPointId = newEntryPointId
+        this.maxLevel = maxLevel
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Get all nouns in the index
+   */
+  public getNouns(): Map<string, HNSWNoun> {
+    return new Map(this.nouns)
+  }
+
+
+  /**
+   * Clear the index
+   */
+  public clear(): void {
+    this.nouns.clear()
+    this.entryPointId = null
+    this.maxLevel = 0
+  }
+
+  /**
+   * Get the size of the index
+   */
+  public size(): number {
+    return this.nouns.size
+  }
+
+  /**
+   * Get the distance function used by the index
+   */
+  public getDistanceFunction(): DistanceFunction {
+    return this.distanceFunction
+  }
+
+  /**
+   * Get the entry point ID
+   */
+  public getEntryPointId(): string | null {
+    return this.entryPointId
+  }
+
+  /**
+   * Get the maximum level
+   */
+  public getMaxLevel(): number {
+    return this.maxLevel
+  }
+
+  /**
+   * Get the dimension
+   */
+  public getDimension(): number | null {
+    return this.dimension
+  }
+
+  /**
+   * Get the configuration
+   */
+  public getConfig(): HNSWConfig {
+    return { ...this.config }
+  }
+
+  /**
+   * Search within a specific layer
+   * Returns a map of noun IDs to distances, sorted by distance
+   */
+  private async searchLayer(
+    queryVector: Vector,
+    entryPoint: HNSWNoun,
+    ef: number,
+    level: number
+  ): Promise<Map<string, number>> {
+    // Set of visited nouns
+    const visited = new Set<string>([entryPoint.id])
+
+    // Priority queue of candidates (closest first)
+    const candidates = new Map<string, number>()
+    candidates.set(
+      entryPoint.id,
+      this.distanceFunction(queryVector, entryPoint.vector)
+    )
+
+    // Priority queue of nearest neighbors found so far (closest first)
+    const nearest = new Map<string, number>()
+    nearest.set(
+      entryPoint.id,
+      this.distanceFunction(queryVector, entryPoint.vector)
+    )
+
+    // While there are candidates to explore
+    while (candidates.size > 0) {
+      // Get closest candidate
+      const [closestId, closestDist] = [...candidates][0]
+      candidates.delete(closestId)
+
+      // If this candidate is farther than the farthest in our result set, we're done
+      const farthestInNearest = [...nearest][nearest.size - 1]
+      if (nearest.size >= ef && closestDist > farthestInNearest[1]) {
+        break
+      }
+
+      // Explore neighbors of the closest candidate
+      const noun = this.nouns.get(closestId)
+      if (!noun) {
+        console.error(`Noun with ID ${closestId} not found in searchLayer`)
+        continue
+      }
+      const connections = noun.connections.get(level) || new Set<string>()
+
+      // If we have enough connections and parallelization is enabled, use parallel distance calculation
+      if (this.useParallelization && connections.size >= 10) {
+        // Collect unvisited neighbors
+        const unvisitedNeighbors: Array<{ id: string; vector: Vector }> = []
+        for (const neighborId of connections) {
+          if (!visited.has(neighborId)) {
+            visited.add(neighborId)
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) continue
+            unvisitedNeighbors.push({ id: neighborId, vector: neighbor.vector })
+          }
+        }
+
+        if (unvisitedNeighbors.length > 0) {
+          // Calculate distances in parallel
+          const distances = await this.calculateDistancesInParallel(queryVector, unvisitedNeighbors)
+
+          // Process the results
+          for (const { id, distance } of distances) {
+            // If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
+            if (nearest.size < ef || distance < farthestInNearest[1]) {
+              candidates.set(id, distance)
+              nearest.set(id, distance)
+
+              // If we have more than ef neighbors, remove the farthest one
+              if (nearest.size > ef) {
+                const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
+                nearest.clear()
+                for (let i = 0; i < ef; i++) {
+                  nearest.set(sortedNearest[i][0], sortedNearest[i][1])
+                }
+              }
+            }
+          }
+        }
+      } else {
+        // Use sequential processing for small number of connections
+        for (const neighborId of connections) {
+          if (!visited.has(neighborId)) {
+            visited.add(neighborId)
+
+            const neighbor = this.nouns.get(neighborId)
+            if (!neighbor) {
+              // Skip neighbors that don't exist (expected during rapid additions/deletions)
+              continue
+            }
+            const distToNeighbor = this.distanceFunction(
+              queryVector,
+              neighbor.vector
+            )
+
+            // If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
+            if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
+              candidates.set(neighborId, distToNeighbor)
+              nearest.set(neighborId, distToNeighbor)
+
+              // If we have more than ef neighbors, remove the farthest one
+              if (nearest.size > ef) {
+                const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
+                nearest.clear()
+                for (let i = 0; i < ef; i++) {
+                  nearest.set(sortedNearest[i][0], sortedNearest[i][1])
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // Sort nearest by distance
+    return new Map([...nearest].sort((a, b) => a[1] - b[1]))
+  }
+
+  /**
+   * Select M nearest neighbors from the candidate set
+   */
+  private selectNeighbors(
+    queryVector: Vector,
+    candidates: Map<string, number>,
+    M: number
+  ): Map<string, number> {
+    if (candidates.size <= M) {
+      return candidates
+    }
+
+    // Simple heuristic: just take the M closest
+    const sortedCandidates = [...candidates].sort((a, b) => a[1] - b[1])
+    const result = new Map<string, number>()
+
+    for (let i = 0; i < Math.min(M, sortedCandidates.length); i++) {
+      result.set(sortedCandidates[i][0], sortedCandidates[i][1])
+    }
+
+    return result
+  }
+
+  /**
+   * Ensure a noun doesn't have too many connections at a given level
+   */
+  private pruneConnections(noun: HNSWNoun, level: number): void {
+    const connections = noun.connections.get(level)!
+    if (connections.size <= this.config.M) {
+      return
+    }
+
+    // Calculate distances to all neighbors
+    const distances = new Map<string, number>()
+    const validNeighborIds = new Set<string>()
+
+    for (const neighborId of connections) {
+      const neighbor = this.nouns.get(neighborId)
+      if (!neighbor) {
+        // Skip neighbors that don't exist (expected during rapid additions/deletions)
+        continue
+      }
+
+      // Only add valid neighbors to the distances map
+      distances.set(
+        neighborId,
+        this.distanceFunction(noun.vector, neighbor.vector)
+      )
+      validNeighborIds.add(neighborId)
+    }
+
+    // Only proceed if we have valid neighbors
+    if (distances.size === 0) {
+      // If no valid neighbors, clear connections at this level
+      noun.connections.set(level, new Set())
+      return
+    }
+
+    // Select M closest neighbors from valid ones
+    const selectedNeighbors = this.selectNeighbors(
+      noun.vector,
+      distances,
+      this.config.M
+    )
+
+    // Update connections with only valid neighbors
+    noun.connections.set(level, new Set(selectedNeighbors.keys()))
+  }
+
+  /**
+   * Generate a random level for a new noun
+   * Uses the same distribution as in the original HNSW paper
+   */
+  private getRandomLevel(): number {
+    const r = Math.random()
+    return Math.floor(-Math.log(r) * (1.0 / Math.log(this.config.M)))
+  }
+}

src/hnsw/hnswIndexOptimized.ts

@@ -0,0 +1,586 @@
+/**
+ * Optimized HNSW (Hierarchical Navigable Small World) Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+
+import {
+  DistanceFunction,
+  HNSWConfig,
+  HNSWNoun,
+  Vector,
+  VectorDocument
+} from '../coreTypes.js'
+import { HNSWIndex } from './hnswIndex.js'
+import { StorageAdapter } from '../coreTypes.js'
+
+// Configuration for the optimized HNSW index
+export interface HNSWOptimizedConfig extends HNSWConfig {
+  // Memory threshold in bytes - when exceeded, will use disk-based approach
+  memoryThreshold?: number
+
+  // Product quantization settings
+  productQuantization?: {
+    // Whether to use product quantization
+    enabled: boolean
+    // Number of subvectors to split the vector into
+    numSubvectors?: number
+    // Number of centroids per subvector
+    numCentroids?: number
+  }
+
+  // Whether to use disk-based storage for the index
+  useDiskBasedIndex?: boolean
+}
+
+// Default configuration for the optimized HNSW index
+const DEFAULT_OPTIMIZED_CONFIG: HNSWOptimizedConfig = {
+  M: 16,
+  efConstruction: 200,
+  efSearch: 50,
+  ml: 16,
+  memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
+  productQuantization: {
+    enabled: false,
+    numSubvectors: 16,
+    numCentroids: 256
+  },
+  useDiskBasedIndex: false
+}
+
+/**
+ * Product Quantization implementation
+ * Reduces vector dimensionality by splitting vectors into subvectors
+ * and quantizing each subvector to the nearest centroid
+ */
+class ProductQuantizer {
+  private numSubvectors: number
+  private numCentroids: number
+  private centroids: Vector[][] = []
+  private subvectorSize: number = 0
+  private initialized: boolean = false
+  private dimension: number = 0
+
+  constructor(numSubvectors: number = 16, numCentroids: number = 256) {
+    this.numSubvectors = numSubvectors
+    this.numCentroids = numCentroids
+  }
+
+  /**
+   * Initialize the product quantizer with training data
+   * @param vectors Training vectors to use for learning centroids
+   */
+  public train(vectors: Vector[]): void {
+    if (vectors.length === 0) {
+      throw new Error('Cannot train product quantizer with empty vector set')
+    }
+
+    this.dimension = vectors[0].length
+    this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors)
+
+    // Initialize centroids for each subvector
+    for (let i = 0; i < this.numSubvectors; i++) {
+      // Extract subvectors from training data
+      const subvectors: Vector[] = vectors.map((vector) => {
+        const start = i * this.subvectorSize
+        const end = Math.min(start + this.subvectorSize, this.dimension)
+        return vector.slice(start, end)
+      })
+
+      // Initialize centroids for this subvector using k-means++
+      this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids)
+    }
+
+    this.initialized = true
+  }
+
+  /**
+   * Quantize a vector using product quantization
+   * @param vector Vector to quantize
+   * @returns Array of centroid indices, one for each subvector
+   */
+  public quantize(vector: Vector): number[] {
+    if (!this.initialized) {
+      throw new Error('Product quantizer not initialized. Call train() first.')
+    }
+
+    if (vector.length !== this.dimension) {
+      throw new Error(
+        `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`
+      )
+    }
+
+    const codes: number[] = []
+
+    // Quantize each subvector
+    for (let i = 0; i < this.numSubvectors; i++) {
+      const start = i * this.subvectorSize
+      const end = Math.min(start + this.subvectorSize, this.dimension)
+      const subvector = vector.slice(start, end)
+
+      // Find nearest centroid
+      let minDist = Number.MAX_VALUE
+      let nearestCentroidIndex = 0
+
+      for (let j = 0; j < this.centroids[i].length; j++) {
+        const centroid = this.centroids[i][j]
+        const dist = this.euclideanDistanceSquared(subvector, centroid)
+
+        if (dist < minDist) {
+          minDist = dist
+          nearestCentroidIndex = j
+        }
+      }
+
+      codes.push(nearestCentroidIndex)
+    }
+
+    return codes
+  }
+
+  /**
+   * Reconstruct a vector from its quantized representation
+   * @param codes Array of centroid indices
+   * @returns Reconstructed vector
+   */
+  public reconstruct(codes: number[]): Vector {
+    if (!this.initialized) {
+      throw new Error('Product quantizer not initialized. Call train() first.')
+    }
+
+    if (codes.length !== this.numSubvectors) {
+      throw new Error(
+        `Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}`
+      )
+    }
+
+    const reconstructed: Vector = []
+
+    // Reconstruct each subvector
+    for (let i = 0; i < this.numSubvectors; i++) {
+      const centroidIndex = codes[i]
+      const centroid = this.centroids[i][centroidIndex]
+
+      // Add centroid components to reconstructed vector
+      for (const component of centroid) {
+        reconstructed.push(component)
+      }
+    }
+
+    // Trim to original dimension if needed
+    return reconstructed.slice(0, this.dimension)
+  }
+
+  /**
+   * Compute squared Euclidean distance between two vectors
+   * @param a First vector
+   * @param b Second vector
+   * @returns Squared Euclidean distance
+   */
+  private euclideanDistanceSquared(a: Vector, b: Vector): number {
+    let sum = 0
+    const length = Math.min(a.length, b.length)
+
+    for (let i = 0; i < length; i++) {
+      const diff = a[i] - b[i]
+      sum += diff * diff
+    }
+
+    return sum
+  }
+
+  /**
+   * Implement k-means++ algorithm to initialize centroids
+   * @param vectors Vectors to cluster
+   * @param k Number of clusters
+   * @returns Array of centroids
+   */
+  private kMeansPlusPlus(vectors: Vector[], k: number): Vector[] {
+    if (vectors.length < k) {
+      // If we have fewer vectors than centroids, use the vectors as centroids
+      return [...vectors]
+    }
+
+    const centroids: Vector[] = []
+
+    // Choose first centroid randomly
+    const firstIndex = Math.floor(Math.random() * vectors.length)
+    centroids.push([...vectors[firstIndex]])
+
+    // Choose remaining centroids
+    for (let i = 1; i < k; i++) {
+      // Compute distances to nearest centroid for each vector
+      const distances: number[] = vectors.map((vector) => {
+        let minDist = Number.MAX_VALUE
+
+        for (const centroid of centroids) {
+          const dist = this.euclideanDistanceSquared(vector, centroid)
+          minDist = Math.min(minDist, dist)
+        }
+
+        return minDist
+      })
+
+      // Compute sum of distances
+      const distSum = distances.reduce((sum, dist) => sum + dist, 0)
+
+      // Choose next centroid with probability proportional to distance
+      let r = Math.random() * distSum
+      let nextIndex = 0
+
+      for (let j = 0; j < distances.length; j++) {
+        r -= distances[j]
+        if (r <= 0) {
+          nextIndex = j
+          break
+        }
+      }
+
+      centroids.push([...vectors[nextIndex]])
+    }
+
+    return centroids
+  }
+
+  /**
+   * Get the centroids for each subvector
+   * @returns Array of centroid arrays
+   */
+  public getCentroids(): Vector[][] {
+    return this.centroids
+  }
+
+  /**
+   * Set the centroids for each subvector
+   * @param centroids Array of centroid arrays
+   */
+  public setCentroids(centroids: Vector[][]): void {
+    this.centroids = centroids
+    this.numSubvectors = centroids.length
+    this.numCentroids = centroids[0].length
+    this.initialized = true
+  }
+
+  /**
+   * Get the dimension of the vectors
+   * @returns Dimension
+   */
+  public getDimension(): number {
+    return this.dimension
+  }
+
+  /**
+   * Set the dimension of the vectors
+   * @param dimension Dimension
+   */
+  public setDimension(dimension: number): void {
+    this.dimension = dimension
+    this.subvectorSize = Math.ceil(dimension / this.numSubvectors)
+  }
+}
+
+/**
+ * Optimized HNSW Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+export class HNSWIndexOptimized extends HNSWIndex {
+  private optimizedConfig: HNSWOptimizedConfig
+  private productQuantizer: ProductQuantizer | null = null
+  private storage: StorageAdapter | null = null
+  private useDiskBasedIndex: boolean = false
+  private useProductQuantization: boolean = false
+  private quantizedVectors: Map<string, number[]> = new Map()
+  private memoryUsage: number = 0
+  private vectorCount: number = 0
+
+  constructor(
+    config: Partial<HNSWOptimizedConfig> = {},
+    distanceFunction: DistanceFunction,
+    storage: StorageAdapter | null = null
+  ) {
+    // Initialize base HNSW index with standard config
+    super(config, distanceFunction)
+
+    // Set optimized config
+    this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config }
+
+    // Set storage adapter
+    this.storage = storage
+
+    // Initialize product quantizer if enabled
+    if (this.optimizedConfig.productQuantization?.enabled) {
+      this.useProductQuantization = true
+      this.productQuantizer = new ProductQuantizer(
+        this.optimizedConfig.productQuantization.numSubvectors,
+        this.optimizedConfig.productQuantization.numCentroids
+      )
+    }
+
+    // Set disk-based index flag
+    this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false
+  }
+
+  /**
+   * Add a vector to the index
+   * Uses product quantization if enabled and memory threshold is exceeded
+   */
+  public override async addItem(item: VectorDocument): Promise<string> {
+    // Check if item is defined
+    if (!item) {
+      throw new Error('Item is undefined or null')
+    }
+
+    const { id, vector } = item
+
+    // Check if vector is defined
+    if (!vector) {
+      throw new Error('Vector is undefined or null')
+    }
+
+    // Estimate memory usage for this vector
+    const vectorMemory = vector.length * 8 // 8 bytes per number (Float64)
+    const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16 // Estimate for connections
+    const totalMemory = vectorMemory + connectionsMemory
+
+    // Update memory usage estimate
+    this.memoryUsage += totalMemory
+    this.vectorCount++
+
+    // Check if we should switch to product quantization
+    if (
+      this.useProductQuantization &&
+      this.memoryUsage > this.optimizedConfig.memoryThreshold! &&
+      this.productQuantizer &&
+      !this.productQuantizer.getDimension()
+    ) {
+      // Initialize product quantizer with existing vectors
+      this.initializeProductQuantizer()
+    }
+
+    // If product quantization is active, quantize the vector
+    if (
+      this.useProductQuantization &&
+      this.productQuantizer &&
+      this.productQuantizer.getDimension() > 0
+    ) {
+      // Quantize the vector
+      const codes = this.productQuantizer.quantize(vector)
+
+      // Store the quantized vector
+      this.quantizedVectors.set(id, codes)
+
+      // Reconstruct the vector for indexing
+      const reconstructedVector = this.productQuantizer.reconstruct(codes)
+
+      // Add the reconstructed vector to the index
+      return await super.addItem({ id, vector: reconstructedVector })
+    }
+
+    // If disk-based index is active and storage is available, store the vector
+    if (this.useDiskBasedIndex && this.storage) {
+      // Create a noun object
+      const noun: HNSWNoun = {
+        id,
+        vector,
+        connections: new Map()
+      }
+
+      // Store the noun
+      this.storage.saveNoun(noun).catch((error) => {
+        console.error(`Failed to save noun ${id} to storage:`, error)
+      })
+    }
+
+    // Add the vector to the in-memory index
+    return await super.addItem(item)
+  }
+
+  /**
+   * Search for nearest neighbors
+   * Uses product quantization if enabled
+   */
+  public override async search(
+    queryVector: Vector,
+    k: number = 10
+  ): Promise<Array<[string, number]>> {
+    // Check if query vector is defined
+    if (!queryVector) {
+      throw new Error('Query vector is undefined or null')
+    }
+
+    // If product quantization is active, quantize the query vector
+    if (
+      this.useProductQuantization &&
+      this.productQuantizer &&
+      this.productQuantizer.getDimension() > 0
+    ) {
+      // Quantize the query vector
+      const codes = this.productQuantizer.quantize(queryVector)
+
+      // Reconstruct the query vector
+      const reconstructedVector = this.productQuantizer.reconstruct(codes)
+
+      // Search with the reconstructed vector
+      return await super.search(reconstructedVector, k)
+    }
+
+    // Otherwise, use the standard search
+    return await super.search(queryVector, k)
+  }
+
+  /**
+   * Remove an item from the index
+   */
+  public override removeItem(id: string): boolean {
+    // If product quantization is active, remove the quantized vector
+    if (this.useProductQuantization) {
+      this.quantizedVectors.delete(id)
+    }
+
+    // If disk-based index is active and storage is available, remove the vector from storage
+    if (this.useDiskBasedIndex && this.storage) {
+      this.storage.deleteNoun(id).catch((error) => {
+        console.error(`Failed to delete noun ${id} from storage:`, error)
+      })
+    }
+
+    // Update memory usage estimate
+    if (this.vectorCount > 0) {
+      this.memoryUsage = Math.max(
+        0,
+        this.memoryUsage - this.memoryUsage / this.vectorCount
+      )
+      this.vectorCount--
+    }
+
+    // Remove the item from the in-memory index
+    return super.removeItem(id)
+  }
+
+  /**
+   * Clear the index
+   */
+  public override clear(): void {
+    // Clear product quantization data
+    if (this.useProductQuantization) {
+      this.quantizedVectors.clear()
+      this.productQuantizer = new ProductQuantizer(
+        this.optimizedConfig.productQuantization!.numSubvectors,
+        this.optimizedConfig.productQuantization!.numCentroids
+      )
+    }
+
+    // Reset memory usage
+    this.memoryUsage = 0
+    this.vectorCount = 0
+
+    // Clear the in-memory index
+    super.clear()
+  }
+
+  /**
+   * Initialize product quantizer with existing vectors
+   */
+  private initializeProductQuantizer(): void {
+    if (!this.productQuantizer) {
+      return
+    }
+
+    // Get all vectors from the index
+    const nouns = super.getNouns()
+    const vectors: Vector[] = []
+
+    // Extract vectors
+    for (const [_, noun] of nouns) {
+      vectors.push(noun.vector)
+    }
+
+    // Train the product quantizer
+    if (vectors.length > 0) {
+      this.productQuantizer.train(vectors)
+
+      // Quantize all existing vectors
+      for (const [id, noun] of nouns) {
+        const codes = this.productQuantizer.quantize(noun.vector)
+        this.quantizedVectors.set(id, codes)
+      }
+
+      console.log(
+        `Initialized product quantizer with ${vectors.length} vectors`
+      )
+    }
+  }
+
+  /**
+   * Get the product quantizer
+   * @returns Product quantizer or null if not enabled
+   */
+  public getProductQuantizer(): ProductQuantizer | null {
+    return this.productQuantizer
+  }
+
+  /**
+   * Get the optimized configuration
+   * @returns Optimized configuration
+   */
+  public getOptimizedConfig(): HNSWOptimizedConfig {
+    return { ...this.optimizedConfig }
+  }
+
+  /**
+   * Get the estimated memory usage
+   * @returns Estimated memory usage in bytes
+   */
+  public getMemoryUsage(): number {
+    return this.memoryUsage
+  }
+
+  /**
+   * Set the storage adapter
+   * @param storage Storage adapter
+   */
+  public setStorage(storage: StorageAdapter): void {
+    this.storage = storage
+  }
+
+  /**
+   * Get the storage adapter
+   * @returns Storage adapter or null if not set
+   */
+  public getStorage(): StorageAdapter | null {
+    return this.storage
+  }
+
+  /**
+   * Set whether to use disk-based index
+   * @param useDiskBasedIndex Whether to use disk-based index
+   */
+  public setUseDiskBasedIndex(useDiskBasedIndex: boolean): void {
+    this.useDiskBasedIndex = useDiskBasedIndex
+  }
+
+  /**
+   * Get whether disk-based index is used
+   * @returns Whether disk-based index is used
+   */
+  public getUseDiskBasedIndex(): boolean {
+    return this.useDiskBasedIndex
+  }
+
+  /**
+   * Set whether to use product quantization
+   * @param useProductQuantization Whether to use product quantization
+   */
+  public setUseProductQuantization(useProductQuantization: boolean): void {
+    this.useProductQuantization = useProductQuantization
+  }
+
+  /**
+   * Get whether product quantization is used
+   * @returns Whether product quantization is used
+   */
+  public getUseProductQuantization(): boolean {
+    return this.useProductQuantization
+  }
+}

src/index.ts

@@ -0,0 +1,351 @@
+/**
+ * OPFS BrainyData
+ * A vector database using HNSW indexing with Origin Private File System storage
+ */
+
+// Export main BrainyData class and related types
+import { BrainyData, BrainyDataConfig } from './brainyData.js'
+
+export { BrainyData }
+export type { BrainyDataConfig }
+
+// Export distance functions for convenience
+import {
+  euclideanDistance,
+  cosineDistance,
+  manhattanDistance,
+  dotProductDistance
+} from './utils/index.js'
+
+export {
+  euclideanDistance,
+  cosineDistance,
+  manhattanDistance,
+  dotProductDistance
+}
+
+// Export embedding functionality
+import {
+  UniversalSentenceEncoder,
+  createEmbeddingFunction,
+  createTensorFlowEmbeddingFunction,
+  createThreadedEmbeddingFunction,
+  defaultEmbeddingFunction
+} from './utils/embedding.js'
+
+export {
+  UniversalSentenceEncoder,
+  createEmbeddingFunction,
+  createTensorFlowEmbeddingFunction,
+  createThreadedEmbeddingFunction,
+  defaultEmbeddingFunction
+}
+
+// Export storage adapters
+import {
+  OPFSStorage,
+  MemoryStorage,
+  createStorage
+} from './storage/opfsStorage.js'
+import { FileSystemStorage } from './storage/fileSystemStorage.js'
+import { R2Storage, S3CompatibleStorage } from './storage/s3CompatibleStorage.js'
+
+export {
+  OPFSStorage,
+  MemoryStorage,
+  FileSystemStorage,
+  R2Storage,
+  S3CompatibleStorage,
+  createStorage
+}
+
+// Export unified pipeline
+import {
+  Pipeline,
+  pipeline,
+  augmentationPipeline,
+  ExecutionMode,
+  PipelineOptions,
+  PipelineResult,
+  executeStreamlined,
+  executeByType,
+  executeSingle,
+  processStaticData,
+  processStreamingData,
+  createPipeline,
+  createStreamingPipeline,
+  StreamlinedExecutionMode,
+  StreamlinedPipelineOptions,
+  StreamlinedPipelineResult
+} from './pipeline.js'
+
+// Export sequential pipeline (for backward compatibility)
+import {
+  SequentialPipeline,
+  sequentialPipeline,
+  SequentialPipelineOptions
+} from './sequentialPipeline.js'
+
+// Export augmentation factory
+import {
+  createSenseAugmentation,
+  addWebSocketSupport,
+  executeAugmentation,
+  loadAugmentationModule,
+  AugmentationOptions
+} from './augmentationFactory.js'
+
+export {
+  // Unified pipeline exports
+  Pipeline,
+  pipeline,
+  augmentationPipeline,
+  ExecutionMode,
+  SequentialPipeline,
+  sequentialPipeline,
+
+  // Streamlined pipeline exports (now part of unified pipeline)
+  executeStreamlined,
+  executeByType,
+  executeSingle,
+  processStaticData,
+  processStreamingData,
+  createPipeline,
+  createStreamingPipeline,
+  StreamlinedExecutionMode,
+
+  // Augmentation factory exports
+  createSenseAugmentation,
+  addWebSocketSupport,
+  executeAugmentation,
+  loadAugmentationModule
+}
+export type { 
+  PipelineOptions, 
+  PipelineResult, 
+  SequentialPipelineOptions,
+  StreamlinedPipelineOptions,
+  StreamlinedPipelineResult,
+  AugmentationOptions
+}
+
+// Export augmentation registry for build-time loading
+import {
+  availableAugmentations,
+  registerAugmentation,
+  initializeAugmentationPipeline,
+  setAugmentationEnabled,
+  getAugmentationsByType
+} from './augmentationRegistry.js'
+
+export {
+  availableAugmentations,
+  registerAugmentation,
+  initializeAugmentationPipeline,
+  setAugmentationEnabled,
+  getAugmentationsByType
+}
+
+// Export augmentation registry loader for build tools
+import {
+  loadAugmentationsFromModules,
+  createAugmentationRegistryPlugin,
+  createAugmentationRegistryRollupPlugin
+} from './augmentationRegistryLoader.js'
+import type {
+  AugmentationRegistryLoaderOptions,
+  AugmentationLoadResult
+} from './augmentationRegistryLoader.js'
+
+export {
+  loadAugmentationsFromModules,
+  createAugmentationRegistryPlugin,
+  createAugmentationRegistryRollupPlugin
+}
+export type {
+  AugmentationRegistryLoaderOptions,
+  AugmentationLoadResult
+}
+
+// Export augmentation implementations
+import {
+  MemoryStorageAugmentation,
+  FileSystemStorageAugmentation,
+  OPFSStorageAugmentation,
+  createMemoryAugmentation
+} from './augmentations/memoryAugmentations.js'
+import {
+  WebSocketConduitAugmentation,
+  WebRTCConduitAugmentation,
+  createConduitAugmentation
+} from './augmentations/conduitAugmentations.js'
+import {
+  ServerSearchConduitAugmentation,
+  ServerSearchActivationAugmentation,
+  createServerSearchAugmentations
+} from './augmentations/serverSearchAugmentations.js'
+
+// Non-LLM exports
+export {
+  MemoryStorageAugmentation,
+  FileSystemStorageAugmentation,
+  OPFSStorageAugmentation,
+  createMemoryAugmentation,
+  WebSocketConduitAugmentation,
+  WebRTCConduitAugmentation,
+  createConduitAugmentation,
+  ServerSearchConduitAugmentation,
+  ServerSearchActivationAugmentation,
+  createServerSearchAugmentations
+}
+
+// LLM augmentations are optional and not imported by default
+// They can be imported directly from their module if needed:
+// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
+
+
+// Export types
+import type {
+  Vector,
+  VectorDocument,
+  SearchResult,
+  DistanceFunction,
+  EmbeddingFunction,
+  EmbeddingModel,
+  HNSWNoun,
+  GraphVerb,
+  HNSWConfig,
+  StorageAdapter
+} from './coreTypes.js'
+
+// Export HNSW index and optimized version
+import { HNSWIndex } from './hnsw/hnswIndex.js'
+import { HNSWIndexOptimized, HNSWOptimizedConfig } from './hnsw/hnswIndexOptimized.js'
+
+export {
+  HNSWIndex,
+  HNSWIndexOptimized
+}
+
+export type {
+  Vector,
+  VectorDocument,
+  SearchResult,
+  DistanceFunction,
+  EmbeddingFunction,
+  EmbeddingModel,
+  HNSWNoun,
+  GraphVerb,
+  HNSWConfig,
+  HNSWOptimizedConfig,
+  StorageAdapter
+}
+
+// Export augmentation types
+import type {
+  IAugmentation,
+  AugmentationResponse,
+  IWebSocketSupport,
+  ISenseAugmentation,
+  IConduitAugmentation,
+  ICognitionAugmentation,
+  IMemoryAugmentation,
+  IPerceptionAugmentation,
+  IDialogAugmentation,
+  IActivationAugmentation
+} from './types/augmentations.js'
+import { AugmentationType, BrainyAugmentations } from './types/augmentations.js'
+
+export type {
+  IAugmentation,
+  AugmentationResponse,
+  IWebSocketSupport
+}
+export {
+  AugmentationType,
+  BrainyAugmentations,
+  ISenseAugmentation,
+  IConduitAugmentation,
+  ICognitionAugmentation,
+  IMemoryAugmentation,
+  IPerceptionAugmentation,
+  IDialogAugmentation,
+  IActivationAugmentation
+}
+
+// Export combined WebSocket augmentation interfaces
+export type {
+  IWebSocketCognitionAugmentation,
+  IWebSocketSenseAugmentation,
+  IWebSocketPerceptionAugmentation,
+  IWebSocketActivationAugmentation,
+  IWebSocketDialogAugmentation,
+  IWebSocketConduitAugmentation,
+  IWebSocketMemoryAugmentation
+} from './types/augmentations.js'
+
+// Export graph types
+import type {
+  GraphNoun,
+  EmbeddedGraphVerb,
+  Person,
+  Place,
+  Thing,
+  Event,
+  Concept,
+  Content
+} from './types/graphTypes.js'
+import { NounType, VerbType } from './types/graphTypes.js'
+
+export type {
+  GraphNoun,
+  EmbeddedGraphVerb,
+  Person,
+  Place,
+  Thing,
+  Event,
+  Concept,
+  Content
+}
+export { NounType, VerbType }
+
+// Export MCP (Model Control Protocol) components
+import {
+  BrainyMCPAdapter,
+  MCPAugmentationToolset,
+  BrainyMCPService
+} from './mcp/index.js' // Import from mcp/index.js
+import {
+  MCPRequest,
+  MCPResponse,
+  MCPDataAccessRequest,
+  MCPToolExecutionRequest,
+  MCPSystemInfoRequest,
+  MCPAuthenticationRequest,
+  MCPRequestType,
+  MCPServiceOptions,
+  MCPTool,
+  MCP_VERSION
+} from './types/mcpTypes.js'
+
+export {
+  // MCP classes
+  BrainyMCPAdapter,
+  MCPAugmentationToolset,
+  BrainyMCPService,
+
+  // MCP types
+  MCPRequestType,
+  MCP_VERSION
+}
+
+export type {
+  MCPRequest,
+  MCPResponse,
+  MCPDataAccessRequest,
+  MCPToolExecutionRequest,
+  MCPSystemInfoRequest,
+  MCPAuthenticationRequest,
+  MCPServiceOptions,
+  MCPTool
+}

src/mcp/README.md

@@ -0,0 +1,180 @@
+# Model Control Protocol (MCP) for Brainy
+
+This document provides information about the Model Control Protocol (MCP) implementation in Brainy, which allows external models to access Brainy data and use the augmentation pipeline as tools.
+
+## Components
+
+The MCP implementation consists of three main components:
+
+1. **BrainyMCPAdapter**: Provides access to Brainy data through MCP
+2. **MCPAugmentationToolset**: Exposes the augmentation pipeline as tools
+3. **BrainyMCPService**: Integrates the adapter and toolset, providing WebSocket and REST server implementations for external model access
+
+## Environment Compatibility
+
+### BrainyMCPAdapter
+
+The `BrainyMCPAdapter` has no environment-specific dependencies and can run in any environment where Brainy itself runs, including:
+
+- Browser environments
+- Node.js environments
+- Server environments
+
+### MCPAugmentationToolset
+
+The `MCPAugmentationToolset` also has no environment-specific dependencies and can run in any environment where Brainy itself runs, including:
+
+- Browser environments
+- Node.js environments
+- Server environments
+
+### BrainyMCPService
+
+The `BrainyMCPService` has been refactored to separate the core functionality from the Node.js-specific server functionality:
+
+1. **Core Functionality**: The core request handling functionality (`handleMCPRequest`) can run in any environment where Brainy itself runs. This is what remains in the main Brainy package.
+
+2. **Server Functionality**: The WebSocket and REST server functionality has been moved to the cloud-wrapper project to avoid including Node.js-specific dependencies in the browser bundle:
+   - `ws` for WebSocket server
+   - `express` for REST API
+   - `cors` for Cross-Origin Resource Sharing
+
+This separation ensures that the browser bundle remains lightweight and doesn't include unnecessary Node.js-specific dependencies. In browser or other environments, you can still use the core functionality through the `handleMCPRequest` method.
+
+## Usage
+
+### In Any Environment (Browser, Node.js, Server)
+
+```typescript
+import { BrainyData, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy'
+
+// Create a BrainyData instance
+const brainyData = new BrainyData()
+await brainyData.init()
+
+// Create an MCP adapter
+const adapter = new BrainyMCPAdapter(brainyData)
+
+// Create a toolset
+const toolset = new MCPAugmentationToolset()
+
+// Use the adapter to access Brainy data
+const response = await adapter.handleRequest({
+  type: 'data_access',
+  operation: 'search',
+  requestId: adapter.generateRequestId(),
+  version: '1.0.0',
+  parameters: {
+    query: 'example query',
+    k: 5
+  }
+})
+
+// Use the toolset to execute augmentation pipeline tools
+const toolResponse = await toolset.handleRequest({
+  type: 'tool_execution',
+  toolName: 'brainy_memory_storeData',
+  requestId: toolset.generateRequestId(),
+  version: '1.0.0',
+  parameters: {
+    args: ['key1', { some: 'data' }]
+  }
+})
+```
+
+### In Node.js Environment with Server Functionality
+
+To use the MCP service with WebSocket and REST server functionality, you should use the cloud-wrapper project:
+
+```typescript
+import { BrainyData } from '@soulcraft/brainy'
+import { initializeBrainy } from './services/brainyService.js'
+import { initializeMCPService } from './services/mcpService.js'
+
+// Initialize Brainy
+const brainyData = await initializeBrainy()
+
+// Initialize MCP service with WebSocket and REST server functionality
+const mcpService = initializeMCPService(brainyData, {
+  wsPort: 8080,
+  restPort: 3000,
+  enableAuth: true,
+  apiKeys: ['your-api-key'],
+  rateLimit: {
+    maxRequests: 100,
+    windowMs: 60000 // 1 minute
+  },
+  cors: {
+    origin: '*',
+    credentials: true
+  }
+})
+```
+
+Alternatively, you can configure the MCP service using environment variables in the cloud-wrapper:
+
+```
+# MCP configuration
+MCP_WS_PORT=8080
+MCP_REST_PORT=3000
+MCP_ENABLE_AUTH=true
+MCP_API_KEYS=your-api-key,another-key
+MCP_RATE_LIMIT_REQUESTS=100
+MCP_RATE_LIMIT_WINDOW_MS=60000
+MCP_ENABLE_CORS=true
+```
+
+### In Browser Environment (Core Functionality Only)
+
+```typescript
+import { BrainyData, BrainyMCPService } from '@soulcraft/brainy'
+
+// Create a BrainyData instance
+const brainyData = new BrainyData()
+await brainyData.init()
+
+// Create an MCP service (server functionality will be disabled in browser)
+const mcpService = new BrainyMCPService(brainyData)
+
+// Use the core functionality
+const response = await mcpService.handleMCPRequest({
+  type: 'data_access',
+  operation: 'search',
+  requestId: mcpService.generateRequestId(),
+  version: '1.0.0',
+  parameters: {
+    query: 'example query',
+    k: 5
+  }
+})
+```
+
+## Cloud Wrapper Integration
+
+The MCP service's server functionality has been integrated directly into the cloud-wrapper project. The cloud-wrapper automatically initializes the MCP service if the appropriate environment variables are set:
+
+```
+# MCP configuration
+MCP_WS_PORT=8080
+MCP_REST_PORT=3000
+MCP_ENABLE_AUTH=true
+MCP_API_KEYS=your-api-key,another-key
+MCP_RATE_LIMIT_REQUESTS=100
+MCP_RATE_LIMIT_WINDOW_MS=60000
+MCP_ENABLE_CORS=true
+```
+
+You can deploy the cloud wrapper to various cloud platforms using the npm scripts from the root directory:
+
+```bash
+# Deploy to AWS Lambda and API Gateway
+npm run deploy:cloud:aws
+
+# Deploy to Google Cloud Run
+npm run deploy:cloud:gcp
+
+# Deploy to Cloudflare Workers
+npm run deploy:cloud:cloudflare
+```
+
+The cloud wrapper is specifically designed for server environments and includes additional features like logging, security headers, and deployment scripts for various cloud providers. See the [Cloud Wrapper README](../../cloud-wrapper/README.md) for detailed configuration instructions and API documentation.

src/mcp/brainyMCPAdapter.ts

@@ -0,0 +1,203 @@
+/**
+ * BrainyMCPAdapter
+ * 
+ * This class provides an adapter for accessing Brainy data through the Model Control Protocol (MCP).
+ * It wraps a BrainyData instance and exposes methods for getting vectors, searching similar items,
+ * and getting relationships.
+ */
+
+import { v4 as uuidv4 } from 'uuid'
+import { BrainyDataInterface } from '../types/brainyDataInterface.js'
+import { 
+  MCPRequest, 
+  MCPResponse, 
+  MCPDataAccessRequest,
+  MCPRequestType,
+  MCP_VERSION
+} from '../types/mcpTypes.js'
+
+export class BrainyMCPAdapter {
+  private brainyData: BrainyDataInterface
+
+  /**
+   * Creates a new BrainyMCPAdapter
+   * @param brainyData The BrainyData instance to wrap
+   */
+  constructor(brainyData: BrainyDataInterface) {
+    this.brainyData = brainyData
+  }
+
+  /**
+   * Handles an MCP data access request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    try {
+      switch (request.operation) {
+        case 'get':
+          return await this.handleGetRequest(request)
+        case 'search':
+          return await this.handleSearchRequest(request)
+        case 'add':
+          return await this.handleAddRequest(request)
+        case 'getRelationships':
+          return await this.handleGetRelationshipsRequest(request)
+        default:
+          return this.createErrorResponse(
+            request.requestId,
+            'UNSUPPORTED_OPERATION',
+            `Operation ${request.operation} is not supported`
+          )
+      }
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Handles a get request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleGetRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { id } = request.parameters
+
+    if (!id) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "id" is required'
+      )
+    }
+
+    const noun = await this.brainyData.get(id)
+    
+    if (!noun) {
+      return this.createErrorResponse(
+        request.requestId,
+        'NOT_FOUND',
+        `No noun found with id ${id}`
+      )
+    }
+
+    return this.createSuccessResponse(request.requestId, noun)
+  }
+
+  /**
+   * Handles a search request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleSearchRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { query, k = 10 } = request.parameters
+
+    if (!query) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "query" is required'
+      )
+    }
+
+    const results = await this.brainyData.searchText(query, k)
+    return this.createSuccessResponse(request.requestId, results)
+  }
+
+  /**
+   * Handles an add request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleAddRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { text, metadata } = request.parameters
+
+    if (!text) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "text" is required'
+      )
+    }
+
+    const id = await this.brainyData.add(text, metadata)
+    return this.createSuccessResponse(request.requestId, { id })
+  }
+
+  /**
+   * Handles a getRelationships request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleGetRelationshipsRequest(request: MCPDataAccessRequest): Promise<MCPResponse> {
+    const { id } = request.parameters
+
+    if (!id) {
+      return this.createErrorResponse(
+        request.requestId,
+        'MISSING_PARAMETER',
+        'Parameter "id" is required'
+      )
+    }
+
+    // This is a simplified implementation - in a real implementation, we would
+    // need to check if these methods exist on the BrainyDataInterface
+    const outgoing = await (this.brainyData as any).getVerbsBySource?.(id) || []
+    const incoming = await (this.brainyData as any).getVerbsByTarget?.(id) || []
+
+    return this.createSuccessResponse(request.requestId, { outgoing, incoming })
+  }
+
+  /**
+   * Creates a success response
+   * @param requestId The request ID
+   * @param data The response data
+   * @returns An MCP response
+   */
+  private createSuccessResponse(requestId: string, data: any): MCPResponse {
+    return {
+      success: true,
+      requestId,
+      version: MCP_VERSION,
+      data
+    }
+  }
+
+  /**
+   * Creates an error response
+   * @param requestId The request ID
+   * @param code The error code
+   * @param message The error message
+   * @param details Optional error details
+   * @returns An MCP response
+   */
+  private createErrorResponse(
+    requestId: string,
+    code: string,
+    message: string,
+    details?: any
+  ): MCPResponse {
+    return {
+      success: false,
+      requestId,
+      version: MCP_VERSION,
+      error: {
+        code,
+        message,
+        details
+      }
+    }
+  }
+
+  /**
+   * Creates a new request ID
+   * @returns A new UUID
+   */
+  generateRequestId(): string {
+    return uuidv4()
+  }
+}

src/mcp/brainyMCPService.ts

@@ -0,0 +1,347 @@
+/**
+ * BrainyMCPService
+ *
+ * This class provides a unified service for accessing Brainy data and augmentations
+ * through the Model Control Protocol (MCP). It integrates the BrainyMCPAdapter and
+ * MCPAugmentationToolset classes and provides WebSocket and REST server implementations
+ * for external model access.
+ */
+
+import { v4 as uuidv4 } from 'uuid'
+import { BrainyDataInterface } from '../types/brainyDataInterface.js'
+import {
+  MCPRequest,
+  MCPResponse,
+  MCPDataAccessRequest,
+  MCPToolExecutionRequest,
+  MCPSystemInfoRequest,
+  MCPAuthenticationRequest,
+  MCPRequestType,
+  MCPServiceOptions,
+  MCP_VERSION,
+  MCPTool
+} from '../types/mcpTypes.js'
+import { BrainyMCPAdapter } from './brainyMCPAdapter.js'
+import { MCPAugmentationToolset } from './mcpAugmentationToolset.js'
+import { isBrowser, isNode } from '../utils/environment.js'
+
+export class BrainyMCPService {
+  private dataAdapter: BrainyMCPAdapter
+  private toolset: MCPAugmentationToolset
+  private options: MCPServiceOptions
+  private authTokens: Map<string, { userId: string; expires: number }>
+  private rateLimits: Map<string, { count: number; resetTime: number }>
+
+  /**
+   * Creates a new BrainyMCPService
+   * @param brainyData The BrainyData instance to wrap
+   * @param options Configuration options for the service
+   */
+  constructor(
+    brainyData: BrainyDataInterface,
+    options: MCPServiceOptions = {}
+  ) {
+    this.dataAdapter = new BrainyMCPAdapter(brainyData)
+    this.toolset = new MCPAugmentationToolset()
+    this.options = options
+    this.authTokens = new Map()
+    this.rateLimits = new Map()
+  }
+
+  /**
+   * Handles an MCP request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleRequest(request: MCPRequest): Promise<MCPResponse> {
+    try {
+      switch (request.type) {
+        case MCPRequestType.DATA_ACCESS:
+          return await this.dataAdapter.handleRequest(
+            request as MCPDataAccessRequest
+          )
+
+        case MCPRequestType.TOOL_EXECUTION:
+          return await this.toolset.handleRequest(
+            request as MCPToolExecutionRequest
+          )
+
+        case MCPRequestType.SYSTEM_INFO:
+          return await this.handleSystemInfoRequest(
+            request as MCPSystemInfoRequest
+          )
+
+        case MCPRequestType.AUTHENTICATION:
+          return await this.handleAuthenticationRequest(
+            request as MCPAuthenticationRequest
+          )
+
+        default:
+          return this.createErrorResponse(
+            request.requestId,
+            'UNSUPPORTED_REQUEST_TYPE',
+            `Request type ${request.type} is not supported`
+          )
+      }
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Handles a system info request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleSystemInfoRequest(
+    request: MCPSystemInfoRequest
+  ): Promise<MCPResponse> {
+    try {
+      switch (request.infoType) {
+        case 'status':
+          return this.createSuccessResponse(request.requestId, {
+            status: 'active',
+            version: MCP_VERSION,
+            environment: isBrowser() ? 'browser' : isNode() ? 'node' : 'unknown'
+          })
+
+        case 'availableTools':
+          const tools: MCPTool[] = await this.toolset.getAvailableTools()
+          return this.createSuccessResponse(request.requestId, tools)
+
+        case 'version':
+          return this.createSuccessResponse(request.requestId, {
+            version: MCP_VERSION
+          })
+
+        default:
+          return this.createErrorResponse(
+            request.requestId,
+            'UNSUPPORTED_INFO_TYPE',
+            `Info type ${request.infoType} is not supported`
+          )
+      }
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Handles an authentication request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  private async handleAuthenticationRequest(
+    request: MCPAuthenticationRequest
+  ): Promise<MCPResponse> {
+    try {
+      if (!this.options.enableAuth) {
+        return this.createSuccessResponse(request.requestId, {
+          authenticated: true,
+          message: 'Authentication is not enabled'
+        })
+      }
+
+      const { credentials } = request
+
+      // Check API key authentication
+      if (
+        credentials.apiKey &&
+        this.options.apiKeys?.includes(credentials.apiKey)
+      ) {
+        const token = this.generateAuthToken('api-user')
+        return this.createSuccessResponse(request.requestId, {
+          authenticated: true,
+          token
+        })
+      }
+
+      // Check username/password authentication
+      // This is a placeholder - in a real implementation, you would check against a database
+      if (
+        credentials.username === 'admin' &&
+        credentials.password === 'password'
+      ) {
+        const token = this.generateAuthToken(credentials.username)
+        return this.createSuccessResponse(request.requestId, {
+          authenticated: true,
+          token
+        })
+      }
+
+      return this.createErrorResponse(
+        request.requestId,
+        'INVALID_CREDENTIALS',
+        'Invalid credentials'
+      )
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Checks if a request is valid
+   * @param request The request to check
+   * @returns Whether the request is valid
+   */
+  private isValidRequest(request: any): boolean {
+    return (
+      request &&
+      typeof request === 'object' &&
+      request.type &&
+      request.requestId &&
+      request.version
+    )
+  }
+
+  /**
+   * Checks if a request is authenticated
+   * @param request The request to check
+   * @returns Whether the request is authenticated
+   */
+  private isAuthenticated(request: MCPRequest): boolean {
+    if (!this.options.enableAuth) {
+      return true
+    }
+
+    return request.authToken ? this.isValidToken(request.authToken) : false
+  }
+
+  /**
+   * Checks if a token is valid
+   * @param token The token to check
+   * @returns Whether the token is valid
+   */
+  private isValidToken(token: string): boolean {
+    const tokenInfo = this.authTokens.get(token)
+    if (!tokenInfo) {
+      return false
+    }
+
+    if (tokenInfo.expires < Date.now()) {
+      this.authTokens.delete(token)
+      return false
+    }
+
+    return true
+  }
+
+  /**
+   * Generates an authentication token
+   * @param userId The user ID to associate with the token
+   * @returns The generated token
+   */
+  private generateAuthToken(userId: string): string {
+    const token = uuidv4()
+    const expires = Date.now() + 24 * 60 * 60 * 1000 // 24 hours
+
+    this.authTokens.set(token, { userId, expires })
+
+    return token
+  }
+
+  /**
+   * Checks if a client has exceeded the rate limit
+   * @param clientId The client ID to check
+   * @returns Whether the client is within the rate limit
+   */
+  private checkRateLimit(clientId: string): boolean {
+    if (!this.options.rateLimit) {
+      return true
+    }
+
+    const now = Date.now()
+    const limit = this.rateLimits.get(clientId)
+
+    if (!limit) {
+      this.rateLimits.set(clientId, {
+        count: 1,
+        resetTime: now + this.options.rateLimit.windowMs
+      })
+      return true
+    }
+
+    if (limit.resetTime < now) {
+      limit.count = 1
+      limit.resetTime = now + this.options.rateLimit.windowMs
+      return true
+    }
+
+    if (limit.count >= this.options.rateLimit.maxRequests) {
+      return false
+    }
+
+    limit.count++
+    return true
+  }
+
+  /**
+   * Creates a success response
+   * @param requestId The request ID
+   * @param data The response data
+   * @returns An MCP response
+   */
+  private createSuccessResponse(requestId: string, data: any): MCPResponse {
+    return {
+      success: true,
+      requestId,
+      version: MCP_VERSION,
+      data
+    }
+  }
+
+  /**
+   * Creates an error response
+   * @param requestId The request ID
+   * @param code The error code
+   * @param message The error message
+   * @param details Optional error details
+   * @returns An MCP response
+   */
+  private createErrorResponse(
+    requestId: string,
+    code: string,
+    message: string,
+    details?: any
+  ): MCPResponse {
+    return {
+      success: false,
+      requestId,
+      version: MCP_VERSION,
+      error: {
+        code,
+        message,
+        details
+      }
+    }
+  }
+
+  /**
+   * Creates a new request ID
+   * @returns A new UUID
+   */
+  generateRequestId(): string {
+    return uuidv4()
+  }
+
+  /**
+   * Handles an MCP request directly (for in-process models)
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleMCPRequest(request: MCPRequest): Promise<MCPResponse> {
+    return await this.handleRequest(request)
+  }
+}

src/mcp/index.ts

@@ -0,0 +1,19 @@
+/**
+ * Model Control Protocol (MCP) for Brainy
+ * 
+ * This module provides a Model Control Protocol (MCP) implementation for Brainy,
+ * allowing external models to access Brainy data and use the augmentation pipeline as tools.
+ */
+
+// Import and re-export the MCP components
+import { BrainyMCPAdapter } from './brainyMCPAdapter.js'
+import { MCPAugmentationToolset } from './mcpAugmentationToolset.js'
+import { BrainyMCPService } from './brainyMCPService.js'
+
+// Export the MCP components
+export { BrainyMCPAdapter }
+export { MCPAugmentationToolset }
+export { BrainyMCPService }
+
+// Export the MCP types
+export * from '../types/mcpTypes.js'

src/mcp/mcpAugmentationToolset.ts

@@ -0,0 +1,225 @@
+/**
+ * MCPAugmentationToolset
+ * 
+ * This class exposes the Brainy augmentation pipeline as tools through the Model Control Protocol (MCP).
+ * It provides methods for getting available tools and executing tools.
+ */
+
+import { v4 as uuidv4 } from 'uuid'
+import { 
+  MCPResponse, 
+  MCPToolExecutionRequest,
+  MCPTool,
+  MCP_VERSION
+} from '../types/mcpTypes.js'
+import { AugmentationType } from '../types/augmentations.js'
+
+// Import the augmentation pipeline
+import { augmentationPipeline } from '../augmentationPipeline.js'
+
+export class MCPAugmentationToolset {
+  /**
+   * Creates a new MCPAugmentationToolset
+   */
+  constructor() {
+    // No initialization needed
+  }
+
+  /**
+   * Handles an MCP tool execution request
+   * @param request The MCP request
+   * @returns An MCP response
+   */
+  async handleRequest(request: MCPToolExecutionRequest): Promise<MCPResponse> {
+    try {
+      const { toolName, parameters } = request
+
+      // Extract the augmentation type and method from the tool name
+      // Tool names are in the format: brainy_{augmentationType}_{method}
+      const parts = toolName.split('_')
+      
+      if (parts.length < 3 || parts[0] !== 'brainy') {
+        return this.createErrorResponse(
+          request.requestId,
+          'INVALID_TOOL',
+          `Invalid tool name: ${toolName}. Tool names should be in the format: brainy_{augmentationType}_{method}`
+        )
+      }
+
+      const augmentationType = parts[1]
+      const method = parts.slice(2).join('_')
+
+      // Validate the augmentation type
+      if (!this.isValidAugmentationType(augmentationType)) {
+        return this.createErrorResponse(
+          request.requestId,
+          'INVALID_AUGMENTATION_TYPE',
+          `Invalid augmentation type: ${augmentationType}`
+        )
+      }
+
+      // Execute the appropriate pipeline based on the augmentation type
+      const result = await this.executePipeline(augmentationType, method, parameters)
+      
+      return this.createSuccessResponse(request.requestId, result)
+    } catch (error) {
+      return this.createErrorResponse(
+        request.requestId,
+        'INTERNAL_ERROR',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+  }
+
+  /**
+   * Gets all available tools
+   * @returns An array of MCP tools
+   */
+  async getAvailableTools(): Promise<MCPTool[]> {
+    const tools: MCPTool[] = []
+    
+    // Get all available augmentation types
+    const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes()
+    
+    for (const type of augmentationTypes) {
+      // Get all augmentations of this type
+      const augmentations = augmentationPipeline.getAugmentationsByType(type)
+      
+      for (const augmentation of augmentations) {
+        // Get all methods of this augmentation (excluding private methods and base methods)
+        const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(augmentation))
+          .filter(method => 
+            !method.startsWith('_') && 
+            method !== 'constructor' &&
+            method !== 'initialize' &&
+            method !== 'shutDown' &&
+            method !== 'getStatus' &&
+            typeof augmentation[method] === 'function'
+          )
+        
+        // Create a tool for each method
+        for (const method of methods) {
+          tools.push(this.createToolDefinition(type, augmentation.name, method))
+        }
+      }
+    }
+    
+    return tools
+  }
+
+  /**
+   * Creates a tool definition
+   * @param type The augmentation type
+   * @param augmentationName The augmentation name
+   * @param method The method name
+   * @returns An MCP tool definition
+   */
+  private createToolDefinition(type: string, augmentationName: string, method: string): MCPTool {
+    return {
+      name: `brainy_${type}_${method}`,
+      description: `Access to Brainy's ${type} augmentation '${augmentationName}' method '${method}'`,
+      parameters: {
+        type: 'object',
+        properties: {
+          args: {
+            type: 'array',
+            description: `Arguments for the ${method} method`
+          },
+          options: {
+            type: 'object',
+            description: 'Optional execution options'
+          }
+        },
+        required: ['args']
+      }
+    }
+  }
+
+  /**
+   * Executes the appropriate pipeline based on the augmentation type
+   * @param type The augmentation type
+   * @param method The method to execute
+   * @param parameters The parameters for the method
+   * @returns The result of the pipeline execution
+   */
+  private async executePipeline(type: string, method: string, parameters: any): Promise<any> {
+    const { args = [], options = {} } = parameters
+    
+    switch (type) {
+      case AugmentationType.SENSE:
+        return await augmentationPipeline.executeSensePipeline(method, args, options)
+      case AugmentationType.CONDUIT:
+        return await augmentationPipeline.executeConduitPipeline(method, args, options)
+      case AugmentationType.COGNITION:
+        return await augmentationPipeline.executeCognitionPipeline(method, args, options)
+      case AugmentationType.MEMORY:
+        return await augmentationPipeline.executeMemoryPipeline(method, args, options)
+      case AugmentationType.PERCEPTION:
+        return await augmentationPipeline.executePerceptionPipeline(method, args, options)
+      case AugmentationType.DIALOG:
+        return await augmentationPipeline.executeDialogPipeline(method, args, options)
+      case AugmentationType.ACTIVATION:
+        return await augmentationPipeline.executeActivationPipeline(method, args, options)
+      default:
+        throw new Error(`Unsupported augmentation type: ${type}`)
+    }
+  }
+
+  /**
+   * Checks if an augmentation type is valid
+   * @param type The augmentation type to check
+   * @returns Whether the augmentation type is valid
+   */
+  private isValidAugmentationType(type: string): boolean {
+    return Object.values(AugmentationType).includes(type as AugmentationType)
+  }
+
+  /**
+   * Creates a success response
+   * @param requestId The request ID
+   * @param data The response data
+   * @returns An MCP response
+   */
+  private createSuccessResponse(requestId: string, data: any): MCPResponse {
+    return {
+      success: true,
+      requestId,
+      version: MCP_VERSION,
+      data
+    }
+  }
+
+  /**
+   * Creates an error response
+   * @param requestId The request ID
+   * @param code The error code
+   * @param message The error message
+   * @param details Optional error details
+   * @returns An MCP response
+   */
+  private createErrorResponse(
+    requestId: string,
+    code: string,
+    message: string,
+    details?: any
+  ): MCPResponse {
+    return {
+      success: false,
+      requestId,
+      version: MCP_VERSION,
+      error: {
+        code,
+        message,
+        details
+      }
+    }
+  }
+
+  /**
+   * Creates a new request ID
+   * @returns A new UUID
+   */
+  generateRequestId(): string {
+    return uuidv4()
+  }
+}

src/pipeline.ts

@@ -0,0 +1,919 @@
+/**
+ * Unified Pipeline
+ *
+ * This module combines the functionality of the primary augmentation pipeline and the streamlined pipeline
+ * into a single, unified pipeline system. It provides both the registry functionality of the primary pipeline
+ * and the simplified execution API of the streamlined pipeline.
+ */
+
+import {
+  IAugmentation,
+  AugmentationType,
+  AugmentationResponse,
+  IWebSocketSupport,
+  BrainyAugmentations
+} from './types/augmentations.js'
+import { AugmentationRegistry, IPipeline } from './types/pipelineTypes.js'
+import { isThreadingAvailable } from './utils/environment.js'
+import { executeInThread } from './utils/workerUtils.js'
+import { executeAugmentation } from './augmentationFactory.js'
+import { setDefaultPipeline } from './augmentationRegistry.js'
+
+/**
+ * Execution mode for the pipeline
+ */
+export enum ExecutionMode {
+  SEQUENTIAL = 'sequential',
+  PARALLEL = 'parallel',
+  FIRST_SUCCESS = 'firstSuccess',
+  FIRST_RESULT = 'firstResult',
+  THREADED = 'threaded'  // Execute in separate threads when available
+}
+
+/**
+ * Options for pipeline execution
+ */
+export interface PipelineOptions {
+  mode?: ExecutionMode;
+  timeout?: number;
+  stopOnError?: boolean;
+  forceThreading?: boolean;  // Force threading even if not in THREADED mode
+  disableThreading?: boolean;  // Disable threading even if in THREADED mode
+}
+
+/**
+ * Default pipeline options
+ */
+const DEFAULT_PIPELINE_OPTIONS: PipelineOptions = {
+  mode: ExecutionMode.SEQUENTIAL,
+  timeout: 30000,
+  stopOnError: false,
+  forceThreading: false,
+  disableThreading: false
+}
+
+/**
+ * Result of a pipeline execution
+ */
+export interface PipelineResult<T> {
+  results: AugmentationResponse<T>[];
+  errors: Error[];
+  successful: AugmentationResponse<T>[];
+}
+
+/**
+ * Pipeline class
+ *
+ * Manages multiple augmentations of each type and provides methods to execute them.
+ * Implements the IPipeline interface to avoid circular dependencies.
+ */
+export class Pipeline implements IPipeline {
+  private registry: AugmentationRegistry = {
+    sense: [],
+    conduit: [],
+    cognition: [],
+    memory: [],
+    perception: [],
+    dialog: [],
+    activation: [],
+    webSocket: []
+  }
+
+  /**
+   * Register an augmentation with the pipeline
+   *
+   * @param augmentation The augmentation to register
+   * @returns The pipeline instance for chaining
+   */
+  public register<T extends IAugmentation>(augmentation: T): Pipeline {
+    let registered = false
+
+    // Check for specific augmentation types
+    if (this.isAugmentationType<BrainyAugmentations.ISenseAugmentation>(
+      augmentation,
+      'processRawData',
+      'listenToFeed'
+    )) {
+      this.registry.sense.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IConduitAugmentation>(
+      augmentation,
+      'establishConnection',
+      'readData',
+      'writeData',
+      'monitorStream'
+    )) {
+      this.registry.conduit.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.ICognitionAugmentation>(
+      augmentation,
+      'reason',
+      'infer',
+      'executeLogic'
+    )) {
+      this.registry.cognition.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IMemoryAugmentation>(
+      augmentation,
+      'storeData',
+      'retrieveData',
+      'updateData',
+      'deleteData',
+      'listDataKeys'
+    )) {
+      this.registry.memory.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IPerceptionAugmentation>(
+      augmentation,
+      'interpret',
+      'organize',
+      'generateVisualization'
+    )) {
+      this.registry.perception.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IDialogAugmentation>(
+      augmentation,
+      'processUserInput',
+      'generateResponse',
+      'manageContext'
+    )) {
+      this.registry.dialog.push(augmentation)
+      registered = true
+    } else if (this.isAugmentationType<BrainyAugmentations.IActivationAugmentation>(
+      augmentation,
+      'triggerAction',
+      'generateOutput',
+      'interactExternal'
+    )) {
+      this.registry.activation.push(augmentation)
+      registered = true
+    }
+
+    // Check if the augmentation supports WebSocket
+    if (this.isAugmentationType<IWebSocketSupport>(
+      augmentation,
+      'connectWebSocket',
+      'sendWebSocketMessage',
+      'onWebSocketMessage',
+      'closeWebSocket'
+    )) {
+      this.registry.webSocket.push(augmentation as IWebSocketSupport)
+      registered = true
+    }
+
+    // If the augmentation wasn't registered as any known type, throw an error
+    if (!registered) {
+      throw new Error(`Unknown augmentation type: ${augmentation.name}`)
+    }
+
+    return this
+  }
+
+  /**
+   * Unregister an augmentation from the pipeline
+   *
+   * @param augmentationName The name of the augmentation to unregister
+   * @returns The pipeline instance for chaining
+   */
+  public unregister(augmentationName: string): Pipeline {
+    let found = false
+
+    // Remove from all registries
+    for (const type in this.registry) {
+      const typedRegistry = this.registry[type as keyof AugmentationRegistry]
+      const index = typedRegistry.findIndex(aug => aug.name === augmentationName)
+
+      if (index !== -1) {
+        typedRegistry.splice(index, 1)
+        found = true
+      }
+    }
+
+    return this
+  }
+
+  /**
+   * Initialize all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are initialized
+   */
+  public async initialize(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.initialize().catch(error => {
+          console.error(`Failed to initialize augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Shut down all registered augmentations
+   *
+   * @returns A promise that resolves when all augmentations are shut down
+   */
+  public async shutDown(): Promise<void> {
+    const allAugmentations = this.getAllAugmentations()
+
+    await Promise.all(
+      allAugmentations.map(augmentation =>
+        augmentation.shutDown().catch(error => {
+          console.error(`Failed to shut down augmentation ${augmentation.name}:`, error)
+        })
+      )
+    )
+  }
+
+  /**
+   * Get all registered augmentations
+   *
+   * @returns An array of all registered augmentations
+   */
+  public getAllAugmentations(): IAugmentation[] {
+    // Create a Set to avoid duplicates (an augmentation might be in multiple registries)
+    const allAugmentations = new Set<IAugmentation>([
+      ...this.registry.sense,
+      ...this.registry.conduit,
+      ...this.registry.cognition,
+      ...this.registry.memory,
+      ...this.registry.perception,
+      ...this.registry.dialog,
+      ...this.registry.activation,
+      ...this.registry.webSocket
+    ])
+
+    // Convert back to array
+    return Array.from(allAugmentations)
+  }
+
+  /**
+   * Get all augmentations of a specific type
+   *
+   * @param type The type of augmentation to get
+   * @returns An array of all augmentations of the specified type
+   */
+  public getAugmentationsByType(type: AugmentationType): IAugmentation[] {
+    switch (type) {
+      case AugmentationType.SENSE:
+        return [...this.registry.sense]
+      case AugmentationType.CONDUIT:
+        return [...this.registry.conduit]
+      case AugmentationType.COGNITION:
+        return [...this.registry.cognition]
+      case AugmentationType.MEMORY:
+        return [...this.registry.memory]
+      case AugmentationType.PERCEPTION:
+        return [...this.registry.perception]
+      case AugmentationType.DIALOG:
+        return [...this.registry.dialog]
+      case AugmentationType.ACTIVATION:
+        return [...this.registry.activation]
+      case AugmentationType.WEBSOCKET:
+        return [...this.registry.webSocket]
+      default:
+        return []
+    }
+  }
+
+  /**
+   * Get all available augmentation types
+   *
+   * @returns An array of all augmentation types that have at least one registered augmentation
+   */
+  public getAvailableAugmentationTypes(): AugmentationType[] {
+    const availableTypes: AugmentationType[] = []
+
+    if (this.registry.sense.length > 0) availableTypes.push(AugmentationType.SENSE)
+    if (this.registry.conduit.length > 0) availableTypes.push(AugmentationType.CONDUIT)
+    if (this.registry.cognition.length > 0) availableTypes.push(AugmentationType.COGNITION)
+    if (this.registry.memory.length > 0) availableTypes.push(AugmentationType.MEMORY)
+    if (this.registry.perception.length > 0) availableTypes.push(AugmentationType.PERCEPTION)
+    if (this.registry.dialog.length > 0) availableTypes.push(AugmentationType.DIALOG)
+    if (this.registry.activation.length > 0) availableTypes.push(AugmentationType.ACTIVATION)
+    if (this.registry.webSocket.length > 0) availableTypes.push(AugmentationType.WEBSOCKET)
+
+    return availableTypes
+  }
+
+  /**
+   * Get all WebSocket-supporting augmentations
+   *
+   * @returns An array of all augmentations that support WebSocket connections
+   */
+  public getWebSocketAugmentations(): IWebSocketSupport[] {
+    return [...this.registry.webSocket]
+  }
+
+  /**
+   * Check if an augmentation is of a specific type
+   *
+   * @param augmentation The augmentation to check
+   * @param methods The methods that should be present on the augmentation
+   * @returns True if the augmentation is of the specified type
+   */
+  private isAugmentationType<T extends IAugmentation>(
+    augmentation: IAugmentation,
+    ...methods: (keyof T)[]
+  ): augmentation is T {
+    // First check that the augmentation has all the required base methods
+    const baseMethodsExist = [
+      'initialize',
+      'shutDown',
+      'getStatus'
+    ].every(method => typeof (augmentation as any)[method] === 'function')
+
+    if (!baseMethodsExist) {
+      return false
+    }
+
+    // Then check that it has all the specific methods for this type
+    return methods.every(method => typeof (augmentation as any)[method] === 'function')
+  }
+
+  /**
+   * Determines if threading should be used based on options and environment
+   * 
+   * @param options The pipeline options
+   * @returns True if threading should be used, false otherwise
+   */
+  private shouldUseThreading(options: PipelineOptions): boolean {
+    // If threading is explicitly disabled, don't use it
+    if (options.disableThreading) {
+      return false
+    }
+
+    // If threading is explicitly forced, use it if available
+    if (options.forceThreading) {
+      return isThreadingAvailable()
+    }
+
+    // If in THREADED mode, use threading if available
+    if (options.mode === ExecutionMode.THREADED) {
+      return isThreadingAvailable()
+    }
+
+    // Otherwise, don't use threading
+    return false
+  }
+
+  /**
+   * Executes a method on multiple augmentations using the specified execution mode
+   * 
+   * @param augmentations The augmentations to execute the method on
+   * @param method The method to execute
+   * @param args The arguments to pass to the method
+   * @param options Options for the execution
+   * @returns A promise that resolves with the results
+   */
+  public async execute<T>(
+    augmentations: IAugmentation[],
+    method: string,
+    args: any[] = [],
+    options: PipelineOptions = {}
+  ): Promise<PipelineResult<T>> {
+    const opts = { ...DEFAULT_PIPELINE_OPTIONS, ...options }
+    const enabledAugmentations = augmentations.filter(aug => aug.enabled !== false)
+
+    if (enabledAugmentations.length === 0) {
+      return { results: [], errors: [], successful: [] }
+    }
+
+    const result: PipelineResult<T> = {
+      results: [],
+      errors: [],
+      successful: []
+    }
+
+    // Create a function to execute with timeout
+    const executeWithTimeout = async (
+      augmentation: IAugmentation
+    ): Promise<AugmentationResponse<T>> => {
+      try {
+        // Create a timeout promise if a timeout is specified
+        if (opts.timeout) {
+          const timeoutPromise = new Promise<never>((_, reject) => {
+            setTimeout(() => {
+              reject(
+                new Error(`Timeout executing ${method} on ${augmentation.name}`)
+              )
+            }, opts.timeout)
+          })
+
+          // Check if threading should be used
+          const useThreading = this.shouldUseThreading(opts)
+
+          // Execute the method on the augmentation, using threading if appropriate
+          let methodPromise: Promise<AugmentationResponse<T>>
+
+          if (useThreading) {
+            // Execute in a separate thread
+            try {
+              // Create a function that can be serialized and executed in a worker
+              const workerFn = (...workerArgs: any[]) => {
+                // This function will be stringified and executed in the worker
+                // It needs to be self-contained
+                const augFn = augmentation[method as string] as Function
+                return augFn.apply(augmentation, workerArgs)
+              }
+
+              methodPromise = executeInThread<AugmentationResponse<T>>(workerFn.toString(), args)
+            } catch (threadError) {
+              console.warn(`Failed to execute in thread, falling back to main thread: ${threadError}`)
+              // Fall back to executing in the main thread
+              methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<T>)
+            }
+          } else {
+            // Execute in the main thread
+            methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<T>)
+          }
+
+          // Race the method promise against the timeout promise
+          return await Promise.race([methodPromise, timeoutPromise])
+        } else {
+          // No timeout, just execute the method
+          return await executeAugmentation<any, T>(augmentation, method, ...args)
+        }
+      } catch (error) {
+        result.errors.push(
+          error instanceof Error ? error : new Error(String(error))
+        )
+        return {
+          success: false,
+          data: null as any,
+          error: error instanceof Error ? error.message : String(error)
+        }
+      }
+    }
+
+    // Execute based on the specified mode
+    switch (opts.mode) {
+      case ExecutionMode.PARALLEL:
+      case ExecutionMode.THREADED:
+        // Execute all augmentations in parallel
+        result.results = await Promise.all(
+          enabledAugmentations.map((aug) => executeWithTimeout(aug))
+        )
+        break
+
+      case ExecutionMode.FIRST_SUCCESS:
+        // Execute augmentations sequentially until one succeeds
+        for (const augmentation of enabledAugmentations) {
+          const response = await executeWithTimeout(augmentation)
+          result.results.push(response)
+
+          if (response.success) {
+            break
+          }
+        }
+        break
+
+      case ExecutionMode.FIRST_RESULT:
+        // Execute augmentations sequentially until one returns a non-null result
+        for (const augmentation of enabledAugmentations) {
+          const response = await executeWithTimeout(augmentation)
+          result.results.push(response)
+
+          if (
+            response.success &&
+            response.data !== null &&
+            response.data !== undefined
+          ) {
+            break
+          }
+        }
+        break
+
+      case ExecutionMode.SEQUENTIAL:
+      default:
+        // Execute augmentations sequentially
+        for (const augmentation of enabledAugmentations) {
+          const response = await executeWithTimeout(augmentation)
+          result.results.push(response)
+
+          // Check if we need to stop on error
+          if (opts.stopOnError && !response.success) {
+            break
+          }
+        }
+        break
+    }
+
+    // Filter successful results
+    result.successful = result.results.filter((r) => r.success)
+
+    return result
+  }
+
+  /**
+   * Executes a method on augmentations of a specific type
+   * 
+   * @param type The type of augmentations to execute the method on
+   * @param method The method to execute
+   * @param args The arguments to pass to the method
+   * @param options Options for the execution
+   * @returns A promise that resolves with the results
+   */
+  public async executeByType<T>(
+    type: AugmentationType,
+    method: string,
+    args: any[] = [],
+    options: PipelineOptions = {}
+  ): Promise<PipelineResult<T>> {
+    const augmentations = this.getAugmentationsByType(type)
+    return this.execute<T>(augmentations, method, args, options)
+  }
+
+  /**
+   * Executes a method on a single augmentation with automatic error handling
+   * 
+   * @param augmentation The augmentation to execute the method on
+   * @param method The method to execute
+   * @param args The arguments to pass to the method
+   * @returns A promise that resolves with the result
+   */
+  public async executeSingle<T>(
+    augmentation: IAugmentation,
+    method: string,
+    ...args: any[]
+  ): Promise<AugmentationResponse<T>> {
+    return executeAugmentation<any, T>(augmentation, method, ...args)
+  }
+
+  /**
+   * Process static data through a pipeline of augmentations
+   * 
+   * @param data The data to process
+   * @param pipeline An array of processing steps, each with an augmentation, method, and optional args transformer
+   * @param options Options for the execution
+   * @returns A promise that resolves with the final result
+   */
+  public async processStaticData<T, R = any>(
+    data: T,
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    options: PipelineOptions = {}
+  ): Promise<AugmentationResponse<R>> {
+    let currentData = data
+    let prevResult: any = undefined
+
+    for (const step of pipeline) {
+      // Transform args if a transformer is provided, otherwise use the current data as the only arg
+      const args = step.transformArgs
+        ? step.transformArgs(currentData, prevResult)
+        : [currentData]
+
+      // Execute the method
+      const result = await this.executeSingle<any>(
+        step.augmentation,
+        step.method,
+        ...args
+      )
+
+      // If the step failed, return the error
+      if (!result.success) {
+        return result as AugmentationResponse<R>
+      }
+
+      // Update the current data for the next step
+      currentData = result.data
+      prevResult = result
+    }
+
+    // Return the final result
+    return prevResult as AugmentationResponse<R>
+  }
+
+  /**
+   * Process streaming data through a pipeline of augmentations
+   * 
+   * @param source The source augmentation that provides the data stream
+   * @param sourceMethod The method on the source augmentation that provides the data stream
+   * @param sourceArgs The arguments to pass to the source method
+   * @param pipeline An array of processing steps, each with an augmentation, method, and optional args transformer
+   * @param callback Function to call with the results of processing each data item
+   * @param options Options for the execution
+   * @returns A promise that resolves when the pipeline is set up
+   */
+  public async processStreamingData<T>(
+    source: IAugmentation,
+    sourceMethod: string,
+    sourceArgs: any[],
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    callback: (result: AugmentationResponse<T>) => void,
+    options: PipelineOptions = {}
+  ): Promise<void> {
+    // Create a chain of processors
+    const processData = async (data: any) => {
+      let currentData = data
+      let prevResult: any = undefined
+
+      for (const step of pipeline) {
+        // Transform args if a transformer is provided, otherwise use the current data as the only arg
+        const args = step.transformArgs
+          ? step.transformArgs(currentData, prevResult)
+          : [currentData]
+
+        // Execute the method
+        const result = await this.executeSingle<any>(
+          step.augmentation,
+          step.method,
+          ...args
+        )
+
+        // If the step failed, return the error
+        if (!result.success) {
+          callback(result as AugmentationResponse<T>)
+          return
+        }
+
+        // Update the current data for the next step
+        currentData = result.data
+        prevResult = result
+      }
+
+      // Call the callback with the final result
+      callback(prevResult as AugmentationResponse<T>)
+    }
+
+    // The last argument to the source method should be a callback that receives the data
+    const dataCallback = (data: any) => {
+      processData(data).catch((error) => {
+        console.error('Error processing streaming data:', error)
+        callback({
+          success: false,
+          data: null as any,
+          error: error instanceof Error ? error.message : String(error)
+        })
+      })
+    }
+
+    // Execute the source method with the provided args and the data callback
+    await this.executeSingle(source, sourceMethod, ...sourceArgs, dataCallback)
+  }
+
+  /**
+   * Create a reusable pipeline for processing data
+   * 
+   * @param pipeline An array of processing steps
+   * @param options Options for the execution
+   * @returns A function that processes data through the pipeline
+   */
+  public createPipeline<T, R>(
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    options: PipelineOptions = {}
+  ): (data: T) => Promise<AugmentationResponse<R>> {
+    return (data: T) => this.processStaticData<T, R>(data, pipeline, options)
+  }
+
+  /**
+   * Create a reusable streaming pipeline
+   * 
+   * @param source The source augmentation
+   * @param sourceMethod The method on the source augmentation
+   * @param pipeline An array of processing steps
+   * @param options Options for the execution
+   * @returns A function that sets up the streaming pipeline
+   */
+  public createStreamingPipeline<T, R>(
+    source: IAugmentation,
+    sourceMethod: string,
+    pipeline: Array<{
+      augmentation: IAugmentation
+      method: string
+      transformArgs?: (data: any, prevResult?: any) => any[]
+    }>,
+    options: PipelineOptions = {}
+  ): (
+    sourceArgs: any[],
+    callback: (result: AugmentationResponse<R>) => void
+  ) => Promise<void> {
+    return (
+      sourceArgs: any[],
+      callback: (result: AugmentationResponse<R>) => void
+    ) =>
+      this.processStreamingData<R>(
+        source,
+        sourceMethod,
+        sourceArgs,
+        pipeline,
+        callback,
+        options
+      )
+  }
+
+  // Legacy methods for backward compatibility
+
+  /**
+   * Execute a sense pipeline (legacy method)
+   */
+  public async executeSensePipeline<
+    M extends keyof BrainyAugmentations.ISenseAugmentation & string,
+    R extends BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ISenseAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ISenseAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.SENSE, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a conduit pipeline (legacy method)
+   */
+  public async executeConduitPipeline<
+    M extends keyof BrainyAugmentations.IConduitAugmentation & string,
+    R extends BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IConduitAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IConduitAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.CONDUIT, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a cognition pipeline (legacy method)
+   */
+  public async executeCognitionPipeline<
+    M extends keyof BrainyAugmentations.ICognitionAugmentation & string,
+    R extends BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.ICognitionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.ICognitionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.COGNITION, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a memory pipeline (legacy method)
+   */
+  public async executeMemoryPipeline<
+    M extends keyof BrainyAugmentations.IMemoryAugmentation & string,
+    R extends BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IMemoryAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IMemoryAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.MEMORY, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a perception pipeline (legacy method)
+   */
+  public async executePerceptionPipeline<
+    M extends keyof BrainyAugmentations.IPerceptionAugmentation & string,
+    R extends BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IPerceptionAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IPerceptionAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.PERCEPTION, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute a dialog pipeline (legacy method)
+   */
+  public async executeDialogPipeline<
+    M extends keyof BrainyAugmentations.IDialogAugmentation & string,
+    R extends BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IDialogAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IDialogAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.DIALOG, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+
+  /**
+   * Execute an activation pipeline (legacy method)
+   */
+  public async executeActivationPipeline<
+    M extends keyof BrainyAugmentations.IActivationAugmentation & string,
+    R extends BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => AugmentationResponse<infer U> ? U : never
+  >(
+    method: M & (BrainyAugmentations.IActivationAugmentation[M] extends (...args: any[]) => any ? M : never),
+    args: Parameters<Extract<BrainyAugmentations.IActivationAugmentation[M], (...args: any[]) => any>>,
+    options: PipelineOptions = {}
+  ): Promise<Promise<{ success: boolean; data: R; error?: string }>[]> {
+    const result = await this.executeByType<R>(AugmentationType.ACTIVATION, method, args, options)
+    return result.results.map(r => Promise.resolve(r))
+  }
+}
+
+// Create and export a default instance of the pipeline
+export const pipeline = new Pipeline()
+
+// Set the default pipeline instance for the augmentation registry
+// This breaks the circular dependency between pipeline.ts and augmentationRegistry.ts
+setDefaultPipeline(pipeline)
+
+// Re-export the legacy pipeline for backward compatibility
+export const augmentationPipeline = pipeline
+
+// Re-export the streamlined execution functions for backward compatibility
+export const executeStreamlined = <T>(
+  augmentations: IAugmentation[],
+  method: string,
+  args: any[] = [],
+  options: PipelineOptions = {}
+): Promise<PipelineResult<T>> => {
+  return pipeline.execute<T>(augmentations, method, args, options)
+}
+
+export const executeByType = <T>(
+  type: AugmentationType,
+  method: string,
+  args: any[] = [],
+  options: PipelineOptions = {}
+): Promise<PipelineResult<T>> => {
+  return pipeline.executeByType<T>(type, method, args, options)
+}
+
+export const executeSingle = <T>(
+  augmentation: IAugmentation,
+  method: string,
+  ...args: any[]
+): Promise<AugmentationResponse<T>> => {
+  return pipeline.executeSingle<T>(augmentation, method, ...args)
+}
+
+export const processStaticData = <T, R = any>(
+  data: T,
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  options: PipelineOptions = {}
+): Promise<AugmentationResponse<R>> => {
+  return pipeline.processStaticData<T, R>(data, pipelineSteps, options)
+}
+
+export const processStreamingData = <T>(
+  source: IAugmentation,
+  sourceMethod: string,
+  sourceArgs: any[],
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  callback: (result: AugmentationResponse<T>) => void,
+  options: PipelineOptions = {}
+): Promise<void> => {
+  return pipeline.processStreamingData<T>(source, sourceMethod, sourceArgs, pipelineSteps, callback, options)
+}
+
+export const createPipeline = <T, R>(
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  options: PipelineOptions = {}
+): (data: T) => Promise<AugmentationResponse<R>> => {
+  return pipeline.createPipeline<T, R>(pipelineSteps, options)
+}
+
+export const createStreamingPipeline = <T, R>(
+  source: IAugmentation,
+  sourceMethod: string,
+  pipelineSteps: Array<{
+    augmentation: IAugmentation
+    method: string
+    transformArgs?: (data: any, prevResult?: any) => any[]
+  }>,
+  options: PipelineOptions = {}
+): (
+  sourceArgs: any[],
+  callback: (result: AugmentationResponse<R>) => void
+) => Promise<void> => {
+  return pipeline.createStreamingPipeline<T, R>(source, sourceMethod, pipelineSteps, options)
+}
+
+// For backward compatibility with StreamlinedExecutionMode
+export const StreamlinedExecutionMode = ExecutionMode
+export type StreamlinedPipelineOptions = PipelineOptions
+export type StreamlinedPipelineResult<T> = PipelineResult<T>

src/sequentialPipeline.ts

@@ -0,0 +1,572 @@
+/**
+ * Sequential Augmentation Pipeline
+ *
+ * This module provides a pipeline for executing augmentations in a specific sequence:
+ * ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
+ *
+ * It supports high-performance streaming data from WebSockets without blocking.
+ * Optimized for Node.js 23.11+ using native WebStreams API.
+ */
+
+import {
+  AugmentationType,
+  IAugmentation,
+  IWebSocketSupport,
+  ISenseAugmentation,
+  IMemoryAugmentation,
+  ICognitionAugmentation,
+  IConduitAugmentation,
+  IActivationAugmentation,
+  IPerceptionAugmentation,
+  AugmentationResponse,
+  WebSocketConnection
+} from './types/augmentations.js'
+import { BrainyData } from './brainyData.js'
+import { augmentationPipeline } from './augmentationPipeline.js'
+// Use the browser's built-in WebStreams API or Node.js native WebStreams API
+// This approach ensures compatibility with both environments
+let TransformStream: any, ReadableStream: any, WritableStream: any;
+
+// Function to initialize the stream classes
+const initializeStreamClasses = () => {
+  // Try to use the browser's built-in WebStreams API first
+  if (typeof globalThis.TransformStream !== 'undefined' &&
+      typeof globalThis.ReadableStream !== 'undefined' &&
+      typeof globalThis.WritableStream !== 'undefined') {
+    TransformStream = globalThis.TransformStream;
+    ReadableStream = globalThis.ReadableStream;
+    WritableStream = globalThis.WritableStream;
+    return Promise.resolve();
+  } else {
+    // In Node.js environment, try to import from node:stream/web
+    // This will be executed in Node.js but not in browsers
+    return import('node:stream/web')
+      .then(streamWebModule => {
+        TransformStream = streamWebModule.TransformStream;
+        ReadableStream = streamWebModule.ReadableStream;
+        WritableStream = streamWebModule.WritableStream;
+      })
+      .catch(error => {
+        console.error('Failed to import WebStreams API:', error);
+        // Provide fallback implementations or throw a more helpful error
+        throw new Error('WebStreams API is not available in this environment. Please use a modern browser or Node.js 18+.');
+      });
+  }
+};
+
+// Initialize immediately but don't block module execution
+const streamClassesPromise = initializeStreamClasses();
+
+/**
+ * Options for sequential pipeline execution
+ */
+export interface SequentialPipelineOptions {
+  /**
+   * Timeout for each augmentation execution in milliseconds
+   */
+  timeout?: number;
+
+  /**
+   * Whether to stop execution if an error occurs
+   */
+  stopOnError?: boolean;
+
+  /**
+   * BrainyData instance to use for storage
+   */
+  brainyData?: BrainyData;
+}
+
+/**
+ * Default pipeline options
+ */
+const DEFAULT_SEQUENTIAL_PIPELINE_OPTIONS: SequentialPipelineOptions = {
+  timeout: 30000,
+  stopOnError: false
+}
+
+/**
+ * Result of a pipeline execution
+ */
+export interface PipelineResult<T> {
+  /**
+   * Whether the pipeline execution was successful
+   */
+  success: boolean;
+
+  /**
+   * The data returned by the pipeline
+   */
+  data: T;
+
+  /**
+   * Error message if the pipeline execution failed
+   */
+  error?: string;
+
+  /**
+   * Results from each stage of the pipeline
+   */
+  stageResults: {
+    sense?: AugmentationResponse<unknown>;
+    memory?: AugmentationResponse<unknown>;
+    cognition?: AugmentationResponse<unknown>;
+    conduit?: AugmentationResponse<unknown>;
+    activation?: AugmentationResponse<unknown>;
+    perception?: AugmentationResponse<unknown>;
+  }
+}
+
+/**
+ * SequentialPipeline class
+ *
+ * Executes augmentations in a specific sequence:
+ * ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
+ */
+export class SequentialPipeline {
+  private brainyData: BrainyData;
+
+  /**
+   * Create a new sequential pipeline
+   * 
+   * @param options Options for the pipeline
+   */
+  constructor(options: SequentialPipelineOptions = {}) {
+    this.brainyData = options.brainyData || new BrainyData();
+  }
+
+  /**
+   * Ensure stream classes are initialized
+   * @private
+   */
+  private async ensureStreamClassesInitialized(): Promise<void> {
+    await streamClassesPromise;
+  }
+
+  /**
+   * Initialize the pipeline
+   * 
+   * @returns A promise that resolves when initialization is complete
+   */
+  public async initialize(): Promise<void> {
+    // Initialize stream classes and BrainyData in parallel
+    await Promise.all([
+      this.ensureStreamClassesInitialized(),
+      this.brainyData.init()
+    ]);
+  }
+
+  /**
+   * Process data through the sequential pipeline
+   * 
+   * @param rawData The raw data to process
+   * @param dataType The type of data (e.g., 'text', 'image', 'audio')
+   * @param options Options for pipeline execution
+   * @returns A promise that resolves with the pipeline result
+   */
+  public async processData(
+    rawData: Buffer | string,
+    dataType: string,
+    options: SequentialPipelineOptions = {}
+  ): Promise<PipelineResult<unknown>> {
+    const opts = { ...DEFAULT_SEQUENTIAL_PIPELINE_OPTIONS, ...options };
+    const result: PipelineResult<unknown> = {
+      success: true,
+      data: null,
+      stageResults: {}
+    };
+
+    try {
+      // Step 1: Process raw data with ISense augmentations
+      const senseResults = await augmentationPipeline.executeSensePipeline(
+        'processRawData',
+        [rawData, dataType],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let senseResult: AugmentationResponse<{ nouns: string[], verbs: string[] }> | null = null;
+      for (const resultPromise of senseResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          senseResult = res;
+          break;
+        }
+      }
+
+      if (!senseResult || !senseResult.success) {
+        return {
+          success: false,
+          data: null,
+          error: 'Failed to process raw data with ISense augmentations',
+          stageResults: { sense: senseResult || { success: false, data: null, error: 'No sense augmentations available' } }
+        };
+      }
+
+      result.stageResults.sense = senseResult;
+
+      // Step 2: Store data in BrainyData using IMemory augmentations
+      const memoryAugmentations = augmentationPipeline.getAugmentationsByType(AugmentationType.MEMORY) as IMemoryAugmentation[];
+
+      if (memoryAugmentations.length === 0) {
+        return {
+          success: false,
+          data: null,
+          error: 'No memory augmentations available',
+          stageResults: result.stageResults
+        };
+      }
+
+      // Use the first available memory augmentation
+      const memoryAugmentation = memoryAugmentations[0];
+
+      // Generate a key for the data
+      const dataKey = `data_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
+
+      // Store the data
+      const memoryResult = await memoryAugmentation.storeData(
+        dataKey,
+        {
+          rawData,
+          dataType,
+          nouns: senseResult.data.nouns,
+          verbs: senseResult.data.verbs,
+          timestamp: Date.now()
+        }
+      );
+
+      if (!memoryResult.success) {
+        return {
+          success: false,
+          data: null,
+          error: `Failed to store data: ${memoryResult.error}`,
+          stageResults: { ...result.stageResults, memory: memoryResult }
+        };
+      }
+
+      result.stageResults.memory = memoryResult;
+
+      // Step 3: Trigger ICognition augmentations to analyze the data
+      const cognitionResults = await augmentationPipeline.executeCognitionPipeline(
+        'reason',
+        [`Analyze data with key ${dataKey}`, { dataKey }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let cognitionResult: AugmentationResponse<{ inference: string, confidence: number }> | null = null;
+      for (const resultPromise of cognitionResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          cognitionResult = res;
+          break;
+        }
+      }
+
+      if (cognitionResult) {
+        result.stageResults.cognition = cognitionResult;
+      }
+
+      // Step 4: Send notifications to IConduit augmentations
+      const conduitResults = await augmentationPipeline.executeConduitPipeline(
+        'writeData',
+        [{ dataKey, nouns: senseResult.data.nouns, verbs: senseResult.data.verbs }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let conduitResult: AugmentationResponse<unknown> | null = null;
+      for (const resultPromise of conduitResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          conduitResult = res;
+          break;
+        }
+      }
+
+      if (conduitResult) {
+        result.stageResults.conduit = conduitResult;
+      }
+
+      // Step 5: Send notifications to IActivation augmentations
+      const activationResults = await augmentationPipeline.executeActivationPipeline(
+        'triggerAction',
+        ['dataProcessed', { dataKey }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let activationResult: AugmentationResponse<unknown> | null = null;
+      for (const resultPromise of activationResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          activationResult = res;
+          break;
+        }
+      }
+
+      if (activationResult) {
+        result.stageResults.activation = activationResult;
+      }
+
+      // Step 6: Send notifications to IPerception augmentations
+      const perceptionResults = await augmentationPipeline.executePerceptionPipeline(
+        'interpret',
+        [senseResult.data.nouns, senseResult.data.verbs, { dataKey }],
+        { timeout: opts.timeout, stopOnError: opts.stopOnError }
+      );
+
+      // Get the first successful result
+      let perceptionResult: AugmentationResponse<Record<string, unknown>> | null = null;
+      for (const resultPromise of perceptionResults) {
+        const res = await resultPromise;
+        if (res.success) {
+          perceptionResult = res;
+          break;
+        }
+      }
+
+      if (perceptionResult) {
+        result.stageResults.perception = perceptionResult;
+        result.data = perceptionResult.data;
+      } else {
+        // If no perception result, use the cognition result as the final data
+        result.data = cognitionResult ? cognitionResult.data : { dataKey };
+      }
+
+      return result;
+    } catch (error: unknown) {
+      return {
+        success: false,
+        data: null,
+        error: `Pipeline execution failed: ${error}`,
+        stageResults: result.stageResults
+      };
+    }
+  }
+
+  /**
+   * Process WebSocket data through the sequential pipeline
+   * 
+   * @param connection The WebSocket connection
+   * @param dataType The type of data (e.g., 'text', 'image', 'audio')
+   * @param options Options for pipeline execution
+   * @returns A function to handle incoming WebSocket messages
+   */
+  public async createWebSocketHandler(
+    connection: WebSocketConnection,
+    dataType: string,
+    options: SequentialPipelineOptions = {}
+  ): Promise<(data: unknown) => void> {
+    // Ensure stream classes are initialized
+    await this.ensureStreamClassesInitialized();
+
+    // Create a transform stream for processing data
+    const transformStream = new TransformStream({
+      transform: async (chunk: unknown, controller: TransformStreamDefaultController) => {
+        try {
+          const data = typeof chunk === 'string' ? chunk : JSON.stringify(chunk);
+          const result = await this.processData(data, dataType, options);
+          if (result.success) {
+            controller.enqueue(result);
+          } else {
+            console.warn('Pipeline processing failed:', result.error);
+          }
+        } catch (error: unknown) {
+          console.error('Error in transform stream:', error);
+        }
+      }
+    });
+
+    // Create a writable stream that will be the sink for our data
+    const writableStream = new WritableStream({
+      write: async (result: PipelineResult<unknown>) => {
+        // Handle the processed result if needed
+        if (connection.send && typeof connection.send === 'function') {
+          try {
+            // Only send back results if the connection supports it
+            await connection.send(JSON.stringify(result));
+          } catch (error: unknown) {
+            console.error('Error sending result back to WebSocket:', error);
+          }
+        }
+      }
+    });
+
+    // Connect the transform stream to the writable stream
+    transformStream.readable.pipeTo(writableStream).catch((error: Error) => {
+      console.error('Error in pipeline stream:', error);
+    });
+
+    // Return a function that writes to the transform stream
+    return (data: unknown) => {
+      try {
+        // Write to the transform stream's writable side
+        const writer = transformStream.writable.getWriter();
+        writer.write(data).catch((error: Error) => {
+          console.error('Error writing to stream:', error);
+        }).finally(() => {
+          writer.releaseLock();
+        });
+      } catch (error: unknown) {
+        console.error('Error getting writer for transform stream:', error);
+      }
+    };
+  }
+
+  /**
+   * Set up a WebSocket connection to process data through the pipeline
+   * 
+   * @param url The WebSocket URL to connect to
+   * @param dataType The type of data (e.g., 'text', 'image', 'audio')
+   * @param options Options for pipeline execution
+   * @returns A promise that resolves with the WebSocket connection and associated streams
+   */
+  public async setupWebSocketPipeline(
+    url: string,
+    dataType: string,
+    options: SequentialPipelineOptions = {}
+  ): Promise<WebSocketConnection & { 
+    readableStream?: ReadableStream<unknown>,
+    writableStream?: WritableStream<unknown>
+  }> {
+    // Ensure stream classes are initialized
+    await this.ensureStreamClassesInitialized();
+
+    // Get WebSocket-supporting augmentations
+    const webSocketAugmentations = augmentationPipeline.getWebSocketAugmentations();
+
+    if (webSocketAugmentations.length === 0) {
+      throw new Error('No WebSocket-supporting augmentations available');
+    }
+
+    // Use the first available WebSocket augmentation
+    const webSocketAugmentation = webSocketAugmentations[0];
+
+    // Connect to the WebSocket
+    const connection = await webSocketAugmentation.connectWebSocket(url);
+
+    // Create a readable stream from the WebSocket messages
+    const readableStream = new ReadableStream({
+      start: (controller: ReadableStreamDefaultController) => {
+        // Define a message handler that writes to the stream
+        const messageHandler = (event: { data: unknown }) => {
+          try {
+            const data = typeof event.data === 'string' 
+              ? event.data 
+              : event.data instanceof Blob 
+                ? new Promise(resolve => {
+                    const reader = new FileReader();
+                    reader.onload = () => resolve(reader.result as string);
+                    reader.readAsText(event.data as Blob);
+                  })
+                : JSON.stringify(event.data);
+
+            // Handle both string data and promises
+            if (data instanceof Promise) {
+              data.then(resolvedData => {
+                controller.enqueue(resolvedData);
+              }).catch((error: Error) => {
+                console.error('Error processing blob data:', error);
+              });
+            } else {
+              controller.enqueue(data);
+            }
+          } catch (error: unknown) {
+            console.error('Error processing WebSocket message:', error);
+          }
+        };
+
+        // Create a wrapper function that adapts the event-based handler to the data-based callback
+        const messageHandlerWrapper = (data: unknown) => {
+          messageHandler({ data });
+        };
+
+        // Store both handlers for later cleanup
+        connection._streamMessageHandler = messageHandler;
+        connection._messageHandlerWrapper = messageHandlerWrapper;
+
+        webSocketAugmentation.onWebSocketMessage(
+          connection.connectionId, 
+          messageHandlerWrapper
+        ).catch((error: Error) => {
+          console.error('Error registering WebSocket message handler:', error);
+          controller.error(error);
+        });
+      },
+      cancel: () => {
+        // Clean up the message handler when the stream is cancelled
+        if (connection._messageHandlerWrapper) {
+          webSocketAugmentation.offWebSocketMessage(
+            connection.connectionId, 
+            connection._messageHandlerWrapper
+          ).catch((error: Error) => {
+            console.error('Error removing WebSocket message handler:', error);
+          });
+          delete connection._streamMessageHandler;
+          delete connection._messageHandlerWrapper;
+        }
+      }
+    });
+
+    // Create a handler for processing the data
+    const handlerPromise = this.createWebSocketHandler(connection, dataType, options);
+
+    // Create a writable stream that sends data to the WebSocket
+    const writableStream = new WritableStream({
+      write: async (chunk: unknown) => {
+        if (connection.send && typeof connection.send === 'function') {
+          try {
+            const data = typeof chunk === 'string' ? chunk : JSON.stringify(chunk);
+            await connection.send(data);
+          } catch (error: unknown) {
+            console.error('Error sending data to WebSocket:', error);
+            throw error;
+          }
+        } else {
+          throw new Error('WebSocket connection does not support sending data');
+        }
+      },
+      close: () => {
+        // Close the WebSocket connection when the stream is closed
+        if (connection.close && typeof connection.close === 'function') {
+          connection.close().catch((error: Error) => {
+            console.error('Error closing WebSocket connection:', error);
+          });
+        }
+      }
+    });
+
+    // Pipe the readable stream through our processing pipeline
+    readableStream
+      .pipeThrough(new TransformStream({
+        transform: async (chunk: unknown, controller: TransformStreamDefaultController) => {
+          // Process each chunk through our handler
+          const handler = await handlerPromise;
+          handler(chunk);
+          // Pass through the original data
+          controller.enqueue(chunk);
+        }
+      }))
+      .pipeTo(new WritableStream({
+        write: () => {},
+        abort: (error: Error) => {
+          console.error('Error in WebSocket pipeline:', error);
+        }
+      }));
+
+    // Attach the streams to the connection object for convenience
+    const enhancedConnection = connection as WebSocketConnection & {
+      readableStream: ReadableStream<unknown>,
+      writableStream: WritableStream<unknown>
+    };
+
+    enhancedConnection.readableStream = readableStream;
+    enhancedConnection.writableStream = writableStream;
+
+    return enhancedConnection;
+  }
+}
+
+// Create and export a default instance of the sequential pipeline
+export const sequentialPipeline = new SequentialPipeline();

src/storage/fileSystemStorage.ts

@@ -0,0 +1,931 @@
+import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
+
+// We'll dynamically import Node.js built-in modules
+let fs: any
+let path: any
+
+// Constants for directory and file names
+const ROOT_DIR = 'brainy-data'
+const NOUNS_DIR = 'nouns'
+const VERBS_DIR = 'verbs'
+const METADATA_DIR = 'metadata'
+
+// Constants for noun type directories
+const PERSON_DIR = 'person'
+const PLACE_DIR = 'place'
+const THING_DIR = 'thing'
+const EVENT_DIR = 'event'
+const CONCEPT_DIR = 'concept'
+const CONTENT_DIR = 'content'
+const DEFAULT_DIR = 'default' // For nodes without a noun type
+
+/**
+ * File system storage adapter for Node.js environments
+ */
+export class FileSystemStorage implements StorageAdapter {
+  private rootDir: string
+  private nounsDir: string
+  private verbsDir: string
+  private metadataDir: string
+  private personDir: string
+  private placeDir: string
+  private thingDir: string
+  private eventDir: string
+  private conceptDir: string
+  private contentDir: string
+  private defaultDir: string
+  private isInitialized = false
+
+  constructor(rootDirectory?: string) {
+    // We'll set the paths in the init method after dynamically importing the modules
+    this.rootDir = rootDirectory || ''
+    this.nounsDir = ''
+    this.verbsDir = ''
+    this.metadataDir = ''
+    this.personDir = ''
+    this.placeDir = ''
+    this.thingDir = ''
+    this.eventDir = ''
+    this.conceptDir = ''
+    this.contentDir = ''
+    this.defaultDir = ''
+  }
+
+  /**
+   * Initialize the storage adapter
+   */
+  public async init(): Promise<void> {
+    if (this.isInitialized) {
+      return
+    }
+
+    try {
+      // Dynamically import Node.js built-in modules
+      try {
+        // Import the modules
+        const fsModule = await import('fs')
+        const pathModule = await import('path')
+
+        // Assign to our module-level variables
+        fs = fsModule.default || fsModule
+        path = pathModule.default || pathModule
+
+        // Now set up the directory paths
+        const rootDir = this.rootDir || process.cwd()
+        this.rootDir = path.resolve(rootDir, ROOT_DIR)
+        this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
+        this.verbsDir = path.join(this.rootDir, VERBS_DIR)
+        this.metadataDir = path.join(this.rootDir, METADATA_DIR)
+
+        // Set up noun type directory paths
+        this.personDir = path.join(this.nounsDir, PERSON_DIR)
+        this.placeDir = path.join(this.nounsDir, PLACE_DIR)
+        this.thingDir = path.join(this.nounsDir, THING_DIR)
+        this.eventDir = path.join(this.nounsDir, EVENT_DIR)
+        this.conceptDir = path.join(this.nounsDir, CONCEPT_DIR)
+        this.contentDir = path.join(this.nounsDir, CONTENT_DIR)
+        this.defaultDir = path.join(this.nounsDir, DEFAULT_DIR)
+      } catch (importError) {
+        throw new Error(`Failed to import Node.js modules: ${importError}. This adapter requires a Node.js environment.`)
+      }
+
+      // Create directories if they don't exist
+      await this.ensureDirectoryExists(this.rootDir)
+      await this.ensureDirectoryExists(this.nounsDir)
+      await this.ensureDirectoryExists(this.verbsDir)
+      await this.ensureDirectoryExists(this.metadataDir)
+
+      // Create noun type directories
+      await this.ensureDirectoryExists(this.personDir)
+      await this.ensureDirectoryExists(this.placeDir)
+      await this.ensureDirectoryExists(this.thingDir)
+      await this.ensureDirectoryExists(this.eventDir)
+      await this.ensureDirectoryExists(this.conceptDir)
+      await this.ensureDirectoryExists(this.contentDir)
+      await this.ensureDirectoryExists(this.defaultDir)
+
+      this.isInitialized = true
+    } catch (error) {
+      console.error('Failed to initialize file system storage:', error)
+      throw new Error(`Failed to initialize file system storage: ${error}`)
+    }
+  }
+
+  /**
+   * Save a node to storage
+   */
+  public async saveNoun(noun: HNSWNoun): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Convert connections Map to a serializable format
+      const serializableNode = {
+        ...noun,
+        connections: this.mapToObject(noun.connections, (set) => Array.from(set as Set<string>))
+      }
+
+      // Get the appropriate directory based on the node's metadata
+      const nodeDir = await this.getNodeDirectory(noun.id)
+
+      const filePath = path.join(nodeDir, `${noun.id}.json`)
+      await fs.promises.writeFile(
+        filePath,
+        JSON.stringify(serializableNode, null, 2),
+        'utf8'
+      )
+    } catch (error) {
+      console.error(`Failed to save node ${noun.id}:`, error)
+      throw new Error(`Failed to save node ${noun.id}: ${error}`)
+    }
+  }
+
+  /**
+   * Get a node from storage
+   */
+  public async getNoun(id: string): Promise<HNSWNoun | null> {
+    await this.ensureInitialized()
+
+    try {
+      // Get the appropriate directory based on the node's metadata
+      const nodeDir = await this.getNodeDirectory(id)
+
+      const filePath = path.join(nodeDir, `${id}.json`)
+
+      // Check if a file exists
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        // If the file doesn't exist in the expected directory, try the default directory
+        if (nodeDir !== this.defaultDir) {
+          const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
+          try {
+            await fs.promises.access(defaultFilePath)
+            // If found in default directory, use that path
+            const data = await fs.promises.readFile(defaultFilePath, 'utf8')
+            const parsedNode = JSON.parse(data)
+
+            // Convert serialized connections back to Map<number, Set<string>>
+            const connections = new Map<number, Set<string>>()
+            for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+              connections.set(Number(level), new Set(nodeIds as string[]))
+            }
+
+            return {
+              id: parsedNode.id,
+              vector: parsedNode.vector,
+              connections
+            }
+          } catch {
+            // If not found in default directory either, try all noun type directories
+            const directories = [
+              this.personDir,
+              this.placeDir,
+              this.thingDir,
+              this.eventDir,
+              this.conceptDir,
+              this.contentDir
+            ]
+
+            for (const dir of directories) {
+              if (dir === nodeDir) continue // Skip the already checked directory
+
+              const dirFilePath = path.join(dir, `${id}.json`)
+              try {
+                await fs.promises.access(dirFilePath)
+                // If found in this directory, use that path
+                const data = await fs.promises.readFile(dirFilePath, 'utf8')
+                const parsedNode = JSON.parse(data)
+
+                // Convert serialized connections back to Map<number, Set<string>>
+                const connections = new Map<number, Set<string>>()
+                for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+                  connections.set(Number(level), new Set(nodeIds as string[]))
+                }
+
+                return {
+                  id: parsedNode.id,
+                  vector: parsedNode.vector,
+                  connections
+                }
+              } catch {
+                // Continue to the next directory
+              }
+            }
+
+            return null // File doesn't exist in any directory
+          }
+        }
+        return null // File doesn't exist
+      }
+
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      const parsedNode = JSON.parse(data)
+
+      // Convert serialized connections back to Map<number, Set<string>>
+      const connections = new Map<number, Set<string>>()
+      for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+        connections.set(Number(level), new Set(nodeIds as string[]))
+      }
+
+      return {
+        id: parsedNode.id,
+        vector: parsedNode.vector,
+        connections
+      }
+    } catch (error) {
+      console.error(`Failed to get node ${id}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Get nodes by noun type
+   * @param nounType The noun type to filter by
+   * @returns Promise that resolves to an array of nodes of the specified noun type
+   */
+  public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
+    await this.ensureInitialized()
+
+    try {
+      // Determine the directory based on the noun type
+      let dir: string
+      switch (nounType) {
+        case 'person':
+          dir = this.personDir
+          break
+        case 'place':
+          dir = this.placeDir
+          break
+        case 'thing':
+          dir = this.thingDir
+          break
+        case 'event':
+          dir = this.eventDir
+          break
+        case 'concept':
+          dir = this.conceptDir
+          break
+        case 'content':
+          dir = this.contentDir
+          break
+        default:
+          dir = this.defaultDir
+      }
+
+      const nodes: HNSWNoun[] = []
+
+      try {
+        const files = await fs.promises.readdir(dir)
+        const nodePromises = files
+          .filter((file: string) => file.endsWith('.json'))
+          .map((file: string) => {
+            // Use the file path directly instead of getNode to avoid redundant searches
+            return this.readNodeFromFile(path.join(dir, file))
+          })
+
+        const dirNodes = await Promise.all(nodePromises)
+        nodes.push(...dirNodes.filter((node): node is HNSWNoun => node !== null))
+      } catch (dirError) {
+        // If directory doesn't exist or can't be read, log a warning
+        console.warn(`Could not read directory for noun type ${nounType}:`, dirError)
+      }
+
+      return nodes
+    } catch (error) {
+      console.error(`Failed to get nodes for noun type ${nounType}:`, error)
+      throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
+    }
+  }
+
+  /**
+   * Get all nodes from storage
+   */
+  public async getAllNouns(): Promise<HNSWNoun[]> {
+    await this.ensureInitialized()
+
+    try {
+      // Get all noun types
+      const nounTypes = [
+        'person',
+        'place',
+        'thing',
+        'event',
+        'concept',
+        'content',
+        'default'
+      ]
+
+      // Run searches in parallel for all noun types
+      const nodePromises = nounTypes.map(nounType => this.getNounsByNounType(nounType))
+      const nodeArrays = await Promise.all(nodePromises)
+
+      // Combine all results
+      const allNodes: HNSWNoun[] = []
+      for (const nodes of nodeArrays) {
+        allNodes.push(...nodes)
+      }
+
+      return allNodes
+    } catch (error) {
+      console.error('Failed to get all nodes:', error)
+      throw new Error(`Failed to get all nodes: ${error}`)
+    }
+  }
+
+  /**
+   * Read a node from a file
+   */
+  private async readNodeFromFile(filePath: string): Promise<HNSWNoun | null> {
+    try {
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      const parsedNode = JSON.parse(data)
+
+      // Convert serialized connections back to Map<number, Set<string>>
+      const connections = new Map<number, Set<string>>()
+      for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
+        connections.set(Number(level), new Set(nodeIds as string[]))
+      }
+
+      return {
+        id: parsedNode.id,
+        vector: parsedNode.vector,
+        connections
+      }
+    } catch (error) {
+      console.error(`Failed to read node from file ${filePath}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Delete a node from storage
+   */
+  public async deleteNoun(id: string): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Get the appropriate directory based on the node's metadata
+      const nodeDir = await this.getNodeDirectory(id)
+
+      const filePath = path.join(nodeDir, `${id}.json`)
+
+      // Check if a file exists before attempting to delete
+      try {
+        await fs.promises.access(filePath)
+        await fs.promises.unlink(filePath)
+        return // File found and deleted
+      } catch {
+        // If the file doesn't exist in the expected directory, try the default directory
+        if (nodeDir !== this.defaultDir) {
+          const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
+          try {
+            await fs.promises.access(defaultFilePath)
+            await fs.promises.unlink(defaultFilePath)
+            return // File found and deleted
+          } catch {
+            // If not found in default directory either, try all noun type directories
+            const directories = [
+              this.personDir,
+              this.placeDir,
+              this.thingDir,
+              this.eventDir,
+              this.conceptDir,
+              this.contentDir
+            ]
+
+            for (const dir of directories) {
+              if (dir === nodeDir) continue // Skip the already checked directory
+
+              const dirFilePath = path.join(dir, `${id}.json`)
+              try {
+                await fs.promises.access(dirFilePath)
+                await fs.promises.unlink(dirFilePath)
+                return // File found and deleted
+              } catch {
+                // Continue to the next directory
+              }
+            }
+
+            return // File doesn't exist in any directory, nothing to delete
+          }
+        }
+        return // File doesn't exist, nothing to delete
+      }
+    } catch (error) {
+      console.error(`Failed to delete node ${id}:`, error)
+      throw new Error(`Failed to delete node ${id}: ${error}`)
+    }
+  }
+
+  /**
+   * Save an edge to storage
+   */
+  public async saveVerb(verb: GraphVerb): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Convert connections Map to a serializable format
+      const serializableEdge = {
+        ...verb,
+        connections: this.mapToObject(verb.connections, (set) => Array.from(set as Set<string>))
+      }
+
+      const filePath = path.join(this.verbsDir, `${verb.id}.json`)
+      await fs.promises.writeFile(
+        filePath,
+        JSON.stringify(serializableEdge, null, 2),
+        'utf8'
+      )
+    } catch (error) {
+      console.error(`Failed to save edge ${verb.id}:`, error)
+      throw new Error(`Failed to save edge ${verb.id}: ${error}`)
+    }
+  }
+
+  /**
+   * Get an edge from storage
+   */
+  public async getVerb(id: string): Promise<GraphVerb | null> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.verbsDir, `${id}.json`)
+
+      // Check if a file exists
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        return null // File doesn't exist
+      }
+
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      const parsedEdge = JSON.parse(data)
+
+      // Convert serialized connections back to Map<number, Set<string>>
+      const connections = new Map<number, Set<string>>()
+      for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
+        connections.set(Number(level), new Set(nodeIds as string[]))
+      }
+
+      return {
+        id: parsedEdge.id,
+        vector: parsedEdge.vector,
+        connections,
+        sourceId: parsedEdge.sourceId,
+        targetId: parsedEdge.targetId,
+        type: parsedEdge.type,
+        weight: parsedEdge.weight,
+        metadata: parsedEdge.metadata
+      }
+    } catch (error) {
+      console.error(`Failed to get edge ${id}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Get all edges from storage
+   */
+  public async getAllVerbs(): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const files = await fs.promises.readdir(this.verbsDir)
+      const edgePromises = files
+        .filter((file: string) => file.endsWith('.json'))
+        .map((file: string) => {
+          const id = path.basename(file, '.json')
+          return this.getVerb(id)
+        })
+
+      const edges = await Promise.all(edgePromises)
+      return edges.filter((edge): edge is GraphVerb => edge !== null)
+    } catch (error) {
+      console.error('Failed to get all edges:', error)
+      throw new Error(`Failed to get all edges: ${error}`)
+    }
+  }
+
+  /**
+   * Get edges by source node ID
+   */
+  public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const allEdges = await this.getAllVerbs()
+      return allEdges.filter(edge => edge.sourceId === sourceId)
+    } catch (error) {
+      console.error(`Failed to get edges by source ${sourceId}:`, error)
+      throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
+    }
+  }
+
+  /**
+   * Get edges by target node ID
+   */
+  public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const allEdges = await this.getAllVerbs()
+      return allEdges.filter(edge => edge.targetId === targetId)
+    } catch (error) {
+      console.error(`Failed to get edges by target ${targetId}:`, error)
+      throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
+    }
+  }
+
+  /**
+   * Get edges by type
+   */
+  public async getVerbsByType(type: string): Promise<GraphVerb[]> {
+    await this.ensureInitialized()
+
+    try {
+      const allEdges = await this.getAllVerbs()
+      return allEdges.filter(edge => edge.type === type)
+    } catch (error) {
+      console.error(`Failed to get edges by type ${type}:`, error)
+      throw new Error(`Failed to get edges by type ${type}: ${error}`)
+    }
+  }
+
+  /**
+   * Delete an edge from storage
+   */
+  public async deleteVerb(id: string): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.verbsDir, `${id}.json`)
+
+      // Check if a file exists before attempting to delete
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        return // File doesn't exist, nothing to delete
+      }
+
+      await fs.promises.unlink(filePath)
+    } catch (error) {
+      console.error(`Failed to delete edge ${id}:`, error)
+      throw new Error(`Failed to delete edge ${id}: ${error}`)
+    }
+  }
+
+  /**
+   * Save metadata to storage
+   */
+  public async saveMetadata(id: string, metadata: any): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.metadataDir, `${id}.json`)
+      await fs.promises.writeFile(
+        filePath,
+        JSON.stringify(metadata, null, 2),
+        'utf8'
+      )
+    } catch (error) {
+      console.error(`Failed to save metadata for ${id}:`, error)
+      throw new Error(`Failed to save metadata for ${id}: ${error}`)
+    }
+  }
+
+  /**
+   * Get metadata from storage
+   */
+  public async getMetadata(id: string): Promise<any | null> {
+    await this.ensureInitialized()
+
+    try {
+      const filePath = path.join(this.metadataDir, `${id}.json`)
+
+      // Check if a file exists
+      try {
+        await fs.promises.access(filePath)
+      } catch {
+        return null // File doesn't exist
+      }
+
+      const data = await fs.promises.readFile(filePath, 'utf8')
+      return JSON.parse(data)
+    } catch (error) {
+      console.error(`Failed to get metadata for ${id}:`, error)
+      return null
+    }
+  }
+
+  /**
+   * Clear all data from storage
+   */
+  public async clear(): Promise<void> {
+    await this.ensureInitialized()
+
+    try {
+      // Delete and recreate the nodes, edges, and metadata directories
+      await this.deleteDirectory(this.nounsDir)
+      await this.deleteDirectory(this.verbsDir)
+      await this.deleteDirectory(this.metadataDir)
+
+      await this.ensureDirectoryExists(this.nounsDir)
+      await this.ensureDirectoryExists(this.verbsDir)
+      await this.ensureDirectoryExists(this.metadataDir)
+
+      // Create noun type directories
+      await this.ensureDirectoryExists(this.personDir)
+      await this.ensureDirectoryExists(this.placeDir)
+      await this.ensureDirectoryExists(this.thingDir)
+      await this.ensureDirectoryExists(this.eventDir)
+      await this.ensureDirectoryExists(this.conceptDir)
+      await this.ensureDirectoryExists(this.contentDir)
+      await this.ensureDirectoryExists(this.defaultDir)
+    } catch (error) {
+      console.error('Failed to clear storage:', error)
+      throw new Error(`Failed to clear storage: ${error}`)
+    }
+  }
+
+  /**
+   * Ensure the storage adapter is initialized
+   */
+  private async ensureInitialized(): Promise<void> {
+    if (!this.isInitialized) {
+      await this.init()
+    }
+  }
+
+  /**
+   * Ensure a directory exists, creating it if necessary
+   */
+  private async ensureDirectoryExists(dirPath: string): Promise<void> {
+    try {
+      await fs.promises.access(dirPath)
+    } catch {
+      // Directory doesn't exist, create it
+      await fs.promises.mkdir(dirPath, { recursive: true })
+    }
+  }
+
+  /**
+   * Delete a directory and all its contents recursively
+   */
+  private async deleteDirectory(dirPath: string): Promise<void> {
+    try {
+      const files = await fs.promises.readdir(dirPath)
+
+      for (const file of files) {
+        const filePath = path.join(dirPath, file)
+        const stats = await fs.promises.stat(filePath)
+
+        if (stats.isDirectory()) {
+          // Recursively delete subdirectories
+          await this.deleteDirectory(filePath)
+        } else {
+          // Delete files
+          await fs.promises.unlink(filePath)
+        }
+      }
+
+      // After all contents are deleted, remove the directory itself
+      await fs.promises.rmdir(dirPath)
+    } catch (error) {
+      // If the directory doesn't exist, that's fine
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        throw error
+      }
+    }
+  }
+
+  /**
+   * Count the number of JSON files in a directory
+   */
+  private async countFilesInDirectory(dirPath: string): Promise<number> {
+    try {
+      const files = await fs.promises.readdir(dirPath)
+      return files.filter((file: string) => file.endsWith('.json')).length
+    } catch (error) {
+      // If the directory doesn't exist, return 0
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+        return 0
+      }
+      throw error
+    }
+  }
+
+  /**
+   * Convert a Map to a plain object for serialization
+   */
+  private mapToObject<K extends string | number, V>(
+    map: Map<K, V>,
+    valueTransformer: (value: V) => any = (v) => v
+  ): Record<string, any> {
+    const obj: Record<string, any> = {}
+    for (const [key, value] of map.entries()) {
+      obj[key.toString()] = valueTransformer(value)
+    }
+    return obj
+  }
+
+  /**
+   * Get the appropriate directory for a node based on its metadata
+   */
+  private async getNodeDirectory(id: string): Promise<string> {
+    try {
+      // Try to get the metadata for the node
+      const metadata = await this.getMetadata(id)
+
+      // If metadata exists and has a noun field, use the corresponding directory
+      if (metadata && metadata.noun) {
+        switch (metadata.noun) {
+          case 'person':
+            return this.personDir
+          case 'place':
+            return this.placeDir
+          case 'thing':
+            return this.thingDir
+          case 'event':
+            return this.eventDir
+          case 'concept':
+            return this.conceptDir
+          case 'content':
+            return this.contentDir
+          default:
+            return this.defaultDir
+        }
+      }
+
+      // If no metadata or no noun field, use the default directory
+      return this.defaultDir
+    } catch (error) {
+      // If there's an error getting the metadata, use the default directory
+      return this.defaultDir
+    }
+  }
+
+  /**
+   * Get information about storage usage and capacity
+   */
+  public async getStorageStatus(): Promise<{
+    type: string;
+    used: number;
+    quota: number | null;
+    details?: Record<string, any>;
+  }> {
+    await this.ensureInitialized()
+
+    try {
+      // Calculate the total size of all files in the storage directories
+      let totalSize = 0
+
+      // Helper function to calculate directory size
+      const calculateDirSize = async (dirPath: string): Promise<number> => {
+        let size = 0
+        try {
+          const files = await fs.promises.readdir(dirPath)
+
+          for (const file of files) {
+            const filePath = path.join(dirPath, file)
+            const stats = await fs.promises.stat(filePath)
+
+            if (stats.isDirectory()) {
+              size += await calculateDirSize(filePath)
+            } else {
+              size += stats.size
+            }
+          }
+        } catch (error) {
+          console.warn(`Error calculating size for ${dirPath}:`, error)
+        }
+        return size
+      }
+
+      // Calculate size for each directory
+      const nodesDirSize = await calculateDirSize(this.nounsDir)
+      const edgesDirSize = await calculateDirSize(this.verbsDir)
+      const metadataDirSize = await calculateDirSize(this.metadataDir)
+
+      // Calculate sizes of noun type directories
+      const personDirSize = await calculateDirSize(this.personDir)
+      const placeDirSize = await calculateDirSize(this.placeDir)
+      const thingDirSize = await calculateDirSize(this.thingDir)
+      const eventDirSize = await calculateDirSize(this.eventDir)
+      const conceptDirSize = await calculateDirSize(this.conceptDir)
+      const contentDirSize = await calculateDirSize(this.contentDir)
+      const defaultDirSize = await calculateDirSize(this.defaultDir)
+
+      // Note: The noun type directories are subdirectories of the nodes directory,
+      // so their sizes are already included in nodesDirSize.
+      // We don't need to add them again to avoid double counting.
+      totalSize = nodesDirSize + edgesDirSize + metadataDirSize
+
+      // Get filesystem information
+      let quota = null
+      let details: {
+        nounTypes?: {
+          person: { size: number; count: number };
+          place: { size: number; count: number };
+          thing: { size: number; count: number };
+          event: { size: number; count: number };
+          concept: { size: number; count: number };
+          content: { size: number; count: number };
+          default: { size: number; count: number };
+        };
+        availableSpace?: number;
+        totalSpace?: number;
+        freePercentage?: number;
+      } = {
+        nounTypes: {
+          person: {
+            size: personDirSize,
+            count: await this.countFilesInDirectory(this.personDir)
+          },
+          place: {
+            size: placeDirSize,
+            count: await this.countFilesInDirectory(this.placeDir)
+          },
+          thing: {
+            size: thingDirSize,
+            count: await this.countFilesInDirectory(this.thingDir)
+          },
+          event: {
+            size: eventDirSize,
+            count: await this.countFilesInDirectory(this.eventDir)
+          },
+          concept: {
+            size: conceptDirSize,
+            count: await this.countFilesInDirectory(this.conceptDir)
+          },
+          content: {
+            size: contentDirSize,
+            count: await this.countFilesInDirectory(this.contentDir)
+          },
+          default: {
+            size: defaultDirSize,
+            count: await this.countFilesInDirectory(this.defaultDir)
+          }
+        }
+      }
+
+      try {
+        // Try to get disk space information
+        const stats = await fs.promises.statfs(this.rootDir)
+        if (stats) {
+          const availableSpace = stats.bavail * stats.bsize
+          const totalSpace = stats.blocks * stats.bsize
+
+          quota = totalSpace
+          details = {
+            availableSpace,
+            totalSpace,
+            freePercentage: (availableSpace / totalSpace) * 100
+          }
+        }
+      } catch (error) {
+        console.warn('Unable to get filesystem stats:', error)
+        // If statfs is not available, try to use df command on Unix-like systems
+        try {
+          const { exec } = await import('child_process')
+          const util = await import('util')
+          const execPromise = util.promisify(exec)
+
+          const { stdout } = await execPromise(`df -k "${this.rootDir}"`)
+          const lines = stdout.trim().split('\n')
+          if (lines.length > 1) {
+            const parts = lines[1].split(/\s+/)
+            if (parts.length >= 4) {
+              const totalKB = parseInt(parts[1], 10)
+              const usedKB = parseInt(parts[2], 10)
+              const availableKB = parseInt(parts[3], 10)
+
+              quota = totalKB * 1024
+              details = {
+                availableSpace: availableKB * 1024,
+                totalSpace: totalKB * 1024,
+                freePercentage: (availableKB / totalKB) * 100
+              }
+            }
+          }
+        } catch (dfError) {
+          console.warn('Unable to get disk space using df command:', dfError)
+        }
+      }
+
+      return {
+        type: 'filesystem',
+        used: totalSize,
+        quota,
+        details
+      }
+    } catch (error) {
+      console.error('Failed to get storage status:', error)
+      return {
+        type: 'filesystem',
+        used: 0,
+        quota: null,
+        details: { error: String(error) }
+      }
+    }
+  }
+}

src/types/augmentations.ts

@@ -0,0 +1,413 @@
+/** Common types for the augmentation system */
+
+/**
+ * Enum representing all types of augmentations available in the Brainy system.
+ */
+export enum AugmentationType {
+  SENSE = 'sense',
+  CONDUIT = 'conduit',
+  COGNITION = 'cognition',
+  MEMORY = 'memory',
+  PERCEPTION = 'perception',
+  DIALOG = 'dialog',
+  ACTIVATION = 'activation',
+  WEBSOCKET = 'webSocket'
+}
+
+export type WebSocketConnection = {
+  connectionId: string
+  url: string
+  status: 'connected' | 'disconnected' | 'error'
+  send?: (data: string | ArrayBufferLike | Blob | ArrayBufferView) => Promise<void>
+  close?: () => Promise<void>
+  _streamMessageHandler?: (event: { data: unknown }) => void
+  _messageHandlerWrapper?: (data: unknown) => void
+}
+
+type DataCallback<T> = (data: T) => void
+export type AugmentationResponse<T> = {
+  success: boolean
+  data: T
+  error?: string
+}
+
+/**
+ * Base interface for all Brainy augmentations.
+ * All augmentations must implement these core properties.
+ */
+export interface IAugmentation {
+  /** A unique identifier for the augmentation (e.g., "my-reasoner-v1") */
+  readonly name: string
+  /** A human-readable description of the augmentation's purpose */
+  readonly description: string
+  /** Whether this augmentation is enabled */
+  enabled: boolean
+
+  /**
+   * Initializes the augmentation. This method is called when Brainy starts up.
+   * @returns A Promise that resolves when initialization is complete
+   */
+  initialize(): Promise<void>
+
+  shutDown(): Promise<void>
+
+  getStatus(): Promise<'active' | 'inactive' | 'error'>
+
+  // Allow string indexing for dynamic method access
+  [key: string]: any;
+}
+
+/**
+ * Interface for WebSocket support.
+ * Augmentations that implement this interface can communicate via WebSockets.
+ */
+export interface IWebSocketSupport extends IAugmentation {
+  /**
+   * Establishes a WebSocket connection.
+   * @param url The WebSocket server URL to connect to
+   * @param protocols Optional subprotocols
+   * @returns A Promise resolving to a connection handle or status
+   */
+  connectWebSocket(url: string, protocols?: string | string[]): Promise<WebSocketConnection>
+
+  /**
+   * Sends data through an established WebSocket connection.
+   * @param connectionId The identifier of the established connection
+   * @param data The data to send (will be serialized if not a string)
+   */
+  sendWebSocketMessage(connectionId: string, data: unknown): Promise<void>
+
+  /**
+   * Registers a callback for incoming WebSocket messages.
+   * @param connectionId The identifier of the established connection
+   * @param callback The function to call when a message is received
+   */
+  onWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
+
+  /**
+   * Removes a callback for incoming WebSocket messages.
+   * @param connectionId The identifier of the established connection
+   * @param callback The function to remove from the callbacks
+   */
+  offWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
+
+  /**
+   * Closes an established WebSocket connection.
+   * @param connectionId The identifier of the established connection
+   * @param code Optional close code
+   * @param reason Optional close reason
+   */
+  closeWebSocket(connectionId: string, code?: number, reason?: string): Promise<void>
+}
+
+export namespace BrainyAugmentations {
+  /**
+   * Interface for Senses augmentations.
+   * These augmentations ingest and process raw, unstructured data into nouns and verbs.
+   */
+  export interface ISenseAugmentation extends IAugmentation {
+    /**
+     * Processes raw input data into structured nouns and verbs.
+     * @param rawData The raw, unstructured data (e.g., text, image buffer, audio stream)
+     * @param dataType The type of raw data (e.g., 'text', 'image', 'audio')
+     */
+    processRawData(rawData: Buffer | string, dataType: string): Promise<AugmentationResponse<{
+      nouns: string[]
+      verbs: string[]
+    }>>
+
+    /**
+     * Registers a listener for real-time data feeds.
+     * @param feedUrl The URL or identifier of the real-time feed
+     * @param callback A function to call with processed data
+     */
+    listenToFeed(
+      feedUrl: string,
+      callback: DataCallback<{ nouns: string[]; verbs: string[] }>
+    ): Promise<void>
+  }
+
+  /**
+   * Interface for Conduits augmentations.
+   * These augmentations establish and manage high-bandwidth, dedicated channels for structured, programmatic two-way data exchange.
+   */
+  export interface IConduitAugmentation extends IAugmentation {
+    /**
+     * Establishes a connection for programmatic data exchange.
+     * @param targetSystemId The identifier of the external system to connect to
+     * @param config Configuration details for the connection (e.g., API keys, endpoints)
+     */
+    establishConnection(
+      targetSystemId: string,
+      config: Record<string, unknown>
+    ): Promise<AugmentationResponse<WebSocketConnection>>
+
+    /**
+     * Reads structured data directly from Brainy's knowledge graph.
+     * @param query A structured query (e.g., graph query language, object path)
+     * @param options Optional query options (e.g., depth, filters)
+     */
+    readData(
+      query: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<unknown>>
+
+    /**
+     * Writes or updates structured data directly into Brainy's knowledge graph.
+     * @param data The structured data to write/update
+     * @param options Optional write options (e.g., merge, overwrite)
+     */
+    writeData(
+      data: Record<string, unknown>,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<unknown>>
+
+    /**
+     * Monitors a specific data stream or event within Brainy for external systems.
+     * @param streamId The identifier of the data stream or event
+     * @param callback A function to call when new data/events occur
+     */
+    monitorStream(streamId: string, callback: DataCallback<unknown>): Promise<void>
+  }
+
+  /**
+   * Interface for Cognitions augmentations.
+   * These augmentations enable advanced reasoning, inference, and logical operations.
+   */
+  export interface ICognitionAugmentation extends IAugmentation {
+    /**
+     * Performs a reasoning operation based on current knowledge.
+     * @param query The specific reasoning task or question
+     * @param context Optional additional context for the reasoning
+     */
+    reason(query: string, context?: Record<string, unknown>): AugmentationResponse<{
+      inference: string
+      confidence: number
+    }>
+
+    /**
+     * Infers relationships or new facts from existing data.
+     * @param dataSubset A subset of data to infer from
+     */
+    infer(dataSubset: Record<string, unknown>): AugmentationResponse<Record<string, unknown>>
+
+    /**
+     * Executes a logical operation or rule set.
+     * @param ruleId The identifier of the rule or logic to apply
+     * @param input Data to apply the logic to
+     */
+    executeLogic(ruleId: string, input: Record<string, unknown>): AugmentationResponse<boolean>
+  }
+
+  /**
+   * Interface for Memory augmentations.
+   * These augmentations provide storage capabilities for data in different formats (e.g., fileSystem, in-memory).
+   */
+  export interface IMemoryAugmentation extends IAugmentation {
+    /**
+     * Stores data in the memory system.
+     * @param key The unique identifier for the data
+     * @param data The data to store
+     * @param options Optional storage options (e.g., expiration, format)
+     */
+    storeData(
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<boolean>>
+
+    /**
+     * Retrieves data from the memory system.
+     * @param key The unique identifier for the data
+     * @param options Optional retrieval options (e.g., format, version)
+     */
+    retrieveData(
+      key: string,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<unknown>>
+
+    /**
+     * Updates existing data in the memory system.
+     * @param key The unique identifier for the data
+     * @param data The updated data
+     * @param options Optional update options (e.g., merge, overwrite)
+     */
+    updateData(
+      key: string,
+      data: unknown,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<boolean>>
+
+    /**
+     * Deletes data from the memory system.
+     * @param key The unique identifier for the data
+     * @param options Optional deletion options
+     */
+    deleteData(
+      key: string,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<boolean>>
+
+    /**
+     * Lists available data keys in the memory system.
+     * @param pattern Optional pattern to filter keys (e.g., prefix, regex)
+     * @param options Optional listing options (e.g., limit, offset)
+     */
+    listDataKeys(
+      pattern?: string,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<string[]>>
+
+    /**
+     * Searches for data in the memory system using vector similarity.
+     * @param query The query vector or data to search for
+     * @param k Number of results to return
+     * @param options Optional search options
+     */
+    search(
+      query: unknown,
+      k?: number,
+      options?: Record<string, unknown>
+    ): Promise<AugmentationResponse<Array<{
+      id: string;
+      score: number;
+      data: unknown;
+    }>>>
+  }
+
+  /**
+   * Interface for Perceptions augmentations.
+   * These augmentations interpret, contextualize, and visualize identified nouns and verbs.
+   */
+  export interface IPerceptionAugmentation extends IAugmentation {
+    /**
+     * Interprets and contextualizes processed nouns and verbs.
+     * @param nouns The list of identified nouns
+     * @param verbs The list of identified verbs
+     * @param context Optional additional context for interpretation
+     */
+    interpret(
+      nouns: string[],
+      verbs: string[],
+      context?: Record<string, unknown>
+    ): AugmentationResponse<Record<string, unknown>>
+
+    /**
+     * Organizes and filters information.
+     * @param data The data to organize (e.g., interpreted perceptions)
+     * @param criteria Optional criteria for filtering/prioritization
+     */
+    organize(
+      data: Record<string, unknown>,
+      criteria?: Record<string, unknown>
+    ): AugmentationResponse<Record<string, unknown>>
+
+    /**
+     * Generates a visualization based on the provided data.
+     * @param data The data to visualize (e.g., interpreted patterns)
+     * @param visualizationType The desired type of visualization (e.g., 'graph', 'chart')
+     */
+    generateVisualization(
+      data: Record<string, unknown>,
+      visualizationType: string
+    ): AugmentationResponse<string | Buffer | Record<string, unknown>>
+  }
+
+  /**
+   * Interface for Dialogs augmentations.
+   * These augmentations facilitate natural language understanding and generation for conversational interaction.
+   */
+  export interface IDialogAugmentation extends IAugmentation {
+    /**
+     * Processes a user's natural language input (query).
+     * @param naturalLanguageQuery The raw text query from the user
+     * @param sessionId An optional session ID for conversational context
+     */
+    processUserInput(naturalLanguageQuery: string, sessionId?: string): AugmentationResponse<{
+      intent: string
+      nouns: string[]
+      verbs: string[]
+      context: Record<string, unknown>
+    }>
+
+    /**
+     * Generates a natural language response based on Brainy's knowledge and interpreted input.
+     * @param interpretedInput The output from `processUserInput` or similar
+     * @param knowledgeContext Relevant knowledge retrieved from Brainy
+     * @param sessionId An optional session ID for conversational context
+     */
+    generateResponse(
+      interpretedInput: Record<string, unknown>,
+      knowledgeContext: Record<string, unknown>,
+      sessionId?: string
+    ): AugmentationResponse<string>
+
+    /**
+     * Manages and updates conversational context.
+     * @param sessionId The session ID
+     * @param contextUpdate The data to update the context with
+     */
+    manageContext(sessionId: string, contextUpdate: Record<string, unknown>): Promise<void>
+  }
+
+  /**
+   * Interface for Activations augmentations.
+   * These augmentations dictate how Brainy initiates actions, responses, or data manipulations.
+   */
+  export interface IActivationAugmentation extends IAugmentation {
+    /**
+     * Triggers an action based on a processed command or internal state.
+     * @param actionName The name of the action to trigger
+     * @param parameters Optional parameters for the action
+     */
+    triggerAction(
+      actionName: string,
+      parameters?: Record<string, unknown>
+    ): AugmentationResponse<unknown>
+
+    /**
+     * Generates an expressive output or response from Brainy.
+     * @param knowledgeId The identifier of the knowledge to express
+     * @param format The desired output format (e.g., 'text', 'json')
+     */
+    generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown>>
+
+    /**
+     * Interacts with an external system or API.
+     * @param systemId The identifier of the external system
+     * @param payload The data to send to the external system
+     */
+    interactExternal(systemId: string, payload: Record<string, unknown>): AugmentationResponse<unknown>
+  }
+}
+
+/** Direct exports of augmentation interfaces for easier imports */
+export interface ISenseAugmentation extends BrainyAugmentations.ISenseAugmentation {
+}
+
+export interface IConduitAugmentation extends BrainyAugmentations.IConduitAugmentation {
+}
+
+export interface ICognitionAugmentation extends BrainyAugmentations.ICognitionAugmentation {
+}
+
+export interface IMemoryAugmentation extends BrainyAugmentations.IMemoryAugmentation {
+}
+
+export interface IPerceptionAugmentation extends BrainyAugmentations.IPerceptionAugmentation {
+}
+
+export interface IDialogAugmentation extends BrainyAugmentations.IDialogAugmentation {
+}
+
+export interface IActivationAugmentation extends BrainyAugmentations.IActivationAugmentation {
+}
+
+/** WebSocket-enabled augmentation interfaces */
+export type IWebSocketSenseAugmentation = BrainyAugmentations.ISenseAugmentation & IWebSocketSupport
+export type IWebSocketConduitAugmentation = BrainyAugmentations.IConduitAugmentation & IWebSocketSupport
+export type IWebSocketCognitionAugmentation = BrainyAugmentations.ICognitionAugmentation & IWebSocketSupport
+export type IWebSocketMemoryAugmentation = BrainyAugmentations.IMemoryAugmentation & IWebSocketSupport
+export type IWebSocketPerceptionAugmentation = BrainyAugmentations.IPerceptionAugmentation & IWebSocketSupport
+export type IWebSocketDialogAugmentation = BrainyAugmentations.IDialogAugmentation & IWebSocketSupport
+export type IWebSocketActivationAugmentation = BrainyAugmentations.IActivationAugmentation & IWebSocketSupport

src/types/brainyDataInterface.ts

@@ -0,0 +1,55 @@
+/**
+ * BrainyDataInterface
+ * 
+ * This interface defines the methods from BrainyData that are used by serverSearchAugmentations.ts.
+ * It's used to break the circular dependency between brainyData.ts and serverSearchAugmentations.ts.
+ */
+
+import { Vector } from '../coreTypes.js'
+
+export interface BrainyDataInterface<T = unknown> {
+  /**
+   * Initialize the database
+   */
+  init(): Promise<void>
+
+  /**
+   * Get a noun by ID
+   * @param id The ID of the noun to get
+   */
+  get(id: string): Promise<unknown>
+
+  /**
+   * Add a vector or data to the database
+   * @param vectorOrData Vector or data to add
+   * @param metadata Optional metadata to associate with the vector
+   * @returns The ID of the added vector
+   */
+  add(vectorOrData: Vector | unknown, metadata?: T): Promise<string>
+
+  /**
+   * Search for text in the database
+   * @param text The text to search for
+   * @param limit Maximum number of results to return
+   * @returns Search results
+   */
+  searchText(text: string, limit?: number): Promise<unknown[]>
+
+  /**
+   * Create a relationship between two entities
+   * @param sourceId The ID of the source entity
+   * @param targetId The ID of the target entity
+   * @param relationType The type of relationship
+   * @param metadata Optional metadata about the relationship
+   * @returns The ID of the created relationship
+   */
+  relate(sourceId: string, targetId: string, relationType: string, metadata?: unknown): Promise<string>
+
+  /**
+   * Find entities similar to a given entity ID
+   * @param id ID of the entity to find similar entities for
+   * @param options Additional options
+   * @returns Array of search results with similarity scores
+   */
+  findSimilar(id: string, options?: { limit?: number }): Promise<unknown[]>
+}

src/types/fileSystemTypes.ts

@@ -0,0 +1,14 @@
+/**
+ * Type declarations for the File System Access API
+ * Extends the FileSystemDirectoryHandle interface to include the [Symbol.asyncIterator] method
+ */
+
+// Extend the FileSystemDirectoryHandle interface
+interface FileSystemDirectoryHandle {
+  [Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
+  keys(): AsyncIterableIterator<string>;
+  entries(): AsyncIterableIterator<[string, FileSystemHandle]>;
+}
+
+// Export something to make this a module
+export const fileSystemTypesLoaded = true;

src/types/graphTypes.ts

@@ -0,0 +1,150 @@
+// Common metadata types
+/**
+ * Represents a high-precision timestamp with seconds and nanoseconds
+ * Used for tracking creation and update times of graph elements
+ */
+interface Timestamp {
+  seconds: number
+  nanoseconds: number
+}
+
+/**
+ * Metadata about the creator/source of a graph noun
+ * Tracks which augmentation and model created the element
+ */
+interface CreatorMetadata {
+  augmentation: string // Name of the augmentation that created this element
+  version: string // Version of the augmentation
+}
+
+/**
+ * Base interface for nodes (nouns) in the graph
+ * Represents entities like people, places, things, etc.
+ */
+export interface GraphNoun {
+  id: string // Unique identifier for the noun
+  createdBy: CreatorMetadata // Information about what created this noun
+  noun: NounType // Type classification of the noun
+  createdAt: Timestamp // When the noun was created
+  updatedAt: Timestamp // When the noun was last updated
+  label?: string // Optional descriptive label
+  data?: Record<string, any> // Additional flexible data storage
+  embeddedVerbs?: EmbeddedGraphVerb[] // Optional embedded relationships
+  embedding?: number[] // Vector representation of the noun
+}
+
+/**
+ * Base interface for edges (verbs) in the graph
+ * Represents relationships between nouns
+ */
+export interface GraphVerb {
+  id: string // Unique identifier for the verb
+  source: string // ID of the source noun
+  target: string // ID of the target noun
+  label?: string // Optional descriptive label
+  verb: VerbType // Type of relationship
+  createdAt: Timestamp // When the verb was created
+  updatedAt: Timestamp // When the verb was last updated
+  data?: Record<string, any> // Additional flexible data storage
+  embedding?: number[] // Vector representation of the relationship
+  confidence?: number // Confidence score (0-1)
+  weight?: number // Strength/importance of the relationship
+}
+
+/**
+ * Version of GraphVerb for embedded relationships
+ * Used when the source is implicit from the parent document
+ */
+export type EmbeddedGraphVerb = Omit<GraphVerb, 'source'>
+
+// Proper Noun interfaces - extend GraphNoun with specific noun types
+
+/**
+ * Represents a person entity in the graph
+ */
+export interface Person extends GraphNoun {
+  noun: typeof NounType.Person
+}
+
+/**
+ * Represents a physical location in the graph
+ */
+export interface Place extends GraphNoun {
+  noun: typeof NounType.Place
+}
+
+/**
+ * Represents a physical or virtual object in the graph
+ */
+export interface Thing extends GraphNoun {
+  noun: typeof NounType.Thing
+}
+
+/**
+ * Represents an event or occurrence in the graph
+ */
+export interface Event extends GraphNoun {
+  noun: typeof NounType.Event
+}
+
+/**
+ * Represents an abstract concept or idea in the graph
+ */
+export interface Concept extends GraphNoun {
+  noun: typeof NounType.Concept
+}
+
+export interface Group extends GraphNoun {
+  noun: typeof NounType.Group
+}
+
+export interface List extends GraphNoun {
+  noun: typeof NounType.List
+}
+
+/**
+ * Represents content (text, media, etc.) in the graph
+ */
+export interface Content extends GraphNoun {
+  noun: typeof NounType.Content
+}
+
+/**
+ * Defines valid noun types for graph entities
+ * Used for categorizing different types of nodes
+ */
+
+export const NounType = {
+  Person: 'person', // Person entities
+  Place: 'place', // Physical locations
+  Thing: 'thing', // Physical or virtual objects
+  Event: 'event', // Events or occurrences
+  Concept: 'concept', // Abstract concepts or ideas
+  Content: 'content', // Content items
+  Group: 'group', // Groups of related entities
+  List: 'list', // Ordered collections of entities
+  Category: 'category' // Categories for content items including tags
+} as const
+export type NounType = (typeof NounType)[keyof typeof NounType]
+
+/**
+ * Defines valid verb types for relationships
+ * Used for categorizing different types of connections
+ */
+export const VerbType = {
+  AttributedTo: 'attributedTo', // Indicates attribution or authorship
+  Controls: 'controls', // Indicates control or ownership
+  Created: 'created', // Indicates creation or authorship
+  Earned: 'earned', // Indicates achievement or acquisition
+  Owns: 'owns', // Indicates ownership
+  MemberOf: 'memberOf', // Indicates membership or affiliation
+  RelatedTo: 'relatedTo', // Indicates family relationship
+  WorksWith: 'worksWith', // Indicates professional relationship
+  FriendOf: 'friendOf', // Indicates friendship
+  ReportsTo: 'reportsTo', // Indicates reporting relationship
+  Supervises: 'supervises', // Indicates supervisory relationship
+  Mentors: 'mentors', // Indicates mentorship relationship
+  Follows: 'follows', // Indicates the following relationship
+  Likes: 'likes' // Indicates liking relationship
+} as const
+export type VerbType = (typeof VerbType)[keyof typeof VerbType]

src/types/mcpTypes.ts

@@ -0,0 +1,149 @@
+/**
+ * Model Control Protocol (MCP) Types
+ * 
+ * This file defines the types and interfaces for the Model Control Protocol (MCP)
+ * implementation in Brainy. MCP allows external models to access Brainy data and
+ * use the augmentation pipeline as tools.
+ */
+
+/**
+ * MCP version information
+ */
+export const MCP_VERSION = '1.0.0'
+
+/**
+ * MCP request types
+ */
+export enum MCPRequestType {
+  DATA_ACCESS = 'data_access',
+  TOOL_EXECUTION = 'tool_execution',
+  SYSTEM_INFO = 'system_info',
+  AUTHENTICATION = 'authentication'
+}
+
+/**
+ * Base interface for all MCP requests
+ */
+export interface MCPRequest {
+  /** The type of request */
+  type: MCPRequestType
+  /** Request ID for tracking and correlation */
+  requestId: string
+  /** API version */
+  version: string
+  /** Authentication token (if required) */
+  authToken?: string
+}
+
+/**
+ * Interface for data access requests
+ */
+export interface MCPDataAccessRequest extends MCPRequest {
+  type: MCPRequestType.DATA_ACCESS
+  /** The data access operation to perform */
+  operation: 'get' | 'search' | 'add' | 'getRelationships'
+  /** Parameters for the operation */
+  parameters: Record<string, any>
+}
+
+/**
+ * Interface for tool execution requests
+ */
+export interface MCPToolExecutionRequest extends MCPRequest {
+  type: MCPRequestType.TOOL_EXECUTION
+  /** The name of the tool to execute */
+  toolName: string
+  /** Parameters for the tool */
+  parameters: Record<string, any>
+}
+
+/**
+ * Interface for system info requests
+ */
+export interface MCPSystemInfoRequest extends MCPRequest {
+  type: MCPRequestType.SYSTEM_INFO
+  /** The type of information to retrieve */
+  infoType: 'status' | 'availableTools' | 'version'
+}
+
+/**
+ * Interface for authentication requests
+ */
+export interface MCPAuthenticationRequest extends MCPRequest {
+  type: MCPRequestType.AUTHENTICATION
+  /** The authentication credentials */
+  credentials: {
+    apiKey?: string
+    username?: string
+    password?: string
+  }
+}
+
+/**
+ * Base interface for all MCP responses
+ */
+export interface MCPResponse {
+  /** Whether the request was successful */
+  success: boolean
+  /** The request ID from the original request */
+  requestId: string
+  /** API version */
+  version: string
+  /** Response data (if successful) */
+  data?: any
+  /** Error information (if unsuccessful) */
+  error?: {
+    code: string
+    message: string
+    details?: any
+  }
+}
+
+/**
+ * Interface for MCP tool definitions
+ */
+export interface MCPTool {
+  /** The name of the tool */
+  name: string
+  /** A description of what the tool does */
+  description: string
+  /** The parameters the tool accepts */
+  parameters: {
+    type: 'object'
+    properties: Record<string, {
+      type: string
+      description: string
+      enum?: string[]
+      required?: boolean
+    }>
+    required: string[]
+  }
+}
+
+/**
+ * Configuration options for MCP services
+ */
+export interface MCPServiceOptions {
+  /** Port for the WebSocket server */
+  wsPort?: number
+  /** Port for the REST server */
+  restPort?: number
+  /** Whether to enable authentication */
+  enableAuth?: boolean
+  /** API keys for authentication */
+  apiKeys?: string[]
+  /** Rate limiting configuration */
+  rateLimit?: {
+    /** Maximum number of requests per window */
+    maxRequests: number
+    /** Time window in milliseconds */
+    windowMs: number
+  }
+  /** CORS configuration for REST API */
+  cors?: {
+    /** Allowed origins */
+    origin: string | string[]
+    /** Whether to allow credentials */
+    credentials: boolean
+  }
+}

src/types/pipelineTypes.ts

@@ -0,0 +1,33 @@
+/**
+ * Pipeline Types
+ *
+ * This module provides shared types for the pipeline system to avoid circular dependencies.
+ */
+
+import {
+  BrainyAugmentations,
+  IWebSocketSupport,
+  IAugmentation
+} from './augmentations.js'
+
+/**
+ * Type definitions for the augmentation registry
+ */
+export type AugmentationRegistry = {
+  sense: BrainyAugmentations.ISenseAugmentation[];
+  conduit: BrainyAugmentations.IConduitAugmentation[];
+  cognition: BrainyAugmentations.ICognitionAugmentation[];
+  memory: BrainyAugmentations.IMemoryAugmentation[];
+  perception: BrainyAugmentations.IPerceptionAugmentation[];
+  dialog: BrainyAugmentations.IDialogAugmentation[];
+  activation: BrainyAugmentations.IActivationAugmentation[];
+  webSocket: IWebSocketSupport[];
+}
+
+/**
+ * Interface for the Pipeline class
+ * This is used to break circular dependencies between pipeline.ts and augmentationRegistry.ts
+ */
+export interface IPipeline {
+  register<T extends IAugmentation>(augmentation: T): IPipeline;
+}

src/types/tensorflowTypes.ts

@@ -0,0 +1,14 @@
+// Type declarations for TensorFlow.js models
+
+// This file is a placeholder for TensorFlow.js model types
+// We're using type assertions in the code instead of module declarations
+
+// Define some basic types that might be useful
+export interface TensorflowModel {
+  load(): Promise<any>;
+  embed(data: string[]): any;
+  dispose(): void;
+}
+
+// Export a dummy constant to make this a proper module
+export const tensorflowModelsLoaded = true;

src/unified.ts

@@ -0,0 +1,36 @@
+/**
+ * Unified entry point for Brainy
+ * This file exports everything from index.ts
+ * Environment detection is handled here and made available to all components
+ */
+
+// Export environment information
+export const environment = {
+  isBrowser: typeof window !== 'undefined',
+  isNode:
+    typeof process !== 'undefined' && process.versions && process.versions.node,
+  isServerless:
+    typeof window === 'undefined' &&
+    (typeof process === 'undefined' ||
+      !process.versions ||
+      !process.versions.node)
+}
+
+// Make environment information available globally
+if (typeof globalThis !== 'undefined') {
+  globalThis.__ENV__ = environment
+}
+
+// Log the detected environment
+console.log(
+  `Brainy running in ${
+    environment.isBrowser
+      ? 'browser'
+      : environment.isNode
+        ? 'Node.js'
+        : 'serverless/unknown'
+  } environment`
+)
+
+// Re-export everything from index.ts
+export * from './index.js'

src/utils/distance.ts

@@ -0,0 +1,86 @@
+/**
+ * Distance functions for vector similarity calculations
+ * Optimized for Node.js 23.11+ using enhanced array methods
+ */
+
+import { DistanceFunction, Vector } from '../coreTypes.js'
+
+/**
+ * Calculates the Euclidean distance between two vectors
+ * Lower values indicate higher similarity
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  const sum = a.reduce((acc, val, i) => {
+    const diff = val - b[i]
+    return acc + (diff * diff)
+  }, 0)
+
+  return Math.sqrt(sum)
+}
+
+/**
+ * Calculates the cosine distance between two vectors
+ * Lower values indicate higher similarity
+ * Range: 0 (identical) to 2 (opposite)
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce to calculate all values in a single pass
+  const { dotProduct, normA, normB } = a.reduce((acc, val, i) => {
+    return {
+      dotProduct: acc.dotProduct + (val * b[i]),
+      normA: acc.normA + (val * val),
+      normB: acc.normB + (b[i] * b[i])
+    };
+  }, { dotProduct: 0, normA: 0, normB: 0 });
+
+  if (normA === 0 || normB === 0) {
+    return 2 // Maximum distance for zero vectors
+  }
+
+  const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
+  // Convert cosine similarity (-1 to 1) to distance (0 to 2)
+  return 1 - similarity
+}
+
+/**
+ * Calculates the Manhattan (L1) distance between two vectors
+ * Lower values indicate higher similarity
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0)
+}
+
+/**
+ * Calculates the dot product similarity between two vectors
+ * Higher values indicate higher similarity
+ * Converted to a distance metric (lower is better)
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  const dotProduct = a.reduce((sum, val, i) => sum + (val * b[i]), 0)
+
+  // Convert to a distance metric (lower is better)
+  return -dotProduct
+}

src/utils/embedding.ts

@@ -0,0 +1,293 @@
+/**
+ * Embedding functions for converting data to vectors
+ */
+
+import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js'
+import { executeInThread } from './workerUtils.js'
+
+/**
+ * TensorFlow Universal Sentence Encoder embedding model
+ * This model provides high-quality text embeddings using TensorFlow.js
+ * The required TensorFlow.js dependencies are automatically installed with this package
+ */
+export class UniversalSentenceEncoder implements EmbeddingModel {
+  private model: any = null
+  private initialized = false
+  private tf: any = null
+  private use: any = null
+
+  /**
+   * Initialize the embedding model
+   */
+  public async init(): Promise<void> {
+    try {
+      // Save original console.warn
+      const originalWarn = console.warn
+
+      // Override console.warn to suppress TensorFlow.js Node.js backend message
+      console.warn = function (message?: any, ...optionalParams: any[]) {
+        if (
+          message &&
+          typeof message === 'string' &&
+          message.includes(
+            'Hi, looks like you are running TensorFlow.js in Node.js'
+          )
+        ) {
+          return // Suppress the specific warning
+        }
+        originalWarn(message, ...optionalParams)
+      }
+
+      // Define EPSILON flag before TensorFlow.js is loaded
+      // This prevents the "Cannot evaluate flag 'EPSILON': no evaluation function found" error
+      if (typeof window !== 'undefined') {
+        ;(window as any).EPSILON = 1e-7
+        // Define the flag with an evaluation function for TensorFlow.js
+        ;(window as any).ENV = (window as any).ENV || {}
+        ;(window as any).ENV.flagRegistry =
+          (window as any).ENV.flagRegistry || {}
+        ;(window as any).ENV.flagRegistry.EPSILON = {
+          evaluationFn: () => 1e-7
+        }
+      } else if (typeof global !== 'undefined') {
+        ;(global as any).EPSILON = 1e-7
+        // Define the flag with an evaluation function for TensorFlow.js
+        ;(global as any).ENV = (global as any).ENV || {}
+        ;(global as any).ENV.flagRegistry =
+          (global as any).ENV.flagRegistry || {}
+        ;(global as any).ENV.flagRegistry.EPSILON = {
+          evaluationFn: () => 1e-7
+        }
+      }
+
+      // Dynamically import TensorFlow.js and Universal Sentence Encoder
+      // Use type assertions to tell TypeScript these modules exist
+      this.tf = await import('@tensorflow/tfjs')
+      this.use = await import('@tensorflow-models/universal-sentence-encoder')
+
+      // Load the model
+      this.model = await this.use.load()
+      this.initialized = true
+
+      // Restore original console.warn
+      console.warn = originalWarn
+    } catch (error) {
+      console.error('Failed to initialize Universal Sentence Encoder:', error)
+      throw new Error(
+        `Failed to initialize Universal Sentence Encoder: ${error}`
+      )
+    }
+  }
+
+  /**
+   * Embed text into a vector using Universal Sentence Encoder
+   * @param data Text to embed
+   */
+  public async embed(data: string | string[]): Promise<Vector> {
+    if (!this.initialized) {
+      await this.init()
+    }
+
+    try {
+      // Handle different input types
+      let textToEmbed: string[]
+      if (typeof data === 'string') {
+        // Handle empty string case
+        if (data.trim() === '') {
+          // Return a zero vector of appropriate dimension (512 is the default for USE)
+          return new Array(512).fill(0)
+        }
+        textToEmbed = [data]
+      } else if (
+        Array.isArray(data) &&
+        data.every((item) => typeof item === 'string')
+      ) {
+        // Handle empty array or array with empty strings
+        if (data.length === 0 || data.every((item) => item.trim() === '')) {
+          return new Array(512).fill(0)
+        }
+        // Filter out empty strings
+        textToEmbed = data.filter((item) => item.trim() !== '')
+        if (textToEmbed.length === 0) {
+          return new Array(512).fill(0)
+        }
+      } else {
+        throw new Error(
+          'UniversalSentenceEncoder only supports string or string[] data'
+        )
+      }
+
+      // Get embeddings
+      const embeddings = await this.model.embed(textToEmbed)
+
+      // Convert to array and return the first embedding
+      const embeddingArray = await embeddings.array()
+      return embeddingArray[0]
+    } catch (error) {
+      console.error(
+        'Failed to embed text with Universal Sentence Encoder:',
+        error
+      )
+      throw new Error(
+        `Failed to embed text with Universal Sentence Encoder: ${error}`
+      )
+    }
+  }
+
+  /**
+   * Dispose of the model resources
+   */
+  public async dispose(): Promise<void> {
+    if (this.model && this.tf) {
+      try {
+        // Dispose of the model and tensors
+        this.model.dispose()
+        this.tf.disposeVariables()
+        this.initialized = false
+      } catch (error) {
+        console.error('Failed to dispose Universal Sentence Encoder:', error)
+      }
+    }
+    return Promise.resolve()
+  }
+}
+
+/**
+ * Create an embedding function from an embedding model
+ * @param model Embedding model to use
+ */
+export function createEmbeddingFunction(
+  model: EmbeddingModel
+): EmbeddingFunction {
+  return async (data: any): Promise<Vector> => {
+    return await model.embed(data)
+  }
+}
+
+/**
+ * Creates a TensorFlow-based Universal Sentence Encoder embedding function
+ * This is the required embedding function for all text embeddings
+ */
+export function createTensorFlowEmbeddingFunction(): EmbeddingFunction {
+  // Create a single shared instance of the model
+  const model = new UniversalSentenceEncoder()
+  let modelInitialized = false
+
+  return async (data: any): Promise<Vector> => {
+    try {
+      // Initialize the model if it hasn't been initialized yet
+      if (!modelInitialized) {
+        await model.init()
+        modelInitialized = true
+      }
+
+      return await model.embed(data)
+    } catch (error) {
+      console.error('Failed to use TensorFlow embedding:', error)
+      throw new Error(
+        `Universal Sentence Encoder is required but failed: ${error}`
+      )
+    }
+  }
+}
+
+/**
+ * Creates a TensorFlow-based Universal Sentence Encoder embedding function that runs in a separate thread
+ * This provides better performance for CPU-intensive embedding operations
+ * @param options Configuration options
+ * @returns An embedding function that runs in a separate thread
+ */
+export function createThreadedEmbeddingFunction(
+  options: { fallbackToMain?: boolean } = {}
+): EmbeddingFunction {
+  // Create a standard embedding function to use as fallback
+  const standardEmbedding = createTensorFlowEmbeddingFunction()
+
+  // Flag to track if we've fallen back to main thread
+  let useFallback = false
+
+  return async (data: any): Promise<Vector> => {
+    // If we've already determined that threading doesn't work, use the fallback
+    if (useFallback) {
+      return standardEmbedding(data)
+    }
+
+    try {
+      // Function to be executed in a worker thread
+      // This must be a regular function (not async) to avoid Promise cloning issues
+      const embedInWorker = (inputData: any) => {
+        // Return a plain object with the input data
+        // All async operations will be performed inside the worker
+        return { data: inputData }
+      }
+
+      // Worker implementation function that will be stringified and run in the worker
+      const workerImplementation = async ({ data }: { data: any }) => {
+        // We need to dynamically import TensorFlow.js and USE in the worker
+        const tf = await import('@tensorflow/tfjs')
+        const use = await import('@tensorflow-models/universal-sentence-encoder')
+
+        // Load the model
+        const model = await use.load()
+
+        // Handle different input types
+        let textToEmbed: string[]
+        if (typeof data === 'string') {
+          if (data.trim() === '') {
+            return new Array(512).fill(0)
+          }
+          textToEmbed = [data]
+        } else if (
+          Array.isArray(data) &&
+          data.every((item) => typeof item === 'string')
+        ) {
+          if (data.length === 0 || data.every((item) => item.trim() === '')) {
+            return new Array(512).fill(0)
+          }
+          textToEmbed = data.filter((item) => item.trim() !== '')
+          if (textToEmbed.length === 0) {
+            return new Array(512).fill(0)
+          }
+        } else {
+          throw new Error(
+            'UniversalSentenceEncoder only supports string or string[] data'
+          )
+        }
+
+        // Get embeddings
+        const embeddings = await model.embed(textToEmbed)
+
+        // Convert to array and return the first embedding
+        const embeddingArray = await embeddings.array()
+
+        // Dispose of the tensor to free memory
+        embeddings.dispose()
+
+        return embeddingArray[0]
+      }
+
+      // Execute the embedding function in a separate thread
+      // Pass the worker implementation as a string to avoid Promise cloning issues
+      return await executeInThread<Vector>(workerImplementation.toString(), embedInWorker(data))
+    } catch (error) {
+      // If threading fails and fallback is enabled, use the standard embedding function
+      if (options.fallbackToMain) {
+        console.warn('Threaded embedding failed, falling back to main thread:', error)
+        useFallback = true
+        return standardEmbedding(data)
+      }
+
+      // Otherwise, propagate the error
+      throw new Error(`Threaded embedding failed: ${error}`)
+    }
+  }
+}
+
+/**
+ * Default embedding function
+ * Uses UniversalSentenceEncoder for all text embeddings
+ * TensorFlow.js is required for this to work
+ * Uses threading when available for better performance
+ */
+export const defaultEmbeddingFunction: EmbeddingFunction =
+  createThreadedEmbeddingFunction({ fallbackToMain: true })

src/utils/environment.ts

@@ -0,0 +1,57 @@
+/**
+ * Utility functions for environment detection
+ */
+
+/**
+ * Check if code is running in a browser environment
+ */
+export function isBrowser(): boolean {
+  return typeof window !== 'undefined' && typeof document !== 'undefined';
+}
+
+/**
+ * Check if code is running in a Node.js environment
+ */
+export function isNode(): boolean {
+  return typeof process !== 'undefined' && 
+    process.versions != null && 
+    process.versions.node != null;
+}
+
+/**
+ * Check if code is running in a Web Worker environment
+ */
+export function isWebWorker(): boolean {
+  return typeof self === 'object' && 
+    self.constructor && 
+    self.constructor.name === 'DedicatedWorkerGlobalScope';
+}
+
+/**
+ * Check if Web Workers are available in the current environment
+ */
+export function areWebWorkersAvailable(): boolean {
+  return isBrowser() && typeof Worker !== 'undefined';
+}
+
+/**
+ * Check if Worker Threads are available in the current environment (Node.js)
+ */
+export function areWorkerThreadsAvailable(): boolean {
+  if (!isNode()) return false;
+  
+  try {
+    // Dynamic import to avoid errors in browser environments
+    require('worker_threads');
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Determine if threading is available in the current environment
+ */
+export function isThreadingAvailable(): boolean {
+  return areWebWorkersAvailable() || areWorkerThreadsAvailable();
+}

src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from './distance.js'
+export * from './embedding.js'

src/utils/version.ts

@@ -0,0 +1,6 @@
+/**
+ * This file is auto-generated during the build process.
+ * Do not modify this file directly.
+ */
+
+export const VERSION = '0.9.10';

src/utils/workerUtils.ts

@@ -0,0 +1,145 @@
+/**
+ * Utility functions for working with Web Workers and Worker Threads
+ */
+
+import { isNode, isBrowser } from './environment.js'
+
+/**
+ * Execute a function in a Web Worker (browser environment)
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
+  return new Promise((resolve, reject) => {
+    // Create a blob URL for the worker script
+    const workerScript = `
+      self.onmessage = async function(e) {
+        try {
+          const fn = ${fnString};
+          const result = await fn(e.data);
+          self.postMessage({ success: true, data: result });
+        } catch (error) {
+          self.postMessage({ 
+            success: false, 
+            error: error instanceof Error ? error.message : String(error) 
+          });
+        }
+      };
+    `
+
+    const blob = new Blob([workerScript], { type: 'application/javascript' })
+    const blobURL = URL.createObjectURL(blob)
+
+    // Create a worker
+    const worker = new Worker(blobURL)
+
+    // Set up message handling
+    worker.onmessage = function (
+      e: MessageEvent<{ success: boolean; data: T; error?: string }>
+    ) {
+      URL.revokeObjectURL(blobURL) // Clean up
+      worker.terminate() // Terminate the worker
+
+      if (e.data.success) {
+        resolve(e.data.data)
+      } else {
+        reject(new Error(e.data.error))
+      }
+    }
+
+    worker.onerror = function (error: ErrorEvent) {
+      URL.revokeObjectURL(blobURL) // Clean up
+      worker.terminate() // Terminate the worker
+      reject(error)
+    }
+
+    // Start the worker
+    worker.postMessage(args)
+  })
+}
+
+/**
+ * Execute a function in a Worker Thread (Node.js environment)
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInWorkerThread<T>(
+  fnString: string,
+  args: any
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    try {
+      // Dynamic import to avoid errors in browser environments
+      const { Worker } = require('worker_threads')
+
+      // Create a worker script
+      const workerScript = `
+        const { parentPort } = require('worker_threads');
+
+        parentPort.once('message', async (data) => {
+          try {
+            const fn = ${fnString};
+            const result = await fn(data);
+            parentPort.postMessage({ success: true, data: result });
+          } catch (error) {
+            parentPort.postMessage({ 
+              success: false, 
+              error: error instanceof Error ? error.message : String(error) 
+            });
+          }
+        });
+      `
+
+      // Create a worker
+      const worker = new Worker(workerScript, { eval: true })
+
+      // Set up message handling
+      worker.on(
+        'message',
+        (data: { success: boolean; data: T; error?: string }) => {
+          worker.terminate() // Terminate the worker
+
+          if (data.success) {
+            resolve(data.data)
+          } else {
+            reject(new Error(data.error))
+          }
+        }
+      )
+
+      worker.on('error', (error: Error) => {
+        worker.terminate() // Terminate the worker
+        reject(error)
+      })
+
+      // Start the worker
+      worker.postMessage(args)
+    } catch (error) {
+      reject(error)
+    }
+  })
+}
+
+/**
+ * Execute a function in a separate thread based on the environment
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInThread<T>(fnString: string, args: any): Promise<T> {
+  if (isBrowser()) {
+    return executeInWebWorker<T>(fnString, args)
+  } else if (isNode()) {
+    return executeInWorkerThread<T>(fnString, args)
+  } else {
+    // Fall back to executing in the main thread
+    // Parse the function from string and execute it
+    const fn = new Function('return ' + fnString)()
+    return Promise.resolve(fn(args) as T)
+  }
+}