**feat: enhance threading compatibility and refactor module loading**

- Refactored `TextEncoder` and `TextDecoder` in `src/utils/tensorflowUtils.ts` to handle environments without global availability, logging warnings and providing fallback implementations.
- Enhanced threading utilities in `src/utils/environment.ts`:
  - Introduced `areWorkerThreadsAvailableSync()` and `isThreadingAvailableAsync()` for synchronous and asynchronous checks.
  - Improved compatibility across browser and Node.js environments.
- Migrated `fileSystemStorage.ts` to use dynamic imports for Node.js modules, removing synchronous loading mechanisms to align with ES module standards.
- Updated `cli-package/package.json` keywords to include `browser` and `container`, improving discoverability.
- Added `worker_threads` as an external dependency in `cli-package/rollup.config.js` and `rollup.config.js`.
- Updated `.gitignore` to ignore all `tgz` files dynamically under main and CLI package.

This update improves threading compatibility, enhances environment-specific support, and aligns with modern module loading practices.

.gitignore

@@ -54,6 +54,6 @@ Thumbs.db
 /cli-package/node_modules/
 /npm
 /rollup
-/soulcraft-brainy-0.9.23.tgz
-/cli-package/soulcraft-brainy-cli-0.9.23.tgz
+/soulcraft-brainy-*.tgz
 /data/
+/cli-package/soulcraft-brainy-cli-*.tgz

cli-package/package-lock.json

@@ -1,16 +1,16 @@
 {
   "name": "@soulcraft/brainy-cli",
-  "version": "0.9.29",
+  "version": "0.9.30",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy-cli",
-      "version": "0.9.29",
+      "version": "0.9.30",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
-        "@soulcraft/brainy": "0.9.29",
+        "@soulcraft/brainy": "0.9.30",
         "commander": "^14.0.0",
         "omelette": "^0.4.17"
       },
@@ -2086,9 +2086,9 @@
       }
     },
     "node_modules/@soulcraft/brainy": {
-      "version": "0.9.29",
-      "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.29.tgz",
-      "integrity": "sha512-t1kECxPqO7Y8dC6Vh2BoWQKYtiHCUPombz6ZGNfiyhADMEf8T3E1dd7XGmAX8qUver9/aXKk1I1pfIXA7RP5Cw==",
+      "version": "0.9.30",
+      "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.30.tgz",
+      "integrity": "sha512-CPI0XQLwp2KIy31dK6mzJfbW+F36dx1pG0ORmoY6tDRtRe7w39PaW+fj0Q/p8RpaBxAK5PaBgl/0rN+8bmsrsA==",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {

cli-package/package.json

@@ -21,7 +21,8 @@
     "vector-database",
     "hnsw",
     "cli",
-    "command-line",
+    "browser",
+    "container",
     "graph-database"
   ],
   "author": "David Snelling (david@soulcraft.com)",

cli-package/rollup.config.js

@@ -37,6 +37,7 @@ export default {
     'fs',
     'path',
     'url',
-    'child_process'
+    'child_process',
+    'worker_threads'
   ]
 }

rollup.config.js

@@ -176,8 +176,11 @@ const mainConfig = {
     '@aws-crypto/crc32c',
     'node:stream/web',
     'node:worker_threads',
+    'worker_threads',
     'node:fs',
-    'node:path'
+    'node:path',
+    'fs',
+    'path'
   ]
 }
 

src/storage/fileSystemStorage.ts

@@ -1,51 +1,12 @@
 import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
 
 // Import Node.js built-in modules
-// Using require for compatibility with Node.js
+// Using dynamic imports for compatibility with ES modules
 let fs: any
 let path: any
 
-// Initialize these modules immediately if in Node.js environment
-if (
-  typeof process !== 'undefined' &&
-  process.versions &&
-  process.versions.node
-) {
-  try {
-    // Use require for Node.js built-in modules
-    fs = require('fs')
-    path = require('path')
-
-    // Verify that path has the required methods
-    if (!path || typeof path.resolve !== 'function') {
-      throw new Error('path module is missing required methods')
-    }
-  } catch (e) {
-    console.warn('Failed to load Node.js modules with require:', e)
-
-    // Try using dynamic import as a fallback
-    try {
-      // Use a synchronous approach to ensure modules are loaded before continuing
-      const pathModule = require('node:path')
-      const fsModule = require('node:fs')
-
-      path = pathModule
-      fs = fsModule
-
-      // Verify that path has the required methods
-      if (!path || typeof path.resolve !== 'function') {
-        throw new Error(
-          'path module from node:path is missing required methods'
-        )
-      }
-    } catch (nodeImportError) {
-      console.warn(
-        'Failed to load Node.js modules with node: prefix:',
-        nodeImportError
-      )
-    }
-  }
-}
+// We'll initialize these modules in the init() method
+// No synchronous loading here to avoid issues with ES modules
 
 // Constants for directory and file names
 const ROOT_DIR = 'brainy-data'
@@ -111,37 +72,7 @@ export class FileSystemStorage implements StorageAdapter {
 
         // Try multiple approaches to load the modules
         const loadAttempts = [
-          // Attempt 1: Use require
-          async () => {
-            console.log('Attempting to load Node.js modules with require()')
-            const fsModule = require('fs')
-            const pathModule = require('path')
-
-            if (!pathModule || typeof pathModule.resolve !== 'function') {
-              throw new Error('path.resolve is not a function after require()')
-            }
-
-            return { fs: fsModule, path: pathModule }
-          },
-
-          // Attempt 2: Use require with node: prefix
-          async () => {
-            console.log(
-              'Attempting to load Node.js modules with require("node:...")'
-            )
-            const fsModule = require('node:fs')
-            const pathModule = require('node:path')
-
-            if (!pathModule || typeof pathModule.resolve !== 'function') {
-              throw new Error(
-                'path.resolve is not a function after require("node:path")'
-              )
-            }
-
-            return { fs: fsModule, path: pathModule }
-          },
-
-          // Attempt 3: Use dynamic import
+          // Attempt 1: Use dynamic import
           async () => {
             console.log(
               'Attempting to load Node.js modules with dynamic import'
@@ -161,7 +92,7 @@ export class FileSystemStorage implements StorageAdapter {
             return { fs: fsResolved, path: pathResolved }
           },
 
-          // Attempt 4: Use dynamic import with node: prefix
+          // Attempt 2: Use dynamic import with node: prefix
           async () => {
             console.log(
               'Attempting to load Node.js modules with dynamic import("node:...")'

src/utils/environment.ts

@@ -6,53 +6,75 @@
  * Check if code is running in a browser environment
  */
 export function isBrowser(): boolean {
-  return typeof window !== 'undefined' && typeof document !== 'undefined';
+  return typeof window !== 'undefined' && typeof document !== 'undefined'
 }
 
 /**
  * Check if code is running in a Node.js environment
  */
 export function isNode(): boolean {
-  return typeof process !== 'undefined' && 
-    process.versions != null && 
-    process.versions.node != null;
+  return (
+    typeof process !== 'undefined' &&
+    process.versions != null &&
+    process.versions.node != null
+  )
 }
 
 /**
  * Check if code is running in a Web Worker environment
  */
 export function isWebWorker(): boolean {
-  return typeof self === 'object' && 
-    self.constructor && 
-    self.constructor.name === 'DedicatedWorkerGlobalScope';
+  return (
+    typeof self === 'object' &&
+    self.constructor &&
+    self.constructor.name === 'DedicatedWorkerGlobalScope'
+  )
 }
 
 /**
  * Check if Web Workers are available in the current environment
  */
 export function areWebWorkersAvailable(): boolean {
-  return isBrowser() && typeof Worker !== 'undefined';
+  return isBrowser() && typeof Worker !== 'undefined'
 }
 
 /**
  * Check if Worker Threads are available in the current environment (Node.js)
  */
-export function areWorkerThreadsAvailable(): boolean {
-  if (!isNode()) return false;
+export async function areWorkerThreadsAvailable(): Promise<boolean> {
+  if (!isNode()) return false
 
   try {
-    // Dynamic import to avoid errors in browser environments
-    require('worker_threads');
-    return true;
+    // Use dynamic import to avoid errors in browser environments
+    await import('worker_threads')
+    return true
   } catch (e) {
-    return false;
+    return false
   }
 }
 
+/**
+ * Synchronous version that doesn't actually try to load the module
+ * This is safer in ES module environments
+ */
+export function areWorkerThreadsAvailableSync(): boolean {
+  if (!isNode()) return false
+
+  // In Node.js 12+, worker_threads is available without requiring a flag
+  return parseInt(process.versions.node.split('.')[0]) >= 12
+}
+
 /**
  * Determine if threading is available in the current environment
  * Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
  */
 export function isThreadingAvailable(): boolean {
-  return areWebWorkersAvailable() || areWorkerThreadsAvailable();
+  return areWebWorkersAvailable() || areWorkerThreadsAvailableSync()
+}
+
+/**
+ * Async version of isThreadingAvailable
+ */
+export async function isThreadingAvailableAsync(): Promise<boolean> {
+  return areWebWorkersAvailable() || (await areWorkerThreadsAvailable())
 }

src/utils/tensorflowUtils.ts

@@ -33,13 +33,40 @@ if (
           isFloat32Array,
           isTypedArray,
           // Use the global TextEncoder/TextDecoder directly
-          TextEncoder: typeof TextEncoder !== 'undefined' ? TextEncoder : null,
-          TextDecoder: typeof TextDecoder !== 'undefined' ? TextDecoder : null
+          TextEncoder:
+            typeof TextEncoder !== 'undefined'
+              ? TextEncoder
+              : function () {
+                  console.warn(
+                    'TextEncoder constructor called but not available'
+                  )
+                  return { encode: (str: string) => new Uint8Array([]) }
+                },
+          TextDecoder:
+            typeof TextDecoder !== 'undefined'
+              ? TextDecoder
+              : function () {
+                  console.warn(
+                    'TextDecoder constructor called but not available'
+                  )
+                  return { decode: (bytes: Uint8Array) => '' }
+                }
         }
 
         // Initialize TextEncoder/TextDecoder directly from globals
-        this.textEncoder = new TextEncoder()
-        this.textDecoder = new TextDecoder()
+        if (typeof TextEncoder !== 'undefined') {
+          this.textEncoder = new TextEncoder()
+        } else {
+          console.warn('TextEncoder is not available in this environment')
+          this.textEncoder = { encode: (str: string) => new Uint8Array([]) }
+        }
+
+        if (typeof TextDecoder !== 'undefined') {
+          this.textDecoder = new TextDecoder()
+        } else {
+          console.warn('TextDecoder is not available in this environment')
+          this.textDecoder = { decode: (bytes: Uint8Array) => '' }
+        }
       }
     }