From 703fc9eda3f6c9ff9da10b72230a99f5298305c2 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Jul 2025 16:52:44 -0700 Subject: [PATCH] **feat: enhance threading compatibility and refactor module loading** - Refactored `TextEncoder` and `TextDecoder` in `src/utils/tensorflowUtils.ts` to handle environments without global availability, logging warnings and providing fallback implementations. - Enhanced threading utilities in `src/utils/environment.ts`: - Introduced `areWorkerThreadsAvailableSync()` and `isThreadingAvailableAsync()` for synchronous and asynchronous checks. - Improved compatibility across browser and Node.js environments. - Migrated `fileSystemStorage.ts` to use dynamic imports for Node.js modules, removing synchronous loading mechanisms to align with ES module standards. - Updated `cli-package/package.json` keywords to include `browser` and `container`, improving discoverability. - Added `worker_threads` as an external dependency in `cli-package/rollup.config.js` and `rollup.config.js`. - Updated `.gitignore` to ignore all `tgz` files dynamically under main and CLI package. This update improves threading compatibility, enhances environment-specific support, and aligns with modern module loading practices. --- .gitignore | 4 +- cli-package/package-lock.json | 12 ++--- cli-package/package.json | 3 +- cli-package/rollup.config.js | 3 +- rollup.config.js | 5 +- src/storage/fileSystemStorage.ts | 79 ++------------------------------ src/utils/environment.ts | 52 +++++++++++++++------ src/utils/tensorflowUtils.ts | 35 ++++++++++++-- 8 files changed, 89 insertions(+), 104 deletions(-) diff --git a/.gitignore b/.gitignore index 8c93526b..6dfb2b94 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,6 @@ Thumbs.db /cli-package/node_modules/ /npm /rollup -/soulcraft-brainy-0.9.23.tgz -/cli-package/soulcraft-brainy-cli-0.9.23.tgz +/soulcraft-brainy-*.tgz /data/ +/cli-package/soulcraft-brainy-cli-*.tgz diff --git a/cli-package/package-lock.json b/cli-package/package-lock.json index 187dd7cb..9a76c5da 100644 --- a/cli-package/package-lock.json +++ b/cli-package/package-lock.json @@ -1,16 +1,16 @@ { "name": "@soulcraft/brainy-cli", - "version": "0.9.29", + "version": "0.9.30", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy-cli", - "version": "0.9.29", + "version": "0.9.30", "hasInstallScript": true, "license": "MIT", "dependencies": { - "@soulcraft/brainy": "0.9.29", + "@soulcraft/brainy": "0.9.30", "commander": "^14.0.0", "omelette": "^0.4.17" }, @@ -2086,9 +2086,9 @@ } }, "node_modules/@soulcraft/brainy": { - "version": "0.9.29", - "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.29.tgz", - "integrity": "sha512-t1kECxPqO7Y8dC6Vh2BoWQKYtiHCUPombz6ZGNfiyhADMEf8T3E1dd7XGmAX8qUver9/aXKk1I1pfIXA7RP5Cw==", + "version": "0.9.30", + "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.30.tgz", + "integrity": "sha512-CPI0XQLwp2KIy31dK6mzJfbW+F36dx1pG0ORmoY6tDRtRe7w39PaW+fj0Q/p8RpaBxAK5PaBgl/0rN+8bmsrsA==", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/cli-package/package.json b/cli-package/package.json index b0a67830..d6f36ca9 100644 --- a/cli-package/package.json +++ b/cli-package/package.json @@ -21,7 +21,8 @@ "vector-database", "hnsw", "cli", - "command-line", + "browser", + "container", "graph-database" ], "author": "David Snelling (david@soulcraft.com)", diff --git a/cli-package/rollup.config.js b/cli-package/rollup.config.js index 8fbd61ed..b4ae2e99 100644 --- a/cli-package/rollup.config.js +++ b/cli-package/rollup.config.js @@ -37,6 +37,7 @@ export default { 'fs', 'path', 'url', - 'child_process' + 'child_process', + 'worker_threads' ] } diff --git a/rollup.config.js b/rollup.config.js index c4c12c02..a06fc65e 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -176,8 +176,11 @@ const mainConfig = { '@aws-crypto/crc32c', 'node:stream/web', 'node:worker_threads', + 'worker_threads', 'node:fs', - 'node:path' + 'node:path', + 'fs', + 'path' ] } diff --git a/src/storage/fileSystemStorage.ts b/src/storage/fileSystemStorage.ts index 55608181..19d77476 100644 --- a/src/storage/fileSystemStorage.ts +++ b/src/storage/fileSystemStorage.ts @@ -1,51 +1,12 @@ import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js' // Import Node.js built-in modules -// Using require for compatibility with Node.js +// Using dynamic imports for compatibility with ES modules let fs: any let path: any -// Initialize these modules immediately if in Node.js environment -if ( - typeof process !== 'undefined' && - process.versions && - process.versions.node -) { - try { - // Use require for Node.js built-in modules - fs = require('fs') - path = require('path') - - // Verify that path has the required methods - if (!path || typeof path.resolve !== 'function') { - throw new Error('path module is missing required methods') - } - } catch (e) { - console.warn('Failed to load Node.js modules with require:', e) - - // Try using dynamic import as a fallback - try { - // Use a synchronous approach to ensure modules are loaded before continuing - const pathModule = require('node:path') - const fsModule = require('node:fs') - - path = pathModule - fs = fsModule - - // Verify that path has the required methods - if (!path || typeof path.resolve !== 'function') { - throw new Error( - 'path module from node:path is missing required methods' - ) - } - } catch (nodeImportError) { - console.warn( - 'Failed to load Node.js modules with node: prefix:', - nodeImportError - ) - } - } -} +// We'll initialize these modules in the init() method +// No synchronous loading here to avoid issues with ES modules // Constants for directory and file names const ROOT_DIR = 'brainy-data' @@ -111,37 +72,7 @@ export class FileSystemStorage implements StorageAdapter { // Try multiple approaches to load the modules const loadAttempts = [ - // Attempt 1: Use require - async () => { - console.log('Attempting to load Node.js modules with require()') - const fsModule = require('fs') - const pathModule = require('path') - - if (!pathModule || typeof pathModule.resolve !== 'function') { - throw new Error('path.resolve is not a function after require()') - } - - return { fs: fsModule, path: pathModule } - }, - - // Attempt 2: Use require with node: prefix - async () => { - console.log( - 'Attempting to load Node.js modules with require("node:...")' - ) - const fsModule = require('node:fs') - const pathModule = require('node:path') - - if (!pathModule || typeof pathModule.resolve !== 'function') { - throw new Error( - 'path.resolve is not a function after require("node:path")' - ) - } - - return { fs: fsModule, path: pathModule } - }, - - // Attempt 3: Use dynamic import + // Attempt 1: Use dynamic import async () => { console.log( 'Attempting to load Node.js modules with dynamic import' @@ -161,7 +92,7 @@ export class FileSystemStorage implements StorageAdapter { return { fs: fsResolved, path: pathResolved } }, - // Attempt 4: Use dynamic import with node: prefix + // Attempt 2: Use dynamic import with node: prefix async () => { console.log( 'Attempting to load Node.js modules with dynamic import("node:...")' diff --git a/src/utils/environment.ts b/src/utils/environment.ts index 358e53ec..2c416e12 100644 --- a/src/utils/environment.ts +++ b/src/utils/environment.ts @@ -6,53 +6,75 @@ * Check if code is running in a browser environment */ export function isBrowser(): boolean { - return typeof window !== 'undefined' && typeof document !== 'undefined'; + return typeof window !== 'undefined' && typeof document !== 'undefined' } /** * Check if code is running in a Node.js environment */ export function isNode(): boolean { - return typeof process !== 'undefined' && - process.versions != null && - process.versions.node != null; + return ( + typeof process !== 'undefined' && + process.versions != null && + process.versions.node != null + ) } /** * Check if code is running in a Web Worker environment */ export function isWebWorker(): boolean { - return typeof self === 'object' && - self.constructor && - self.constructor.name === 'DedicatedWorkerGlobalScope'; + return ( + typeof self === 'object' && + self.constructor && + self.constructor.name === 'DedicatedWorkerGlobalScope' + ) } /** * Check if Web Workers are available in the current environment */ export function areWebWorkersAvailable(): boolean { - return isBrowser() && typeof Worker !== 'undefined'; + return isBrowser() && typeof Worker !== 'undefined' } /** * Check if Worker Threads are available in the current environment (Node.js) */ -export function areWorkerThreadsAvailable(): boolean { - if (!isNode()) return false; +export async function areWorkerThreadsAvailable(): Promise { + if (!isNode()) return false try { - // Dynamic import to avoid errors in browser environments - require('worker_threads'); - return true; + // Use dynamic import to avoid errors in browser environments + await import('worker_threads') + return true } catch (e) { - return false; + return false } } +/** + * Synchronous version that doesn't actually try to load the module + * This is safer in ES module environments + */ +export function areWorkerThreadsAvailableSync(): boolean { + if (!isNode()) return false + + // In Node.js 12+, worker_threads is available without requiring a flag + return parseInt(process.versions.node.split('.')[0]) >= 12 +} + /** * Determine if threading is available in the current environment * Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available */ export function isThreadingAvailable(): boolean { - return areWebWorkersAvailable() || areWorkerThreadsAvailable(); + return areWebWorkersAvailable() || areWorkerThreadsAvailableSync() +} + +/** + * Async version of isThreadingAvailable + */ +export async function isThreadingAvailableAsync(): Promise { + return areWebWorkersAvailable() || (await areWorkerThreadsAvailable()) } diff --git a/src/utils/tensorflowUtils.ts b/src/utils/tensorflowUtils.ts index ce34fd7f..70213e8e 100644 --- a/src/utils/tensorflowUtils.ts +++ b/src/utils/tensorflowUtils.ts @@ -33,13 +33,40 @@ if ( isFloat32Array, isTypedArray, // Use the global TextEncoder/TextDecoder directly - TextEncoder: typeof TextEncoder !== 'undefined' ? TextEncoder : null, - TextDecoder: typeof TextDecoder !== 'undefined' ? TextDecoder : null + TextEncoder: + typeof TextEncoder !== 'undefined' + ? TextEncoder + : function () { + console.warn( + 'TextEncoder constructor called but not available' + ) + return { encode: (str: string) => new Uint8Array([]) } + }, + TextDecoder: + typeof TextDecoder !== 'undefined' + ? TextDecoder + : function () { + console.warn( + 'TextDecoder constructor called but not available' + ) + return { decode: (bytes: Uint8Array) => '' } + } } // Initialize TextEncoder/TextDecoder directly from globals - this.textEncoder = new TextEncoder() - this.textDecoder = new TextDecoder() + if (typeof TextEncoder !== 'undefined') { + this.textEncoder = new TextEncoder() + } else { + console.warn('TextEncoder is not available in this environment') + this.textEncoder = { encode: (str: string) => new Uint8Array([]) } + } + + if (typeof TextDecoder !== 'undefined') { + this.textDecoder = new TextDecoder() + } else { + console.warn('TextDecoder is not available in this environment') + this.textDecoder = { decode: (bytes: Uint8Array) => '' } + } } }