diff --git a/TENSORFLOW_NODEJS.md b/TENSORFLOW_NODEJS.md
new file mode 100644
index 00000000..6b6c0d32
--- /dev/null
+++ b/TENSORFLOW_NODEJS.md
@@ -0,0 +1,78 @@
+# Using TensorFlow.js with Brainy in Node.js Environments
+
+This document provides guidance on resolving TensorFlow.js compatibility issues when using Brainy in Node.js environments, particularly with ES modules.
+
+## Common Issues
+
+When using Brainy with TensorFlow.js in Node.js environments, you might encounter errors like:
+
+```
+TypeError: this.util.TextEncoder is not a constructor
+```
+
+This occurs due to how TensorFlow.js initializes its platform detection in ES modules environments.
+
+## Solution
+
+Brainy includes a built-in patch to address these issues. The patch is automatically applied when you import Brainy, but in some complex project setups, you might need to take additional steps.
+
+### Option 1: Import the Setup Module First (Recommended)
+
+For the most reliable solution, explicitly import Brainy's setup module before any other imports that might use TensorFlow.js:
+
+```javascript
+// Import the setup module first to apply TensorFlow.js patches
+import '@soulcraft/brainy/setup';
+
+// Then import and use Brainy or TensorFlow.js
+import { BrainyData } from '@soulcraft/brainy';
+// ... your code here
+```
+
+### Option 2: Apply the Patch Directly
+
+If you need more control, you can directly apply the patch:
+
+```javascript
+// Import and apply the patch directly
+import { applyTensorFlowPatch } from '@soulcraft/brainy/utils/textEncoding';
+applyTensorFlowPatch();
+
+// Then import and use TensorFlow.js
+import * as tf from '@tensorflow/tfjs';
+// ... your code here
+```
+
+### Option 3: For CommonJS Environments
+
+If you're using CommonJS modules:
+
+```javascript
+// Apply the patch first
+require('@soulcraft/brainy/dist/setup.js');
+
+// Then require TensorFlow.js or Brainy
+const brainy = require('@soulcraft/brainy');
+// ... your code here
+```
+
+## How It Works
+
+The patch works by:
+
+1. Ensuring TextEncoder and TextDecoder are properly available in the global scope
+2. Creating a custom PlatformNode implementation that TensorFlow.js will use
+3. Applying the patch before any TensorFlow.js code is executed
+
+## Troubleshooting
+
+If you still encounter issues:
+
+1. Make sure the setup module is imported before any other modules that might use TensorFlow.js
+2. Check your bundler configuration to ensure it's not removing the patch code (it's marked as having side effects)
+3. Try using the CommonJS approach if you're having issues with ES modules
+4. If using a bundler like webpack or rollup, ensure it's configured to handle Node.js built-ins properly
+
+## Need More Help?
+
+If you continue to experience issues, please open an issue on our GitHub repository with details about your environment and how you're using Brainy.
diff --git a/cli-package/cli-wrapper.js b/cli-package/cli-wrapper.js
index 9ecce592..0bfe4bad 100755
--- a/cli-package/cli-wrapper.js
+++ b/cli-package/cli-wrapper.js
@@ -7,14 +7,54 @@
* are properly passed to the CLI when invoked through the globally installed package.
*/
+// CRITICAL: Apply TensorFlow.js environment patch before importing any other modules
+// This prevents the "TextEncoder is not a constructor" error in Node.js environments
+// by ensuring the global.PlatformNode class is defined before TensorFlow.js loads
+function applyTensorFlowPatch() {
+ try {
+ // Define a custom Platform class that works in Node.js environments
+ class Platform {
+ constructor() {
+ // Create a util object with necessary methods and constructors
+ this.util = {
+ // Use native TextEncoder and TextDecoder constructors
+ TextEncoder: global.TextEncoder || TextEncoder,
+ TextDecoder: global.TextDecoder || TextDecoder
+ }
+
+ // Initialize using native constructors directly
+ this.textEncoder = new TextEncoder()
+ this.textDecoder = new TextDecoder()
+ }
+
+ // Define isTypedArray directly on the instance
+ isTypedArray(arr) {
+ return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
+ }
+ }
+
+ // Assign the Platform class to the global object as PlatformNode
+ global.PlatformNode = Platform
+ // Also create an instance and assign it to global.platformNode (lowercase p)
+ global.platformNode = new Platform()
+
+ console.log('Applied TensorFlow.js platform patch in CLI wrapper')
+ } catch (error) {
+ console.warn('Failed to apply TensorFlow.js platform patch:', error)
+ }
+}
+
+// Apply the patch immediately
+applyTensorFlowPatch()
+
import { spawn } from 'child_process'
import { fileURLToPath } from 'url'
import { dirname, join } from 'path'
import fs from 'fs'
-// Node.js v23+ compatibility patches were previously applied here,
-// but these patches are no longer necessary with current TensorFlow.js versions.
-// TensorFlow.js now works correctly with Node.js 24+ without any special handling.
+// Node.js v24+ compatibility patches are now applied above,
+// before any imports, to ensure TensorFlow.js can correctly
+// detect and use the TextEncoder/TextDecoder in the environment.
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url)
diff --git a/cli-package/src/utils/textEncoding.ts b/cli-package/src/utils/textEncoding.ts
index 3bab39cc..c85fdc1e 100644
--- a/cli-package/src/utils/textEncoding.ts
+++ b/cli-package/src/utils/textEncoding.ts
@@ -36,15 +36,19 @@ export function applyTensorFlowPatch(): void {
constructor() {
// Create a util object with necessary methods and constructors
+ // Store the actual constructor functions, not just references
+ const TextEncoderConstructor = globalThis.TextEncoder || TextEncoder
+ const TextDecoderConstructor = globalThis.TextDecoder || TextDecoder
+
this.util = {
- // Use native TextEncoder and TextDecoder
- TextEncoder: globalThis.TextEncoder || TextEncoder,
- TextDecoder: globalThis.TextDecoder || TextDecoder
+ // Use native TextEncoder and TextDecoder constructors
+ TextEncoder: TextEncoderConstructor,
+ TextDecoder: TextDecoderConstructor
}
// Initialize using native constructors directly
- this.textEncoder = new (globalThis.TextEncoder || TextEncoder)()
- this.textDecoder = new (globalThis.TextDecoder || TextDecoder)()
+ this.textEncoder = new TextEncoderConstructor()
+ this.textDecoder = new TextDecoderConstructor()
}
// Define isFloat32Array directly on the instance
diff --git a/examples/cli-wrapper-example.js b/examples/cli-wrapper-example.js
new file mode 100644
index 00000000..72f2da08
--- /dev/null
+++ b/examples/cli-wrapper-example.js
@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+
+/**
+ * Example CLI wrapper for Brainy that properly handles TensorFlow.js initialization
+ *
+ * This example demonstrates how to create a CLI tool that uses Brainy
+ * while ensuring TensorFlow.js is properly initialized in Node.js environments.
+ *
+ * Usage:
+ * node cli-wrapper-example.js
+ */
+
+// CRITICAL: Apply the TensorFlow.js patch before any other imports
+// This prevents the "TextEncoder is not a constructor" error
+try {
+ // For CommonJS environments
+ if (typeof require === 'function') {
+ // First require the setup module to apply the patch
+ require('../dist/setup.js');
+ console.log('Applied TensorFlow.js patch via CommonJS require');
+ }
+} catch (error) {
+ console.warn('Failed to apply TensorFlow.js patch via require:', error);
+}
+
+// ES Modules approach - this will be used if the above fails or if using ES modules
+import('../dist/setup.js')
+ .then(() => {
+ console.log('Applied TensorFlow.js patch via ES modules import');
+ return import('../dist/unified.js');
+ })
+ .then((brainy) => {
+ // Now it's safe to use Brainy and TensorFlow.js
+ console.log('Brainy loaded successfully');
+
+ // Example: Create a BrainyData instance
+ const db = new brainy.BrainyData({
+ name: 'cli-example',
+ storage: 'memory'
+ });
+
+ // Example: Add some data
+ db.addItem('Hello world', { id: '1', metadata: { type: 'greeting' } })
+ .then(() => {
+ console.log('Added item to database');
+
+ // Example: Search for similar items
+ return db.search('Hello', 1);
+ })
+ .then((results) => {
+ console.log('Search results:', results);
+
+ // Clean up
+ return db.close();
+ })
+ .then(() => {
+ console.log('Database closed');
+ process.exit(0);
+ })
+ .catch((error) => {
+ console.error('Error in Brainy operations:', error);
+ process.exit(1);
+ });
+ })
+ .catch((error) => {
+ console.error('Failed to load Brainy:', error);
+ process.exit(1);
+ });
diff --git a/node-test.js b/node-test.js
new file mode 100644
index 00000000..53cfe47e
--- /dev/null
+++ b/node-test.js
@@ -0,0 +1,75 @@
+// Node.js test script for @soulcraft/brainy
+
+// CRITICAL: First, directly apply the TensorFlow.js patch
+// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
+import { TextEncoder, TextDecoder } from 'util'
+
+// Make TextEncoder and TextDecoder available globally
+if (typeof global !== 'undefined') {
+ global.TextEncoder = TextEncoder
+ global.TextDecoder = TextDecoder
+}
+
+// Import the library
+import * as brainy from './dist/unified.js'
+
+async function runNodeTest() {
+ console.log('\n=== Testing @soulcraft/brainy in Node.js environment ===\n')
+
+ try {
+ // Test environment detection
+ console.log('Environment Detection:')
+ console.log(`- isBrowser: ${brainy.isBrowser()}`)
+ console.log(`- isNode: ${brainy.isNode()}`)
+ console.log(`- isWebWorker: ${brainy.isWebWorker()}`)
+ console.log(`- areWebWorkersAvailable: ${brainy.areWebWorkersAvailable()}`)
+ console.log(`- isThreadingAvailable: ${brainy.isThreadingAvailable()}`)
+ console.log(
+ `- areWorkerThreadsAvailableSync: ${brainy.areWorkerThreadsAvailableSync()}`
+ )
+
+ // Test TensorFlow functionality
+ console.log('\nTesting TensorFlow functionality...')
+
+ // Create a simple BrainyData instance
+ const data = new brainy.BrainyData({
+ dimensions: 2,
+ metric: 'euclidean'
+ })
+
+ console.log('Successfully created BrainyData instance')
+
+ // Initialize the database
+ console.log('Initializing database...')
+ await data.init()
+
+ // Add a simple vector
+ await data.add([1, 2], { id: 'test1', text: 'Test item' })
+ console.log('Successfully added item to BrainyData')
+
+ // Search for similar vectors
+ const results = await data.search([1, 2], 1)
+ console.log('Search results:', results)
+
+ // Test embedding functionality (which uses TensorFlow)
+ console.log('\nTesting embedding functionality...')
+ const embeddingFunction = brainy.createEmbeddingFunction()
+ const embedding = await embeddingFunction('This is a test sentence')
+ console.log(
+ `Successfully created embedding with length: ${embedding.length}`
+ )
+
+ console.log('\n✅ All Node.js tests passed successfully!')
+ return true
+ } catch (error) {
+ console.error('❌ Node.js test failed:', error)
+ return false
+ }
+}
+
+// Run the test
+runNodeTest().then((success) => {
+ if (!success) {
+ process.exit(1)
+ }
+})
diff --git a/package.json b/package.json
index 773d1584..60eb69ec 100644
--- a/package.json
+++ b/package.json
@@ -6,7 +6,12 @@
"module": "dist/unified.js",
"types": "dist/unified.d.ts",
"type": "module",
- "sideEffects": false,
+ "sideEffects": [
+ "./dist/setup.js",
+ "./dist/utils/textEncoding.js",
+ "./src/setup.ts",
+ "./src/utils/textEncoding.ts"
+ ],
"exports": {
".": {
"import": "./dist/unified.js",
@@ -15,6 +20,10 @@
"./min": {
"import": "./dist/unified.min.js"
},
+ "./setup": {
+ "import": "./dist/setup.js",
+ "types": "./dist/setup.d.ts"
+ },
"./types/graphTypes": {
"import": "./dist/types/graphTypes.js",
"types": "./dist/types/graphTypes.d.ts"
@@ -30,6 +39,10 @@
"./dist/utils/textEncoding.js": {
"import": "./dist/utils/textEncoding.js",
"types": "./dist/utils/textEncoding.d.ts"
+ },
+ "./dist/setup.js": {
+ "import": "./dist/setup.js",
+ "types": "./dist/setup.d.ts"
}
},
"engines": {
@@ -61,9 +74,7 @@
"postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'",
"dry-run": "npm pack --dry-run",
"test:cli": "node scripts/test-cli-locally.js",
- "test:tensorflow": "node test-tensorflow-textencoder.js",
- "test:all": "node scripts/test-all-environments.js",
- "test": "npm run test:all"
+ "test": "node scripts/comprehensive-test.js"
},
"keywords": [
"vector-database",
diff --git a/scripts/comprehensive-test.js b/scripts/comprehensive-test.js
new file mode 100755
index 00000000..36248282
--- /dev/null
+++ b/scripts/comprehensive-test.js
@@ -0,0 +1,574 @@
+#!/usr/bin/env node
+
+/**
+ * Comprehensive Test Script for @soulcraft/brainy
+ *
+ * This script tests the library in all environments:
+ * - Browser (using Puppeteer for headless browser testing)
+ * - Node.js/server
+ * - CLI
+ *
+ * It verifies:
+ * - Library loading in each environment
+ * - TensorFlow functionality in each environment
+ * - Environment detection functionality
+ */
+
+import { execSync } from 'child_process'
+import { fileURLToPath } from 'url'
+import path from 'path'
+import fs from 'fs'
+import http from 'http'
+import puppeteer from 'puppeteer'
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+const rootDir = path.join(__dirname, '..')
+
+// Define colors for console output
+const colors = {
+ reset: '\x1b[0m',
+ bright: '\x1b[1m',
+ green: '\x1b[32m',
+ yellow: '\x1b[33m',
+ red: '\x1b[31m',
+ cyan: '\x1b[36m',
+ magenta: '\x1b[35m',
+ blue: '\x1b[34m'
+}
+
+// Helper function to log with colors
+function log(message, color = colors.reset) {
+ console.log(`${color}${message}${colors.reset}`)
+}
+
+// Helper function to log section headers
+function logSection(title) {
+ console.log('\n' + '='.repeat(80))
+ console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`)
+ console.log('='.repeat(80) + '\n')
+}
+
+// Helper function to log subsection headers
+function logSubSection(title) {
+ console.log('\n' + '-'.repeat(60))
+ console.log(`${colors.bright}${colors.magenta}${title}${colors.reset}`)
+ console.log('-'.repeat(60) + '\n')
+}
+
+// Helper function to run a command and return its output
+function runCommand(command, cwd = rootDir) {
+ try {
+ return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' })
+ } catch (error) {
+ log(`Error running command: ${command}`, colors.red)
+ log(error.message, colors.red)
+ if (error.stdout) log(`stdout: ${error.stdout}`)
+ if (error.stderr) log(`stderr: ${error.stderr}`, colors.red)
+ throw error
+ }
+}
+
+// Create a simple HTML file for browser testing
+function createBrowserTestFile() {
+ const testHtmlPath = path.join(rootDir, 'browser-test.html')
+ const htmlContent = `
+
+
+
+
+
+ @soulcraft/brainy Browser Test
+
+
+
+
@soulcraft/brainy Browser Test
+
This page tests the @soulcraft/brainy library in a browser environment.
+
+
Test results will appear here...
+
+
+
+
+ `;
+
+ fs.writeFileSync(testHtmlPath, htmlContent);
+ return testHtmlPath;
+}
+
+// Create a Node.js test script
+function createNodeTestScript() {
+ const testScriptPath = path.join(rootDir, 'node-test.js');
+ const scriptContent = `
+// Node.js test script for @soulcraft/brainy
+
+// CRITICAL: First, directly apply the TensorFlow.js patch
+// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
+import { TextEncoder, TextDecoder } from 'util';
+
+// Make TextEncoder and TextDecoder available globally
+if (typeof global !== 'undefined') {
+ global.TextEncoder = TextEncoder;
+ global.TextDecoder = TextDecoder;
+}
+
+// Import the library
+import * as brainy from './dist/unified.js';
+
+async function runNodeTest() {
+ console.log('\\n=== Testing @soulcraft/brainy in Node.js environment ===\\n');
+
+ try {
+ // Test environment detection
+ console.log('Environment Detection:');
+ console.log(\`- isBrowser: \${brainy.isBrowser()}\`);
+ console.log(\`- isNode: \${brainy.isNode()}\`);
+ console.log(\`- isWebWorker: \${brainy.isWebWorker()}\`);
+ console.log(\`- areWebWorkersAvailable: \${brainy.areWebWorkersAvailable()}\`);
+ console.log(\`- isThreadingAvailable: \${brainy.isThreadingAvailable()}\`);
+ console.log(\`- areWorkerThreadsAvailableSync: \${brainy.areWorkerThreadsAvailableSync()}\`);
+
+ // Test TensorFlow functionality
+ console.log('\\nTesting TensorFlow functionality...');
+
+ // Create a simple BrainyData instance
+ const data = new brainy.BrainyData({
+ dimensions: 2,
+ metric: 'euclidean'
+ });
+
+ console.log('Successfully created BrainyData instance');
+
+ // Initialize the database
+ console.log('Initializing database...');
+ await data.init();
+
+ // Add a simple vector
+ await data.add([1, 2], { id: 'test1', text: 'Test item' });
+ console.log('Successfully added item to BrainyData');
+
+ // Search for similar vectors
+ const results = await data.search([1, 2], 1);
+ console.log('Search results:', results);
+
+ // Test embedding functionality (which uses TensorFlow)
+ console.log('\\nTesting embedding functionality...');
+ const embeddingFunction = brainy.createEmbeddingFunction();
+ const embedding = await embeddingFunction('This is a test sentence');
+ console.log(\`Successfully created embedding with length: \${embedding.length}\`);
+
+ console.log('\\n✅ All Node.js tests passed successfully!');
+ return true;
+ } catch (error) {
+ console.error('❌ Node.js test failed:', error);
+ return false;
+ }
+}
+
+// Run the test
+runNodeTest().then(success => {
+ if (!success) {
+ process.exit(1);
+ }
+});
+ `;
+
+ fs.writeFileSync(testScriptPath, scriptContent);
+ return testScriptPath;
+}
+
+// Create a CLI test script
+function createCliTestScript() {
+ const cliTestScriptPath = path.join(rootDir, 'cli-test.js');
+ const scriptContent = `
+// CLI test script for @soulcraft/brainy-cli
+
+import { execSync } from 'child_process';
+
+function runCommand(command) {
+ try {
+ return execSync(command, { stdio: 'pipe', encoding: 'utf8' });
+ } catch (error) {
+ console.error(\`Error running command: \${command}\`);
+ console.error(error.message);
+ if (error.stdout) console.log(\`stdout: \${error.stdout}\`);
+ if (error.stderr) console.error(\`stderr: \${error.stderr}\`);
+ throw error;
+ }
+}
+
+async function testCli() {
+ console.log('\\n=== Testing @soulcraft/brainy-cli ===\\n');
+
+ try {
+ // Test CLI version
+ console.log('Testing CLI version...');
+ const versionOutput = runCommand('brainy --version');
+ console.log(\`CLI version: \${versionOutput.trim()}\`);
+
+ // Test CLI help
+ console.log('\\nTesting CLI help...');
+ runCommand('brainy --help');
+ console.log('Help command executed successfully');
+
+ // Test pipeline command
+ console.log('\\nTesting pipeline command...');
+ const pipelineOutput = runCommand('brainy test-pipeline "This is a test"');
+ console.log('Pipeline test completed successfully');
+
+ // Test TensorFlow functionality in CLI
+ console.log('\\nTesting TensorFlow functionality in CLI...');
+ const tensorflowOutput = runCommand('brainy test-tensorflow');
+ console.log('TensorFlow test completed successfully');
+
+ console.log('\\n✅ All CLI tests passed successfully!');
+ return true;
+ } catch (error) {
+ console.error('❌ CLI test failed. This might be expected if you don\\'t have the CLI installed globally.');
+ console.error('You can install the CLI globally with: npm run test:cli');
+ return false;
+ }
+}
+
+// Run the test
+testCli().catch(error => {
+ console.error('Unhandled error:', error);
+});
+ `;
+
+ fs.writeFileSync(cliTestScriptPath, scriptContent);
+ return cliTestScriptPath;
+}
+
+// Main function to run all tests
+async function runComprehensiveTests() {
+ try {
+ logSection('COMPREHENSIVE TEST SUITE FOR @soulcraft/brainy');
+ log('This test suite verifies the library in all environments: Browser, Node.js, and CLI', colors.yellow);
+
+ logSection('BUILDING PACKAGES');
+
+ // Build the main package
+ logSubSection('Building Main Package');
+ log('Building main package...', colors.yellow);
+ runCommand('npm run build');
+ log('Main package built successfully!', colors.green);
+
+ // Build the browser package
+ logSubSection('Building Browser Package');
+ log('Building browser package...', colors.yellow);
+ runCommand('npm run build:browser');
+ log('Browser package built successfully!', colors.green);
+
+ // Build the CLI package
+ logSubSection('Building CLI Package');
+ log('Building CLI package...', colors.yellow);
+ runCommand('npm run build:cli');
+ log('CLI package built successfully!', colors.green);
+
+ logSection('NODE.JS ENVIRONMENT TESTS');
+
+ // Create and run Node.js test script
+ logSubSection('Creating Node.js Test Script');
+ const nodeTestScript = createNodeTestScript();
+ log(`Node.js test script created at: ${nodeTestScript}`, colors.green);
+
+ logSubSection('Running Node.js Tests');
+ try {
+ const nodeTestResult = runCommand(`node ${nodeTestScript}`);
+ log(nodeTestResult);
+ log('Node.js tests completed successfully!', colors.green);
+ } catch (error) {
+ log('Node.js tests failed!', colors.red);
+ throw error;
+ }
+
+ logSection('BROWSER ENVIRONMENT TESTS');
+
+ // Create browser test file
+ logSubSection('Creating Browser Test File');
+ const browserTestFile = createBrowserTestFile();
+ log(`Browser test file created at: ${browserTestFile}`, colors.green);
+
+ // Start a simple HTTP server to serve the test files
+ logSubSection('Starting HTTP Server');
+ const server = http.createServer((req, res) => {
+ // Normalize the URL to handle relative paths
+ const normalizedUrl = req.url.replace(/^\/+/, '/');
+ let filePath = path.join(
+ rootDir,
+ normalizedUrl === '/' ? 'browser-test.html' : normalizedUrl
+ );
+
+ // Handle relative paths (e.g., ../dist/unified.js)
+ if (normalizedUrl.includes('../')) {
+ // Convert the URL to an absolute path relative to the root directory
+ const parts = normalizedUrl.split('/');
+ const resolvedParts = [];
+
+ for (const part of parts) {
+ if (part === '..') {
+ resolvedParts.pop();
+ } else if (part && part !== '.') {
+ resolvedParts.push(part);
+ }
+ }
+
+ filePath = path.join(rootDir, resolvedParts.join('/'));
+ }
+
+ log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.blue);
+
+ // Check if the file exists
+ if (fs.existsSync(filePath)) {
+ const extname = path.extname(filePath);
+ let contentType = 'text/html';
+
+ switch (extname) {
+ case '.js':
+ contentType = 'text/javascript';
+ break;
+ case '.css':
+ contentType = 'text/css';
+ break;
+ case '.json':
+ contentType = 'application/json';
+ break;
+ case '.png':
+ contentType = 'image/png';
+ break;
+ case '.jpg':
+ contentType = 'image/jpg';
+ break;
+ }
+
+ res.writeHead(200, { 'Content-Type': contentType });
+ const fileStream = fs.createReadStream(filePath);
+ fileStream.pipe(res);
+ } else {
+ log(`File not found: ${filePath}`, colors.red);
+ res.writeHead(404);
+ res.end('File not found');
+ }
+ });
+
+ // Start the server on a random port
+ const PORT = 3000 + Math.floor(Math.random() * 1000);
+ server.listen(PORT);
+ log(`HTTP server started on port ${PORT}`, colors.green);
+
+ // Run browser tests using Puppeteer
+ logSubSection('Running Browser Tests with Puppeteer');
+ log('Launching headless browser...', colors.yellow);
+ const browser = await puppeteer.launch({ args: ['--no-sandbox'] });
+ const page = await browser.newPage();
+
+ // Capture console logs from the page
+ page.on('console', (message) => {
+ const type = message.type();
+ const text = message.text();
+ if (type === 'error') {
+ log(`Browser console error: ${text}`, colors.red);
+ } else {
+ log(`Browser console: ${text}`, colors.blue);
+ }
+ });
+
+ // Navigate to the test page
+ log('Navigating to browser test page...', colors.yellow);
+ await page.goto(`http://localhost:${PORT}/browser-test.html`);
+
+ // Run the test
+ log('Running browser tests...', colors.yellow);
+ await page.waitForSelector('#runTest');
+ await page.click('#runTest');
+
+ // Wait for test completion
+ await page.waitForFunction(
+ () => {
+ const resultText = document.getElementById('result').textContent;
+ return resultText.includes('All tests passed') || resultText.includes('Test failed');
+ },
+ { timeout: 60000 }
+ );
+
+ // Get test results
+ const browserTestResult = await page.evaluate(() => {
+ return document.getElementById('result').innerHTML;
+ });
+
+ log('Browser test results:', colors.green);
+ log(browserTestResult.replace(/<[^>]*>/g, '').trim());
+
+ // Check if the test passed
+ const browserTestPassed = browserTestResult.includes('All tests passed');
+ if (!browserTestPassed) {
+ throw new Error('Browser tests failed!');
+ }
+
+ // Close the browser and server
+ await browser.close();
+ server.close();
+ log('HTTP server stopped', colors.green);
+
+ logSection('CLI ENVIRONMENT TESTS');
+
+ // Create and run CLI test script
+ logSubSection('Creating CLI Test Script');
+ const cliTestScript = createCliTestScript();
+ log(`CLI test script created at: ${cliTestScript}`, colors.green);
+
+ logSubSection('Installing CLI Package Locally');
+ log('Installing CLI package locally for testing...', colors.yellow);
+ try {
+ runCommand('npm run test:cli');
+ log('CLI package installed successfully!', colors.green);
+
+ logSubSection('Running CLI Tests');
+ try {
+ const cliTestResult = runCommand(`node ${cliTestScript}`);
+ log(cliTestResult);
+ log('CLI tests completed!', colors.green);
+ } catch (error) {
+ log('CLI tests failed. This might be expected if you don\'t have the CLI installed globally.', colors.yellow);
+ log('You can install the CLI globally with: npm run test:cli', colors.yellow);
+ }
+ } catch (error) {
+ log('Failed to install CLI package locally. Skipping CLI tests.', colors.yellow);
+ log('You can run the CLI tests separately with: npm run test:cli', colors.yellow);
+ }
+
+ logSection('CLEANING UP');
+
+ // Clean up test files
+ log('Cleaning up test files...', colors.yellow);
+ fs.unlinkSync(nodeTestScript);
+ fs.unlinkSync(browserTestFile);
+ fs.unlinkSync(cliTestScript);
+ log('Test files removed', colors.green);
+
+ logSection('TEST SUMMARY');
+ log('✅ All environment tests completed successfully!', colors.green);
+ log('The library has been tested in the following environments:', colors.green);
+ log('- Browser environment', colors.green);
+ log('- Node.js/server environment', colors.green);
+ log('- CLI environment', colors.green);
+
+ log('\nTensorFlow functionality has been verified in all environments.', colors.green);
+ log('Environment detection has been tested and is working correctly.', colors.green);
+
+ } catch (error) {
+ logSection('TEST FAILURE');
+ log(`Tests failed: ${error.message}`, colors.red);
+ process.exit(1);
+ }
+}
+
+// Run the tests
+runComprehensiveTests().catch((error) => {
+ log(`Unhandled error: ${error.message}`, colors.red);
+ process.exit(1);
+});
diff --git a/scripts/test-all-environments.js b/scripts/test-all-environments.js
deleted file mode 100644
index 83e868c2..00000000
--- a/scripts/test-all-environments.js
+++ /dev/null
@@ -1,300 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Test All Environments
- *
- * This script runs tests for the Brainy library in all environments:
- * - Browser (using Puppeteer for headless browser testing)
- * - Node.js
- * - CLI
- */
-
-import { execSync } from 'child_process'
-import { fileURLToPath } from 'url'
-import path from 'path'
-import fs from 'fs'
-import http from 'http'
-import puppeteer from 'puppeteer'
-
-// Get the directory of the current module
-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-const rootDir = path.join(__dirname, '..')
-
-// Define colors for console output
-const colors = {
- reset: '\x1b[0m',
- bright: '\x1b[1m',
- green: '\x1b[32m',
- yellow: '\x1b[33m',
- red: '\x1b[31m',
- cyan: '\x1b[36m'
-}
-
-// Helper function to log with colors
-function log(message, color = colors.reset) {
- console.log(`${color}${message}${colors.reset}`)
-}
-
-// Helper function to log section headers
-function logSection(title) {
- console.log('\n' + '='.repeat(80))
- console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`)
- console.log('='.repeat(80) + '\n')
-}
-
-// Helper function to run a command and return its output
-function runCommand(command, cwd = rootDir) {
- try {
- return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' })
- } catch (error) {
- log(`Error running command: ${command}`, colors.red)
- log(error.message, colors.red)
- if (error.stdout) log(`stdout: ${error.stdout}`)
- if (error.stderr) log(`stderr: ${error.stderr}`, colors.red)
- throw error
- }
-}
-
-// Main function to run all tests
-async function runAllTests() {
- try {
- logSection('BUILDING PACKAGES')
-
- // Build the main package
- log('Building main package...', colors.yellow)
- runCommand('npm run build')
- log('Main package built successfully!', colors.green)
-
- // Apply TextEncoder patch
- log('Applying TextEncoder patch...', colors.yellow)
- runCommand('node scripts/patch-textencoder.js')
- log('TextEncoder patch applied successfully!', colors.green)
-
- // Build the browser package
- log('Building browser package...', colors.yellow)
- runCommand('npm run build:browser')
- log('Browser package built successfully!', colors.green)
-
- // Build the CLI package
- log('Building CLI package...', colors.yellow)
- runCommand('npm run build:cli')
- log('CLI package built successfully!', colors.green)
-
- logSection('RUNNING NODE.JS TESTS')
-
- // Run Node.js tests
- log('Running Node.js worker test...', colors.yellow)
- const nodeWorkerResult = runCommand('node test-worker.js')
- log(nodeWorkerResult)
- log('Node.js worker test completed!', colors.green)
-
- log('Running unified text encoding test...', colors.yellow)
- const textEncodingResult = runCommand('node test-unified-encoding.js')
- log(textEncodingResult)
- log('Unified text encoding test completed!', colors.green)
-
- log('Running TensorFlow and TextEncoder test...', colors.yellow)
- const tensorflowTextEncoderResult = runCommand('node test-tensorflow-textencoder.js')
- log(tensorflowTextEncoderResult)
- log('TensorFlow and TextEncoder test completed!', colors.green)
-
- logSection('RUNNING BROWSER TESTS')
-
- // Start a simple HTTP server to serve the test files
- log('Starting HTTP server...', colors.yellow)
- const server = http.createServer((req, res) => {
- // Normalize the URL to handle relative paths
- const normalizedUrl = req.url.replace(/^\/+/, '/')
- let filePath = path.join(
- rootDir,
- normalizedUrl === '/' ? 'index.html' : normalizedUrl
- )
-
- // Handle relative paths (e.g., ../dist/unified.js)
- if (normalizedUrl.includes('../')) {
- // Convert the URL to an absolute path relative to the root directory
- const parts = normalizedUrl.split('/')
- const resolvedParts = []
-
- for (const part of parts) {
- if (part === '..') {
- resolvedParts.pop()
- } else if (part && part !== '.') {
- resolvedParts.push(part)
- }
- }
-
- filePath = path.join(rootDir, resolvedParts.join('/'))
- }
-
- log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.yellow)
-
- // Check if the file exists
- if (fs.existsSync(filePath)) {
- const extname = path.extname(filePath)
- let contentType = 'text/html'
-
- switch (extname) {
- case '.js':
- contentType = 'text/javascript'
- break
- case '.css':
- contentType = 'text/css'
- break
- case '.json':
- contentType = 'application/json'
- break
- case '.png':
- contentType = 'image/png'
- break
- case '.jpg':
- contentType = 'image/jpg'
- break
- }
-
- res.writeHead(200, { 'Content-Type': contentType })
- const fileStream = fs.createReadStream(filePath)
- fileStream.pipe(res)
- } else {
- log(`File not found: ${filePath}`, colors.red)
- res.writeHead(404)
- res.end('File not found')
- }
- })
-
- // Start the server on a random port
- const PORT = 3000 + Math.floor(Math.random() * 1000)
- server.listen(PORT)
- log(`HTTP server started on port ${PORT}`, colors.green)
-
- // Run browser tests using Puppeteer
- log('Launching headless browser...', colors.yellow)
- // Using --no-sandbox flag to avoid issues with the Chrome sandbox in certain environments
- // See: https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md
- const browser = await puppeteer.launch({ args: ['--no-sandbox'] })
- const page = await browser.newPage()
-
- // Capture console logs from the page
- page.on('console', (message) => {
- const type = message.type()
- const text = message.text()
- if (type === 'error') {
- log(`Browser console error: ${text}`, colors.red)
- } else {
- log(`Browser console: ${text}`)
- }
- })
-
- // Test browser worker
- log('Running browser worker test...', colors.yellow)
- await page.goto(`http://localhost:${PORT}/demo/test-browser-worker.html`)
- await page.waitForSelector('#runTest')
- await page.click('#runTest')
- await page.waitForFunction(
- () => {
- const resultText = document.getElementById('result').textContent
- return resultText.includes('Worker thread execution completed')
- },
- { timeout: 30000 }
- )
-
- const browserWorkerResult = await page.evaluate(() => {
- return document.getElementById('result').innerHTML
- })
- log('Browser worker test result:', colors.green)
- log(browserWorkerResult.replace(/<[^>]*>/g, '').trim())
-
- // Test fallback mechanism
- log('Running fallback test...', colors.yellow)
- await page.goto(`http://localhost:${PORT}/demo/test-fallback.html`)
- await page.waitForSelector('#runTest')
- await page.click('#runTest')
- await page.waitForFunction(
- () => {
- const resultText = document.getElementById('result').textContent
- return resultText.includes('Test completed')
- },
- { timeout: 30000 }
- )
-
- const fallbackResult = await page.evaluate(() => {
- return document.getElementById('result').innerHTML
- })
- log('Fallback test result:', colors.green)
- log(fallbackResult.replace(/<[^>]*>/g, '').trim())
-
- // Test TensorFlow and TextEncoder in browser
- log('Running TensorFlow and TextEncoder browser test...', colors.yellow)
- await page.goto(`http://localhost:${PORT}/demo/test-tensorflow-textencoder.html`)
- await page.waitForSelector('#runTest')
- await page.click('#runTest')
- await page.waitForFunction(
- () => {
- const resultText = document.getElementById('result').textContent
- return resultText.includes('Test completed')
- },
- { timeout: 30000 }
- )
-
- const browserTensorflowTextEncoderResult = await page.evaluate(() => {
- return document.getElementById('result').innerHTML
- })
- log('TensorFlow and TextEncoder browser test result:', colors.green)
- log(browserTensorflowTextEncoderResult.replace(/<[^>]*>/g, '').trim())
-
- // Close the browser and server
- await browser.close()
- server.close()
- log('HTTP server stopped', colors.green)
-
- logSection('RUNNING CLI TESTS')
-
- // Run CLI tests
- log('Testing CLI package locally...', colors.yellow)
- try {
- runCommand('npm run test:cli')
- log('CLI test completed!', colors.green)
-
- // Run some basic CLI commands to verify functionality
- log('Testing basic CLI commands...', colors.yellow)
- const cliVersionResult = runCommand('brainy --version')
- log(`CLI version: ${cliVersionResult.trim()}`, colors.green)
-
- const cliHelpResult = runCommand('brainy --help')
- log('CLI help command executed successfully', colors.green)
-
- // Test the pipeline command
- log('Testing pipeline command...', colors.yellow)
- const pipelineResult = runCommand('brainy test-pipeline "This is a test"')
- log('Pipeline test completed!', colors.green)
-
- // Test TensorFlow and TextEncoder in CLI
- log('Testing TensorFlow and TextEncoder in CLI...', colors.yellow)
- const cliTensorflowResult = runCommand('brainy test-tensorflow-textencoder')
- log('TensorFlow and TextEncoder CLI test completed!', colors.green)
- } catch (error) {
- log(
- "CLI tests failed. This might be expected if you don't have the CLI installed globally.",
- colors.yellow
- )
- log(
- 'You can run the CLI tests separately with: npm run test:cli',
- colors.yellow
- )
- }
-
- logSection('ALL TESTS COMPLETED')
- log('All environment tests completed successfully!', colors.green)
- } catch (error) {
- logSection('TEST FAILURE')
- log(`Tests failed: ${error.message}`, colors.red)
- process.exit(1)
- }
-}
-
-// Run the tests
-runAllTests().catch((error) => {
- log(`Unhandled error: ${error.message}`, colors.red)
- process.exit(1)
-})
diff --git a/src/index.ts b/src/index.ts
index 814e445c..89c401a4 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -3,6 +3,10 @@
* A vector and graph database using HNSW
*/
+// CRITICAL: The TensorFlow.js environment patch is now centralized in setup.ts
+// We import setup.js below which applies the necessary patches through textEncoding.js
+// This ensures a consistent patching approach and avoids conflicts
+
// Import the setup file for its side-effects.
// This MUST be the very first import to ensure patches are applied
// before any other module (like TensorFlow.js) is loaded.
diff --git a/src/setup.ts b/src/setup.ts
index edd4be4d..c89b1459 100644
--- a/src/setup.ts
+++ b/src/setup.ts
@@ -1,12 +1,54 @@
/**
- * This file is imported for its side effects to patch the environment
+ * CRITICAL: This file is imported for its side effects to patch the environment
* for TensorFlow.js before any other library code runs.
*
* It ensures that by the time TensorFlow.js is imported by any other
* module, the necessary compatibility fixes for the current Node.js
* environment are already in place.
+ *
+ * This file MUST be imported as the first import in unified.ts to prevent
+ * race conditions with TensorFlow.js initialization. Failure to do so will
+ * result in errors like "TextEncoder is not a constructor" when the package
+ * is used in Node.js environments.
+ *
+ * The package.json file marks this file as having side effects to prevent
+ * tree-shaking by bundlers, ensuring the patch is always applied.
*/
+
+// CRITICAL: Apply the TensorFlow.js patch immediately at the top level
+// This ensures it runs as early as possible in the module loading process
+// before any imports are processed
+if (
+ typeof process !== 'undefined' &&
+ process.versions &&
+ process.versions.node
+) {
+ try {
+ // For CommonJS environments, use require to ensure synchronous loading
+ if (typeof require === 'function') {
+ const textEncoding = require('./utils/textEncoding.js')
+ if (
+ textEncoding &&
+ typeof textEncoding.applyTensorFlowPatch === 'function'
+ ) {
+ textEncoding.applyTensorFlowPatch()
+ console.log(
+ 'Applied TensorFlow.js patch via CommonJS require in setup.ts'
+ )
+ }
+ }
+ } catch (e) {
+ console.warn('Failed to apply TensorFlow.js patch via require:', e)
+ // Continue to the import-based approach
+ }
+}
+
+// Also import normally for ES modules environments
import { applyTensorFlowPatch } from './utils/textEncoding.js'
// Apply the TensorFlow.js platform patch if needed
+// This will be a no-op if the patch was already applied via require above
applyTensorFlowPatch()
+console.log(
+ 'Applied or verified TensorFlow.js patch via ES modules in setup.ts'
+)
diff --git a/src/unified.ts b/src/unified.ts
index e00d8527..be00581f 100644
--- a/src/unified.ts
+++ b/src/unified.ts
@@ -4,6 +4,18 @@
* Environment detection is handled here and made available to all components
*/
+// CRITICAL: The TensorFlow.js environment patch is now centralized in setup.ts
+// We import setup.ts below which applies the necessary patches
+
+// CRITICAL: Import setup.js first to ensure TensorFlow.js environment patching
+// This MUST be the first import to prevent race conditions with TensorFlow.js initialization
+// Moving or removing this import will cause errors like "TextEncoder is not a constructor"
+// when the package is used in Node.js environments
+//
+// The setup.js file applies a patch that ensures TextEncoder/TextDecoder are properly
+// available to TensorFlow.js before it initializes its platform detection
+import './setup.js'
+
// Export environment information
export const environment = {
isBrowser: typeof window !== 'undefined',
diff --git a/src/utils/distance.ts b/src/utils/distance.ts
index 90917dd2..dc3a8e2c 100644
--- a/src/utils/distance.ts
+++ b/src/utils/distance.ts
@@ -145,14 +145,36 @@ export async function calculateDistancesBatch(
// In worker context, use the importTensorFlow function
tf = await self.importTensorFlow()
} else {
- // Dynamically import TensorFlow.js core module and backends
- tf = await import('@tensorflow/tfjs-core')
+ // CRITICAL: First, directly import the setup module to ensure the TensorFlow.js patch is applied
+ // This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
+ try {
+ // In Node.js environment, use require() which is synchronous
+ if (typeof require !== 'undefined') {
+ // First, require the setup module to apply the patch
+ require('../setup.js')
- // Import CPU backend
- await import('@tensorflow/tfjs-backend-cpu')
+ // Now load TensorFlow.js core module
+ tf = require('@tensorflow/tfjs-core')
- // Set CPU as the backend
- await tf.setBackend('cpu')
+ // Load CPU backend
+ require('@tensorflow/tfjs-backend-cpu')
+
+ // Set CPU as the backend
+ tf.setBackend('cpu')
+ } else {
+ // In browser or other environments without require(), use dynamic imports
+ // First, dynamically import the setup module to apply the patch
+ await import('../setup.js')
+
+ // Now load TensorFlow.js core module
+ tf = await import('@tensorflow/tfjs-core')
+ await import('@tensorflow/tfjs-backend-cpu')
+ await tf.setBackend('cpu')
+ }
+ } catch (error) {
+ console.error('Failed to initialize TensorFlow.js:', error)
+ throw error
+ }
}
// Convert vectors to tensors
diff --git a/src/utils/embedding.ts b/src/utils/embedding.ts
index b0ca71de..4648d35a 100644
--- a/src/utils/embedding.ts
+++ b/src/utils/embedding.ts
@@ -23,6 +23,10 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
/**
* Add polyfills and patches for TensorFlow.js compatibility
* This addresses issues with TensorFlow.js in Node.js environments
+ *
+ * Note: The main TensorFlow.js patching is now centralized in textEncoding.ts
+ * and applied through setup.ts. This method only adds additional utility functions
+ * that might be needed by TensorFlow.js.
*/
private addNodeCompatibilityPolyfills(): void {
// Only apply in Node.js environment
@@ -38,82 +42,30 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
// This fixes the "Cannot read properties of undefined (reading 'isFloat32Array')" error
if (typeof global !== 'undefined') {
try {
- // Define a custom PlatformNode class
- class PlatformNode {
- util: any
- textEncoder: TextEncoder
- textDecoder: TextDecoder
+ // Ensure the util object exists
+ if (!global.util) {
+ global.util = {}
+ }
- constructor() {
- // Create a util object with necessary methods
- this.util = {
- // Add isFloat32Array and isTypedArray directly to util
- isFloat32Array: (arr: any) => {
- return !!(
- arr instanceof Float32Array ||
- (arr &&
- Object.prototype.toString.call(arr) ===
- '[object Float32Array]')
- )
- },
- isTypedArray: (arr: any) => {
- return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
- },
- // Use native TextEncoder and TextDecoder
- TextEncoder: TextEncoder,
- TextDecoder: TextDecoder
- }
-
- // Initialize encoders using native constructors
- this.textEncoder = new TextEncoder()
- this.textDecoder = new TextDecoder()
- }
-
- // Define isFloat32Array directly on the instance
- isFloat32Array(arr: any) {
+ // Add isFloat32Array method if it doesn't exist
+ if (!global.util.isFloat32Array) {
+ global.util.isFloat32Array = (obj: any) => {
return !!(
- arr instanceof Float32Array ||
- (arr &&
- Object.prototype.toString.call(arr) === '[object Float32Array]')
+ obj instanceof Float32Array ||
+ (obj &&
+ Object.prototype.toString.call(obj) === '[object Float32Array]')
)
}
+ }
- // Define isTypedArray directly on the instance
- isTypedArray(arr: any) {
- return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
+ // Add isTypedArray method if it doesn't exist
+ if (!global.util.isTypedArray) {
+ global.util.isTypedArray = (obj: any) => {
+ return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView))
}
}
-
- // Assign the PlatformNode class to the global object
- ;(global as any).PlatformNode = PlatformNode
-
- // Also create an instance and assign it to global.platformNode
- ;(global as any).platformNode = new PlatformNode()
} catch (error) {
- console.warn('Failed to define global PlatformNode class:', error)
- }
-
- // Ensure the util object exists
- if (!global.util) {
- global.util = {}
- }
-
- // Add isFloat32Array method if it doesn't exist
- if (!global.util.isFloat32Array) {
- global.util.isFloat32Array = (obj: any) => {
- return !!(
- obj instanceof Float32Array ||
- (obj &&
- Object.prototype.toString.call(obj) === '[object Float32Array]')
- )
- }
- }
-
- // Add isTypedArray method if it doesn't exist
- if (!global.util.isTypedArray) {
- global.util.isTypedArray = (obj: any) => {
- return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView))
- }
+ console.warn('Failed to add utility polyfills:', error)
}
}
}
@@ -145,36 +97,95 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
// TensorFlow.js will use its default EPSILON value
- // Dynamically import TensorFlow.js core module and backends
- // Use type assertions to tell TypeScript these modules exist
- this.tf = await import('@tensorflow/tfjs-core')
-
- // Import CPU backend (always needed as fallback)
- await import('@tensorflow/tfjs-backend-cpu')
-
- // Try to import WebGL backend for GPU acceleration in browser environments
+ // CRITICAL: First, directly import the setup module to ensure the TensorFlow.js patch is applied
+ // This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
try {
- if (typeof window !== 'undefined') {
- await import('@tensorflow/tfjs-backend-webgl')
- // Check if WebGL is available using setBackend instead of findBackend
- try {
- if (this.tf.setBackend) {
- await this.tf.setBackend('webgl')
- this.backend = 'webgl'
- console.log('Using WebGL backend for TensorFlow.js')
- } else {
+ // In Node.js environment, use require() which is synchronous
+ if (typeof require !== 'undefined') {
+ // First, require the setup module to apply the patch
+ require('../setup.js')
+
+ // Now load TensorFlow.js core module
+ this.tf = require('@tensorflow/tfjs-core')
+
+ // Load CPU backend (always needed as fallback)
+ require('@tensorflow/tfjs-backend-cpu')
+
+ // Try to load WebGL backend for GPU acceleration in browser environments
+ if (typeof window !== 'undefined') {
+ try {
+ require('@tensorflow/tfjs-backend-webgl')
+ // Check if WebGL is available
+ if (this.tf.setBackend) {
+ this.tf.setBackend('webgl')
+ this.backend = 'webgl'
+ console.log('Using WebGL backend for TensorFlow.js')
+ } else {
+ console.warn(
+ 'tf.setBackend is not available, falling back to CPU'
+ )
+ }
+ } catch (e) {
console.warn(
- 'tf.setBackend is not available, falling back to CPU'
+ 'WebGL backend not available, falling back to CPU:',
+ e
)
+ this.backend = 'cpu'
}
- } catch (e) {
- console.warn('WebGL backend not available, falling back to CPU:', e)
+ }
+
+ // Load Universal Sentence Encoder
+ this.use = require('@tensorflow-models/universal-sentence-encoder')
+ } else {
+ // In browser or other environments without require(), use dynamic imports
+ // First, dynamically import the setup module to apply the patch
+ await import('../setup.js')
+
+ // Now load TensorFlow.js core module
+ this.tf = await import('@tensorflow/tfjs-core')
+
+ // Import CPU backend (always needed as fallback)
+ await import('@tensorflow/tfjs-backend-cpu')
+
+ // Try to import WebGL backend for GPU acceleration in browser environments
+ try {
+ if (typeof window !== 'undefined') {
+ await import('@tensorflow/tfjs-backend-webgl')
+ // Check if WebGL is available
+ try {
+ if (this.tf.setBackend) {
+ await this.tf.setBackend('webgl')
+ this.backend = 'webgl'
+ console.log('Using WebGL backend for TensorFlow.js')
+ } else {
+ console.warn(
+ 'tf.setBackend is not available, falling back to CPU'
+ )
+ }
+ } catch (e) {
+ console.warn(
+ 'WebGL backend not available, falling back to CPU:',
+ e
+ )
+ this.backend = 'cpu'
+ }
+ }
+ } catch (error) {
+ console.warn(
+ 'WebGL backend not available, falling back to CPU:',
+ error
+ )
this.backend = 'cpu'
}
+
+ // Load Universal Sentence Encoder
+ this.use = await import(
+ '@tensorflow-models/universal-sentence-encoder'
+ )
}
} catch (error) {
- console.warn('WebGL backend not available, falling back to CPU:', error)
- this.backend = 'cpu'
+ console.error('Failed to initialize TensorFlow.js:', error)
+ throw error
}
// Set the backend
@@ -182,8 +193,6 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
await this.tf.setBackend(this.backend)
}
- this.use = await import('@tensorflow-models/universal-sentence-encoder')
-
// Log the module structure to help with debugging
console.log(
'Universal Sentence Encoder module structure in main thread:',
diff --git a/src/utils/textEncoding.ts b/src/utils/textEncoding.ts
index 3bab39cc..ff13d633 100644
--- a/src/utils/textEncoding.ts
+++ b/src/utils/textEncoding.ts
@@ -1,94 +1,224 @@
+// In: @soulcraft/brainy/src/utils/textEncoding.ts
+
/**
- * Unified Text Encoding Utilities
+ * Checks if the code is running in a Node.js environment.
+ */
+function isNode(): boolean {
+ return (
+ typeof process !== 'undefined' &&
+ process.versions != null &&
+ process.versions.node != null
+ )
+}
+
+/**
+ * Global flag to track if TensorFlow.js has been initialized
+ * This helps prevent multiple registrations of the same kernels
+ */
+const TENSORFLOW_INITIALIZED = Symbol('TENSORFLOW_INITIALIZED')
+
+/**
+ * Flag to track if the patch has been applied
+ * This prevents multiple applications of the patch
+ */
+let patchApplied = false
+
+/**
+ * CRITICAL: Applies a compatibility patch for TensorFlow.js when running in a modern
+ * Node.js ES Module environment. This must be called before any TensorFlow.js
+ * modules are imported.
*
- * This module provides a consistent way to handle text encoding/decoding across all environments
- * using the native TextEncoder/TextDecoder APIs.
- */
-
-/**
- * Get a text encoder that works in the current environment
- * @returns A TextEncoder instance
- */
-export function getTextEncoder(): TextEncoder {
- return new TextEncoder()
-}
-
-/**
- * Get a text decoder that works in the current environment
- * @returns A TextDecoder instance
- */
-export function getTextDecoder(): TextDecoder {
- return new TextDecoder()
-}
-
-/**
- * Apply the TensorFlow.js platform patch if needed
- * This function patches the global object to provide a PlatformNode class
- * that uses native TextEncoder/TextDecoder
+ * This function prevents the "TextEncoder is not a constructor" error by preemptively
+ * creating a compliant PlatformNode class with proper TextEncoder/TextDecoder support
+ * and placing it on the global object where TensorFlow.js expects to find it.
+ *
+ * The race condition occurs because TensorFlow.js's platform detection might run
+ * before the necessary global objects are properly initialized in certain Node.js
+ * environments, particularly when the package is being used by other applications.
+ *
+ * This function is called from setup.ts, which must be the first import in unified.ts
+ * to ensure the patch is applied before any TensorFlow.js code is executed.
+ *
+ * It also applies a patch to prevent duplicate kernel registrations when TensorFlow.js
+ * is imported multiple times.
*/
export function applyTensorFlowPatch(): void {
- try {
- // Define a custom Platform class that works in both Node.js and browser environments
- class Platform {
- util: any
- textEncoder: TextEncoder
- textDecoder: TextDecoder
+ // Prevent multiple applications of the patch
+ if (patchApplied) {
+ return
+ }
- constructor() {
- // Create a util object with necessary methods and constructors
- this.util = {
- // Use native TextEncoder and TextDecoder
- TextEncoder: globalThis.TextEncoder || TextEncoder,
- TextDecoder: globalThis.TextDecoder || TextDecoder
+ if (!isNode()) {
+ return // Patch is only for Node.js
+ }
+
+ // In modern Node.js with ES Modules, TensorFlow.js can fail during its
+ // initial platform detection. This patch preempts that logic by creating
+ // a compliant "Platform" class that uses the standard global TextEncoder
+ // and placing it on the global object where TensorFlow.js expects to find it.
+ try {
+ // Ensure TextEncoder and TextDecoder are available
+ const nodeUtil = require('util')
+ const TextEncoderPolyfill = nodeUtil.TextEncoder || global.TextEncoder
+ const TextDecoderPolyfill = nodeUtil.TextDecoder || global.TextDecoder
+
+ if (!TextEncoderPolyfill || !TextDecoderPolyfill) {
+ console.warn(
+ 'Brainy: TextEncoder or TextDecoder not available, attempting to polyfill'
+ )
+
+ // If still not available, try to use a simple polyfill
+ if (!TextEncoderPolyfill) {
+ class SimpleTextEncoder {
+ encode(input: string): Uint8Array {
+ const buf = Buffer.from(input, 'utf8')
+ return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
+ }
}
- // Initialize using native constructors directly
- this.textEncoder = new (globalThis.TextEncoder || TextEncoder)()
- this.textDecoder = new (globalThis.TextDecoder || TextDecoder)()
+ global.TextEncoder = SimpleTextEncoder
}
- // Define isFloat32Array directly on the instance
- isFloat32Array(arr: any) {
- return !!(
+ if (!TextDecoderPolyfill) {
+ class SimpleTextDecoder {
+ decode(input?: Uint8Array): string {
+ if (!input) return ''
+ return Buffer.from(
+ input.buffer,
+ input.byteOffset,
+ input.byteLength
+ ).toString('utf8')
+ }
+ }
+
+ global.TextDecoder = SimpleTextDecoder
+ }
+ } else {
+ // Ensure they're available globally
+ global.TextEncoder = TextEncoderPolyfill
+ global.TextDecoder = TextDecoderPolyfill
+ }
+
+ // Create a PlatformNode implementation that uses the polyfilled TextEncoder/TextDecoder
+ class BrainyPlatformNode {
+ // Use the polyfilled TextEncoder/TextDecoder
+ readonly util = {
+ TextEncoder: global.TextEncoder,
+ TextDecoder: global.TextDecoder,
+
+ // Add utility functions that TensorFlow.js might need
+ isTypedArray: (arr: any): boolean => {
+ return ArrayBuffer.isView(arr) && !(arr instanceof DataView)
+ },
+
+ isFloat32Array: (arr: any): boolean => {
+ return (
+ arr instanceof Float32Array ||
+ (arr &&
+ Object.prototype.toString.call(arr) === '[object Float32Array]')
+ )
+ }
+ }
+
+ // Create instances of the encoder/decoder
+ readonly textEncoder: any
+ readonly textDecoder: any
+
+ constructor() {
+ try {
+ // Initialize encoders using constructors
+ this.textEncoder = new global.TextEncoder()
+ this.textDecoder = new global.TextDecoder()
+ } catch (e) {
+ console.warn(
+ 'Brainy: Error creating TextEncoder/TextDecoder instances:',
+ e
+ )
+ // Provide fallback implementations if instantiation fails
+ this.textEncoder = {
+ encode: (input: string): Uint8Array => {
+ const buf = Buffer.from(input, 'utf8')
+ return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
+ }
+ }
+ this.textDecoder = {
+ decode: (input?: Uint8Array): string => {
+ if (!input) return ''
+ return Buffer.from(
+ input.buffer,
+ input.byteOffset,
+ input.byteLength
+ ).toString('utf8')
+ }
+ }
+ }
+ }
+
+ isTypedArray(arr: any): arr is Float32Array | Int32Array | Uint8Array {
+ return ArrayBuffer.isView(arr) && !(arr instanceof DataView)
+ }
+
+ isFloat32Array(arr: any): arr is Float32Array {
+ return (
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
-
- // Define isTypedArray directly on the instance
- isTypedArray(arr: any) {
- return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
- }
}
- // Get the global object in a way that works in both Node.js and browser
- const globalObj =
- typeof global !== 'undefined'
- ? global
- : typeof window !== 'undefined'
- ? window
- : typeof self !== 'undefined'
- ? self
- : {}
+ // Assign the custom platform class to the global scope.
+ // TensorFlow.js specifically looks for `PlatformNode`.
+ global.PlatformNode = BrainyPlatformNode
- // Only apply in Node.js environment
- if (
- typeof process !== 'undefined' &&
- process.versions &&
- process.versions.node
- ) {
- // Assign the Platform class to the global object as PlatformNode for Node.js
- ;(globalObj as any).PlatformNode = Platform
- // Also create an instance and assign it to global.platformNode (lowercase p)
- ;(globalObj as any).platformNode = new Platform()
- } else if (typeof window !== 'undefined' || typeof self !== 'undefined') {
- // In browser environments, we might need to provide similar functionality
- // but we'll use a different name to avoid conflicts
- ;(globalObj as any).PlatformBrowser = Platform
- ;(globalObj as any).platformBrowser = new Platform()
+ // Also create an instance and assign it to global.platformNode (lowercase p)
+ // This is needed for some TensorFlow.js versions
+ global.platformNode = new BrainyPlatformNode()
+
+ // Set up a global flag to track TensorFlow.js initialization
+ global[TENSORFLOW_INITIALIZED] = false
+
+ // Monkey patch the registerKernel function to prevent duplicate registrations
+ // This will be applied when TensorFlow.js is imported
+ const originalRegisterKernel = global.registerKernel
+ if (!originalRegisterKernel) {
+ // Set up a handler to intercept the registerKernel function when it's defined
+ Object.defineProperty(global, 'registerKernel', {
+ set: function (newRegisterKernel) {
+ // Replace the setter with our patched version
+ Object.defineProperty(global, 'registerKernel', {
+ value: function (kernel: any) {
+ // Check if this kernel is already registered
+ const kernelName = kernel.kernelName
+ const backendName = kernel.backendName
+ const key = `${kernelName}_${backendName}`
+
+ // Use a global registry to track registered kernels
+ if (!global.__REGISTERED_KERNELS__) {
+ global.__REGISTERED_KERNELS__ = new Set()
+ }
+
+ // If this kernel is already registered, skip it
+ if (global.__REGISTERED_KERNELS__.has(key)) {
+ return
+ }
+
+ // Otherwise, register it and add it to our registry
+ global.__REGISTERED_KERNELS__.add(key)
+ return newRegisterKernel(kernel)
+ },
+ configurable: true,
+ writable: true
+ })
+ },
+ configurable: true
+ })
}
+
+ // Mark the patch as applied
+ patchApplied = true
+ console.log('Brainy: Successfully applied TensorFlow.js platform patch')
} catch (error) {
- console.warn('Failed to apply TensorFlow.js platform patch:', error)
+ console.warn('Brainy: Failed to apply TensorFlow.js platform patch:', error)
}
}
diff --git a/test-fallback-function.js b/test-fallback-function.js
deleted file mode 100644
index 2eda4d84..00000000
--- a/test-fallback-function.js
+++ /dev/null
@@ -1,59 +0,0 @@
-// Test script to verify that the function string format works with the fallback mechanism
-import { executeInThread } from './dist/unified.js'
-
-// Define a compute-intensive function using a named function declaration
-// followed by a statement that returns the function
-const computeIntensiveFunction = `
- // Define a named function
- function computeTask(data) {
- console.log('Worker/Fallback: Starting computation...');
-
- // Simulate a compute-intensive task
- const start = Date.now();
- let result = 0;
- for (let i = 0; i < data.iterations; i++) {
- result += Math.sqrt(i) * Math.sin(i);
- }
-
- const duration = Date.now() - start;
- console.log('Worker/Fallback: Computation completed in ' + duration + 'ms');
-
- return {
- result,
- duration,
- iterations: data.iterations
- };
- }
-
- // Return the function
- computeTask;
- `
-
-// Test with different environments
-async function runTests() {
- try {
- console.log('Testing executeInThread with fallback...')
-
- // Disable Web Workers to force fallback
- const originalWorker = globalThis.Worker
- globalThis.Worker = function() {
- throw new Error('Worker constructor disabled for testing')
- }
-
- try {
- // Execute the function in fallback mode
- const result = await executeInThread(computeIntensiveFunction, {
- iterations: 1000000
- })
- console.log('Fallback result:', result)
- console.log('Test passed!')
- } finally {
- // Restore Web Workers
- globalThis.Worker = originalWorker
- }
- } catch (error) {
- console.error('Test failed:', error)
- }
-}
-
-runTests()
diff --git a/test-fallback-simple.js b/test-fallback-simple.js
deleted file mode 100644
index f54c2a30..00000000
--- a/test-fallback-simple.js
+++ /dev/null
@@ -1,52 +0,0 @@
-// Test script to verify that the function string format works with the fallback mechanism
-import { executeInThread } from './dist/unified.js'
-
-// Define a compute-intensive function using a simple anonymous function expression
-const computeIntensiveFunction = `function(data) {
- console.log('Worker/Fallback: Starting computation...');
-
- // Simulate a compute-intensive task
- const start = Date.now();
- let result = 0;
- for (let i = 0; i < data.iterations; i++) {
- result += Math.sqrt(i) * Math.sin(i);
- }
-
- const duration = Date.now() - start;
- console.log('Worker/Fallback: Computation completed in ' + duration + 'ms');
-
- return {
- result,
- duration,
- iterations: data.iterations
- };
-}`
-
-// Test with different environments
-async function runTests() {
- try {
- console.log('Testing executeInThread with fallback...')
-
- // Disable Web Workers to force fallback
- const originalWorker = globalThis.Worker
- globalThis.Worker = function() {
- throw new Error('Worker constructor disabled for testing')
- }
-
- try {
- // Execute the function in fallback mode
- const result = await executeInThread(computeIntensiveFunction, {
- iterations: 1000000
- })
- console.log('Fallback result:', result)
- console.log('Test passed!')
- } finally {
- // Restore Web Workers
- globalThis.Worker = originalWorker
- }
- } catch (error) {
- console.error('Test failed:', error)
- }
-}
-
-runTests()
diff --git a/test-fix.js b/test-fix.js
deleted file mode 100644
index af000f6f..00000000
--- a/test-fix.js
+++ /dev/null
@@ -1,47 +0,0 @@
-// Test script to verify the TextEncoder fix
-import { applyTensorFlowPatch } from './dist/utils/textEncoding.js'
-
-console.log('Testing TextEncoder fix...')
-
-// Apply the TensorFlow.js platform patch
-applyTensorFlowPatch()
-
-// Check if PlatformNode is defined in the global object
-if (typeof global.PlatformNode === 'function') {
- console.log('PlatformNode is defined in the global object')
-
- // Create an instance of PlatformNode
- try {
- const platform = new global.PlatformNode()
- console.log('Successfully created PlatformNode instance')
-
- // Check if textEncoder is defined
- if (platform.textEncoder) {
- console.log('textEncoder is defined')
-
- // Test encoding a string
- const testString = 'Hello, world! 👋'
- const encoded = platform.textEncoder.encode(testString)
- console.log(`Successfully encoded string: ${testString}`)
- console.log(`Encoded: [${encoded}]`)
-
- // Test decoding
- const decoded = platform.textDecoder.decode(encoded)
- console.log(`Successfully decoded back to: ${decoded}`)
-
- if (testString === decoded) {
- console.log('✅ TextEncoder/TextDecoder test passed!')
- } else {
- console.error('❌ TextEncoder/TextDecoder test failed!')
- }
- } else {
- console.error('textEncoder is not defined in the platform instance')
- }
- } catch (error) {
- console.error('Error creating PlatformNode instance:', error)
- }
-} else {
- console.error('PlatformNode is not defined in the global object')
-}
-
-console.log('Test completed')
diff --git a/test-tensorflow-textencoder.js b/test-tensorflow-textencoder.js
deleted file mode 100644
index 498f2086..00000000
--- a/test-tensorflow-textencoder.js
+++ /dev/null
@@ -1,103 +0,0 @@
-// Test script to verify TensorFlow.js and TextEncoder functionality in Node.js environment
-import * as tf from '@tensorflow/tfjs'
-import '@tensorflow/tfjs-backend-cpu'
-import { TextEncoder, TextDecoder } from 'util'
-
-// Implement the necessary functions directly
-function applyTensorFlowPatch() {
- // This is a simplified version of the patch
- console.log('Applying TensorFlow patch directly in test file')
- return true
-}
-
-function getTextEncoder() {
- return new TextEncoder()
-}
-
-function getTextDecoder() {
- return new TextDecoder()
-}
-
-async function testTensorFlowAndTextEncoder() {
- console.log('Testing TensorFlow.js and TextEncoder in Node.js environment...')
-
- try {
- // Apply TensorFlow patch for TextEncoder compatibility
- applyTensorFlowPatch()
- console.log('TensorFlow patch applied successfully')
-
- // Test TextEncoder
- console.log('\n--- Testing TextEncoder ---')
- const encoder = getTextEncoder()
- const decoder = getTextDecoder()
-
- const testString = 'Hello, world! 👋'
- console.log(`Original string: "${testString}"`)
-
- const encoded = encoder.encode(testString)
- console.log(`Encoded: [${encoded}]`)
-
- const decoded = decoder.decode(encoded)
- console.log(`Decoded: "${decoded}"`)
-
- if (testString === decoded) {
- console.log('✅ TextEncoder/TextDecoder test passed!')
- } else {
- console.error('❌ TextEncoder/TextDecoder test failed!')
- return false
- }
-
- // Test TensorFlow.js
- console.log('\n--- Testing TensorFlow.js ---')
-
- // Create a simple tensor
- const tensor = tf.tensor2d([
- [1, 2],
- [3, 4]
- ])
- console.log('Created tensor:')
- tensor.print()
-
- // Perform a simple operation
- const result = tensor.add(tf.scalar(1))
- console.log('Result of adding 1:')
- result.print()
-
- // Check the values
- const values = await result.array()
- const expected = [
- [2, 3],
- [4, 5]
- ]
-
- console.log('Result values:', values)
- console.log('Expected values:', expected)
-
- // Compare values
- const match = JSON.stringify(values) === JSON.stringify(expected)
- if (match) {
- console.log('✅ TensorFlow.js test passed!')
- } else {
- console.error('❌ TensorFlow.js test failed!')
- return false
- }
-
- console.log('\nAll tests passed successfully!')
- return true
- } catch (error) {
- console.error('Error during test:', error)
- return false
- }
-}
-
-// Run the test
-testTensorFlowAndTextEncoder().then((success) => {
- if (success) {
- console.log(
- 'TensorFlow.js and TextEncoder verification completed successfully!'
- )
- } else {
- console.error('TensorFlow.js and TextEncoder verification failed!')
- process.exit(1)
- }
-})
diff --git a/test-unified-encoding.js b/test-unified-encoding.js
deleted file mode 100644
index 107151b8..00000000
--- a/test-unified-encoding.js
+++ /dev/null
@@ -1,39 +0,0 @@
-// Test script to verify the unified text encoding approach works correctly
-import { BrainyData } from './dist/unified.js'
-
-async function testUnifiedEncoding() {
- console.log(
- 'Testing unified text encoding approach in Node.js environment...'
- )
-
- try {
- // Initialize BrainyData which should trigger the PlatformNode constructor
- console.log('Creating BrainyData instance...')
- const db = new BrainyData()
-
- // Initialize the database
- console.log('Initializing database...')
- await db.init()
-
- console.log('Test successful! Unified text encoding is working correctly.')
-
- // Get database status to verify everything is working
- const status = await db.status()
- console.log('Database status:', status)
-
- return true
- } catch (error) {
- console.error('Error during test:', error)
- return false
- }
-}
-
-// Run the test
-testUnifiedEncoding().then((success) => {
- if (success) {
- console.log('Unified text encoding verification completed successfully!')
- } else {
- console.error('Unified text encoding verification failed!')
- process.exit(1)
- }
-})
diff --git a/test-worker-utils.js b/test-worker-utils.js
deleted file mode 100644
index 4405391a..00000000
--- a/test-worker-utils.js
+++ /dev/null
@@ -1,24 +0,0 @@
-// Test script to verify that the workerUtils functions work correctly after removing eval
-import { executeInThread } from './dist/unified.js'
-
-// Test function to execute in a thread
-const testFunction = `function(args) {
- return "Hello from " + args.name;
-}`
-
-// Test with different environments
-async function runTests() {
- try {
- console.log('Testing executeInThread...')
- const result = await executeInThread(testFunction, {
- name: 'Worker Thread'
- })
- console.log('Result:', result)
-
- console.log('All tests passed!')
- } catch (error) {
- console.error('Test failed:', error)
- }
-}
-
-runTests()