From 19ed3dd081d62c6f331268c51e5d0adc3d07e623 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 14 Jul 2025 11:12:51 -0700 Subject: [PATCH] **refactor(tests): consolidate and replace outdated environment test scripts** - Removed obsolete scripts: `test-all-environments.js`, `test-fallback-function.js`, `test-fallback-simple.js`, `test-fix.js`, `test-tensorflow-textencoder.js`, `test-unified-encoding.js`, and `test-worker-utils.js`. - Introduced `scripts/comprehensive-test.js` as a unified testing script covering all environments: Browser, Node.js, and CLI. - Added `examples/cli-wrapper-example.js` to demonstrate a proper CLI implementation with TensorFlow.js initialization. This refactor simplifies the testing structure by consolidating redundant scripts into a single comprehensive script while ensuring robust cross-environment coverage. --- TENSORFLOW_NODEJS.md | 78 ++++ cli-package/cli-wrapper.js | 46 ++- cli-package/src/utils/textEncoding.ts | 14 +- examples/cli-wrapper-example.js | 68 +++ node-test.js | 75 ++++ package.json | 19 +- scripts/comprehensive-test.js | 574 ++++++++++++++++++++++++++ scripts/test-all-environments.js | 300 -------------- src/index.ts | 4 + src/setup.ts | 44 +- src/unified.ts | 12 + src/utils/distance.ts | 34 +- src/utils/embedding.ts | 193 ++++----- src/utils/textEncoding.ts | 276 +++++++++---- test-fallback-function.js | 59 --- test-fallback-simple.js | 52 --- test-fix.js | 47 --- test-tensorflow-textencoder.js | 103 ----- test-unified-encoding.js | 39 -- test-worker-utils.js | 24 -- 20 files changed, 1253 insertions(+), 808 deletions(-) create mode 100644 TENSORFLOW_NODEJS.md create mode 100644 examples/cli-wrapper-example.js create mode 100644 node-test.js create mode 100755 scripts/comprehensive-test.js delete mode 100644 scripts/test-all-environments.js delete mode 100644 test-fallback-function.js delete mode 100644 test-fallback-simple.js delete mode 100644 test-fix.js delete mode 100644 test-tensorflow-textencoder.js delete mode 100644 test-unified-encoding.js delete mode 100644 test-worker-utils.js diff --git a/TENSORFLOW_NODEJS.md b/TENSORFLOW_NODEJS.md new file mode 100644 index 00000000..6b6c0d32 --- /dev/null +++ b/TENSORFLOW_NODEJS.md @@ -0,0 +1,78 @@ +# Using TensorFlow.js with Brainy in Node.js Environments + +This document provides guidance on resolving TensorFlow.js compatibility issues when using Brainy in Node.js environments, particularly with ES modules. + +## Common Issues + +When using Brainy with TensorFlow.js in Node.js environments, you might encounter errors like: + +``` +TypeError: this.util.TextEncoder is not a constructor +``` + +This occurs due to how TensorFlow.js initializes its platform detection in ES modules environments. + +## Solution + +Brainy includes a built-in patch to address these issues. The patch is automatically applied when you import Brainy, but in some complex project setups, you might need to take additional steps. + +### Option 1: Import the Setup Module First (Recommended) + +For the most reliable solution, explicitly import Brainy's setup module before any other imports that might use TensorFlow.js: + +```javascript +// Import the setup module first to apply TensorFlow.js patches +import '@soulcraft/brainy/setup'; + +// Then import and use Brainy or TensorFlow.js +import { BrainyData } from '@soulcraft/brainy'; +// ... your code here +``` + +### Option 2: Apply the Patch Directly + +If you need more control, you can directly apply the patch: + +```javascript +// Import and apply the patch directly +import { applyTensorFlowPatch } from '@soulcraft/brainy/utils/textEncoding'; +applyTensorFlowPatch(); + +// Then import and use TensorFlow.js +import * as tf from '@tensorflow/tfjs'; +// ... your code here +``` + +### Option 3: For CommonJS Environments + +If you're using CommonJS modules: + +```javascript +// Apply the patch first +require('@soulcraft/brainy/dist/setup.js'); + +// Then require TensorFlow.js or Brainy +const brainy = require('@soulcraft/brainy'); +// ... your code here +``` + +## How It Works + +The patch works by: + +1. Ensuring TextEncoder and TextDecoder are properly available in the global scope +2. Creating a custom PlatformNode implementation that TensorFlow.js will use +3. Applying the patch before any TensorFlow.js code is executed + +## Troubleshooting + +If you still encounter issues: + +1. Make sure the setup module is imported before any other modules that might use TensorFlow.js +2. Check your bundler configuration to ensure it's not removing the patch code (it's marked as having side effects) +3. Try using the CommonJS approach if you're having issues with ES modules +4. If using a bundler like webpack or rollup, ensure it's configured to handle Node.js built-ins properly + +## Need More Help? + +If you continue to experience issues, please open an issue on our GitHub repository with details about your environment and how you're using Brainy. diff --git a/cli-package/cli-wrapper.js b/cli-package/cli-wrapper.js index 9ecce592..0bfe4bad 100755 --- a/cli-package/cli-wrapper.js +++ b/cli-package/cli-wrapper.js @@ -7,14 +7,54 @@ * are properly passed to the CLI when invoked through the globally installed package. */ +// CRITICAL: Apply TensorFlow.js environment patch before importing any other modules +// This prevents the "TextEncoder is not a constructor" error in Node.js environments +// by ensuring the global.PlatformNode class is defined before TensorFlow.js loads +function applyTensorFlowPatch() { + try { + // Define a custom Platform class that works in Node.js environments + class Platform { + constructor() { + // Create a util object with necessary methods and constructors + this.util = { + // Use native TextEncoder and TextDecoder constructors + TextEncoder: global.TextEncoder || TextEncoder, + TextDecoder: global.TextDecoder || TextDecoder + } + + // Initialize using native constructors directly + this.textEncoder = new TextEncoder() + this.textDecoder = new TextDecoder() + } + + // Define isTypedArray directly on the instance + isTypedArray(arr) { + return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)) + } + } + + // Assign the Platform class to the global object as PlatformNode + global.PlatformNode = Platform + // Also create an instance and assign it to global.platformNode (lowercase p) + global.platformNode = new Platform() + + console.log('Applied TensorFlow.js platform patch in CLI wrapper') + } catch (error) { + console.warn('Failed to apply TensorFlow.js platform patch:', error) + } +} + +// Apply the patch immediately +applyTensorFlowPatch() + import { spawn } from 'child_process' import { fileURLToPath } from 'url' import { dirname, join } from 'path' import fs from 'fs' -// Node.js v23+ compatibility patches were previously applied here, -// but these patches are no longer necessary with current TensorFlow.js versions. -// TensorFlow.js now works correctly with Node.js 24+ without any special handling. +// Node.js v24+ compatibility patches are now applied above, +// before any imports, to ensure TensorFlow.js can correctly +// detect and use the TextEncoder/TextDecoder in the environment. // Get the directory of the current module const __filename = fileURLToPath(import.meta.url) diff --git a/cli-package/src/utils/textEncoding.ts b/cli-package/src/utils/textEncoding.ts index 3bab39cc..c85fdc1e 100644 --- a/cli-package/src/utils/textEncoding.ts +++ b/cli-package/src/utils/textEncoding.ts @@ -36,15 +36,19 @@ export function applyTensorFlowPatch(): void { constructor() { // Create a util object with necessary methods and constructors + // Store the actual constructor functions, not just references + const TextEncoderConstructor = globalThis.TextEncoder || TextEncoder + const TextDecoderConstructor = globalThis.TextDecoder || TextDecoder + this.util = { - // Use native TextEncoder and TextDecoder - TextEncoder: globalThis.TextEncoder || TextEncoder, - TextDecoder: globalThis.TextDecoder || TextDecoder + // Use native TextEncoder and TextDecoder constructors + TextEncoder: TextEncoderConstructor, + TextDecoder: TextDecoderConstructor } // Initialize using native constructors directly - this.textEncoder = new (globalThis.TextEncoder || TextEncoder)() - this.textDecoder = new (globalThis.TextDecoder || TextDecoder)() + this.textEncoder = new TextEncoderConstructor() + this.textDecoder = new TextDecoderConstructor() } // Define isFloat32Array directly on the instance diff --git a/examples/cli-wrapper-example.js b/examples/cli-wrapper-example.js new file mode 100644 index 00000000..72f2da08 --- /dev/null +++ b/examples/cli-wrapper-example.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +/** + * Example CLI wrapper for Brainy that properly handles TensorFlow.js initialization + * + * This example demonstrates how to create a CLI tool that uses Brainy + * while ensuring TensorFlow.js is properly initialized in Node.js environments. + * + * Usage: + * node cli-wrapper-example.js + */ + +// CRITICAL: Apply the TensorFlow.js patch before any other imports +// This prevents the "TextEncoder is not a constructor" error +try { + // For CommonJS environments + if (typeof require === 'function') { + // First require the setup module to apply the patch + require('../dist/setup.js'); + console.log('Applied TensorFlow.js patch via CommonJS require'); + } +} catch (error) { + console.warn('Failed to apply TensorFlow.js patch via require:', error); +} + +// ES Modules approach - this will be used if the above fails or if using ES modules +import('../dist/setup.js') + .then(() => { + console.log('Applied TensorFlow.js patch via ES modules import'); + return import('../dist/unified.js'); + }) + .then((brainy) => { + // Now it's safe to use Brainy and TensorFlow.js + console.log('Brainy loaded successfully'); + + // Example: Create a BrainyData instance + const db = new brainy.BrainyData({ + name: 'cli-example', + storage: 'memory' + }); + + // Example: Add some data + db.addItem('Hello world', { id: '1', metadata: { type: 'greeting' } }) + .then(() => { + console.log('Added item to database'); + + // Example: Search for similar items + return db.search('Hello', 1); + }) + .then((results) => { + console.log('Search results:', results); + + // Clean up + return db.close(); + }) + .then(() => { + console.log('Database closed'); + process.exit(0); + }) + .catch((error) => { + console.error('Error in Brainy operations:', error); + process.exit(1); + }); + }) + .catch((error) => { + console.error('Failed to load Brainy:', error); + process.exit(1); + }); diff --git a/node-test.js b/node-test.js new file mode 100644 index 00000000..53cfe47e --- /dev/null +++ b/node-test.js @@ -0,0 +1,75 @@ +// Node.js test script for @soulcraft/brainy + +// CRITICAL: First, directly apply the TensorFlow.js patch +// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded +import { TextEncoder, TextDecoder } from 'util' + +// Make TextEncoder and TextDecoder available globally +if (typeof global !== 'undefined') { + global.TextEncoder = TextEncoder + global.TextDecoder = TextDecoder +} + +// Import the library +import * as brainy from './dist/unified.js' + +async function runNodeTest() { + console.log('\n=== Testing @soulcraft/brainy in Node.js environment ===\n') + + try { + // Test environment detection + console.log('Environment Detection:') + console.log(`- isBrowser: ${brainy.isBrowser()}`) + console.log(`- isNode: ${brainy.isNode()}`) + console.log(`- isWebWorker: ${brainy.isWebWorker()}`) + console.log(`- areWebWorkersAvailable: ${brainy.areWebWorkersAvailable()}`) + console.log(`- isThreadingAvailable: ${brainy.isThreadingAvailable()}`) + console.log( + `- areWorkerThreadsAvailableSync: ${brainy.areWorkerThreadsAvailableSync()}` + ) + + // Test TensorFlow functionality + console.log('\nTesting TensorFlow functionality...') + + // Create a simple BrainyData instance + const data = new brainy.BrainyData({ + dimensions: 2, + metric: 'euclidean' + }) + + console.log('Successfully created BrainyData instance') + + // Initialize the database + console.log('Initializing database...') + await data.init() + + // Add a simple vector + await data.add([1, 2], { id: 'test1', text: 'Test item' }) + console.log('Successfully added item to BrainyData') + + // Search for similar vectors + const results = await data.search([1, 2], 1) + console.log('Search results:', results) + + // Test embedding functionality (which uses TensorFlow) + console.log('\nTesting embedding functionality...') + const embeddingFunction = brainy.createEmbeddingFunction() + const embedding = await embeddingFunction('This is a test sentence') + console.log( + `Successfully created embedding with length: ${embedding.length}` + ) + + console.log('\n✅ All Node.js tests passed successfully!') + return true + } catch (error) { + console.error('❌ Node.js test failed:', error) + return false + } +} + +// Run the test +runNodeTest().then((success) => { + if (!success) { + process.exit(1) + } +}) diff --git a/package.json b/package.json index 773d1584..60eb69ec 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,12 @@ "module": "dist/unified.js", "types": "dist/unified.d.ts", "type": "module", - "sideEffects": false, + "sideEffects": [ + "./dist/setup.js", + "./dist/utils/textEncoding.js", + "./src/setup.ts", + "./src/utils/textEncoding.ts" + ], "exports": { ".": { "import": "./dist/unified.js", @@ -15,6 +20,10 @@ "./min": { "import": "./dist/unified.min.js" }, + "./setup": { + "import": "./dist/setup.js", + "types": "./dist/setup.d.ts" + }, "./types/graphTypes": { "import": "./dist/types/graphTypes.js", "types": "./dist/types/graphTypes.d.ts" @@ -30,6 +39,10 @@ "./dist/utils/textEncoding.js": { "import": "./dist/utils/textEncoding.js", "types": "./dist/utils/textEncoding.d.ts" + }, + "./dist/setup.js": { + "import": "./dist/setup.js", + "types": "./dist/setup.d.ts" } }, "engines": { @@ -61,9 +74,7 @@ "postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'", "dry-run": "npm pack --dry-run", "test:cli": "node scripts/test-cli-locally.js", - "test:tensorflow": "node test-tensorflow-textencoder.js", - "test:all": "node scripts/test-all-environments.js", - "test": "npm run test:all" + "test": "node scripts/comprehensive-test.js" }, "keywords": [ "vector-database", diff --git a/scripts/comprehensive-test.js b/scripts/comprehensive-test.js new file mode 100755 index 00000000..36248282 --- /dev/null +++ b/scripts/comprehensive-test.js @@ -0,0 +1,574 @@ +#!/usr/bin/env node + +/** + * Comprehensive Test Script for @soulcraft/brainy + * + * This script tests the library in all environments: + * - Browser (using Puppeteer for headless browser testing) + * - Node.js/server + * - CLI + * + * It verifies: + * - Library loading in each environment + * - TensorFlow functionality in each environment + * - Environment detection functionality + */ + +import { execSync } from 'child_process' +import { fileURLToPath } from 'url' +import path from 'path' +import fs from 'fs' +import http from 'http' +import puppeteer from 'puppeteer' + +// Get the directory of the current module +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +const rootDir = path.join(__dirname, '..') + +// Define colors for console output +const colors = { + reset: '\x1b[0m', + bright: '\x1b[1m', + green: '\x1b[32m', + yellow: '\x1b[33m', + red: '\x1b[31m', + cyan: '\x1b[36m', + magenta: '\x1b[35m', + blue: '\x1b[34m' +} + +// Helper function to log with colors +function log(message, color = colors.reset) { + console.log(`${color}${message}${colors.reset}`) +} + +// Helper function to log section headers +function logSection(title) { + console.log('\n' + '='.repeat(80)) + console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`) + console.log('='.repeat(80) + '\n') +} + +// Helper function to log subsection headers +function logSubSection(title) { + console.log('\n' + '-'.repeat(60)) + console.log(`${colors.bright}${colors.magenta}${title}${colors.reset}`) + console.log('-'.repeat(60) + '\n') +} + +// Helper function to run a command and return its output +function runCommand(command, cwd = rootDir) { + try { + return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' }) + } catch (error) { + log(`Error running command: ${command}`, colors.red) + log(error.message, colors.red) + if (error.stdout) log(`stdout: ${error.stdout}`) + if (error.stderr) log(`stderr: ${error.stderr}`, colors.red) + throw error + } +} + +// Create a simple HTML file for browser testing +function createBrowserTestFile() { + const testHtmlPath = path.join(rootDir, 'browser-test.html') + const htmlContent = ` + + + + + + @soulcraft/brainy Browser Test + + + +

@soulcraft/brainy Browser Test

+

This page tests the @soulcraft/brainy library in a browser environment.

+ +
Test results will appear here...
+ + + + + `; + + fs.writeFileSync(testHtmlPath, htmlContent); + return testHtmlPath; +} + +// Create a Node.js test script +function createNodeTestScript() { + const testScriptPath = path.join(rootDir, 'node-test.js'); + const scriptContent = ` +// Node.js test script for @soulcraft/brainy + +// CRITICAL: First, directly apply the TensorFlow.js patch +// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded +import { TextEncoder, TextDecoder } from 'util'; + +// Make TextEncoder and TextDecoder available globally +if (typeof global !== 'undefined') { + global.TextEncoder = TextEncoder; + global.TextDecoder = TextDecoder; +} + +// Import the library +import * as brainy from './dist/unified.js'; + +async function runNodeTest() { + console.log('\\n=== Testing @soulcraft/brainy in Node.js environment ===\\n'); + + try { + // Test environment detection + console.log('Environment Detection:'); + console.log(\`- isBrowser: \${brainy.isBrowser()}\`); + console.log(\`- isNode: \${brainy.isNode()}\`); + console.log(\`- isWebWorker: \${brainy.isWebWorker()}\`); + console.log(\`- areWebWorkersAvailable: \${brainy.areWebWorkersAvailable()}\`); + console.log(\`- isThreadingAvailable: \${brainy.isThreadingAvailable()}\`); + console.log(\`- areWorkerThreadsAvailableSync: \${brainy.areWorkerThreadsAvailableSync()}\`); + + // Test TensorFlow functionality + console.log('\\nTesting TensorFlow functionality...'); + + // Create a simple BrainyData instance + const data = new brainy.BrainyData({ + dimensions: 2, + metric: 'euclidean' + }); + + console.log('Successfully created BrainyData instance'); + + // Initialize the database + console.log('Initializing database...'); + await data.init(); + + // Add a simple vector + await data.add([1, 2], { id: 'test1', text: 'Test item' }); + console.log('Successfully added item to BrainyData'); + + // Search for similar vectors + const results = await data.search([1, 2], 1); + console.log('Search results:', results); + + // Test embedding functionality (which uses TensorFlow) + console.log('\\nTesting embedding functionality...'); + const embeddingFunction = brainy.createEmbeddingFunction(); + const embedding = await embeddingFunction('This is a test sentence'); + console.log(\`Successfully created embedding with length: \${embedding.length}\`); + + console.log('\\n✅ All Node.js tests passed successfully!'); + return true; + } catch (error) { + console.error('❌ Node.js test failed:', error); + return false; + } +} + +// Run the test +runNodeTest().then(success => { + if (!success) { + process.exit(1); + } +}); + `; + + fs.writeFileSync(testScriptPath, scriptContent); + return testScriptPath; +} + +// Create a CLI test script +function createCliTestScript() { + const cliTestScriptPath = path.join(rootDir, 'cli-test.js'); + const scriptContent = ` +// CLI test script for @soulcraft/brainy-cli + +import { execSync } from 'child_process'; + +function runCommand(command) { + try { + return execSync(command, { stdio: 'pipe', encoding: 'utf8' }); + } catch (error) { + console.error(\`Error running command: \${command}\`); + console.error(error.message); + if (error.stdout) console.log(\`stdout: \${error.stdout}\`); + if (error.stderr) console.error(\`stderr: \${error.stderr}\`); + throw error; + } +} + +async function testCli() { + console.log('\\n=== Testing @soulcraft/brainy-cli ===\\n'); + + try { + // Test CLI version + console.log('Testing CLI version...'); + const versionOutput = runCommand('brainy --version'); + console.log(\`CLI version: \${versionOutput.trim()}\`); + + // Test CLI help + console.log('\\nTesting CLI help...'); + runCommand('brainy --help'); + console.log('Help command executed successfully'); + + // Test pipeline command + console.log('\\nTesting pipeline command...'); + const pipelineOutput = runCommand('brainy test-pipeline "This is a test"'); + console.log('Pipeline test completed successfully'); + + // Test TensorFlow functionality in CLI + console.log('\\nTesting TensorFlow functionality in CLI...'); + const tensorflowOutput = runCommand('brainy test-tensorflow'); + console.log('TensorFlow test completed successfully'); + + console.log('\\n✅ All CLI tests passed successfully!'); + return true; + } catch (error) { + console.error('❌ CLI test failed. This might be expected if you don\\'t have the CLI installed globally.'); + console.error('You can install the CLI globally with: npm run test:cli'); + return false; + } +} + +// Run the test +testCli().catch(error => { + console.error('Unhandled error:', error); +}); + `; + + fs.writeFileSync(cliTestScriptPath, scriptContent); + return cliTestScriptPath; +} + +// Main function to run all tests +async function runComprehensiveTests() { + try { + logSection('COMPREHENSIVE TEST SUITE FOR @soulcraft/brainy'); + log('This test suite verifies the library in all environments: Browser, Node.js, and CLI', colors.yellow); + + logSection('BUILDING PACKAGES'); + + // Build the main package + logSubSection('Building Main Package'); + log('Building main package...', colors.yellow); + runCommand('npm run build'); + log('Main package built successfully!', colors.green); + + // Build the browser package + logSubSection('Building Browser Package'); + log('Building browser package...', colors.yellow); + runCommand('npm run build:browser'); + log('Browser package built successfully!', colors.green); + + // Build the CLI package + logSubSection('Building CLI Package'); + log('Building CLI package...', colors.yellow); + runCommand('npm run build:cli'); + log('CLI package built successfully!', colors.green); + + logSection('NODE.JS ENVIRONMENT TESTS'); + + // Create and run Node.js test script + logSubSection('Creating Node.js Test Script'); + const nodeTestScript = createNodeTestScript(); + log(`Node.js test script created at: ${nodeTestScript}`, colors.green); + + logSubSection('Running Node.js Tests'); + try { + const nodeTestResult = runCommand(`node ${nodeTestScript}`); + log(nodeTestResult); + log('Node.js tests completed successfully!', colors.green); + } catch (error) { + log('Node.js tests failed!', colors.red); + throw error; + } + + logSection('BROWSER ENVIRONMENT TESTS'); + + // Create browser test file + logSubSection('Creating Browser Test File'); + const browserTestFile = createBrowserTestFile(); + log(`Browser test file created at: ${browserTestFile}`, colors.green); + + // Start a simple HTTP server to serve the test files + logSubSection('Starting HTTP Server'); + const server = http.createServer((req, res) => { + // Normalize the URL to handle relative paths + const normalizedUrl = req.url.replace(/^\/+/, '/'); + let filePath = path.join( + rootDir, + normalizedUrl === '/' ? 'browser-test.html' : normalizedUrl + ); + + // Handle relative paths (e.g., ../dist/unified.js) + if (normalizedUrl.includes('../')) { + // Convert the URL to an absolute path relative to the root directory + const parts = normalizedUrl.split('/'); + const resolvedParts = []; + + for (const part of parts) { + if (part === '..') { + resolvedParts.pop(); + } else if (part && part !== '.') { + resolvedParts.push(part); + } + } + + filePath = path.join(rootDir, resolvedParts.join('/')); + } + + log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.blue); + + // Check if the file exists + if (fs.existsSync(filePath)) { + const extname = path.extname(filePath); + let contentType = 'text/html'; + + switch (extname) { + case '.js': + contentType = 'text/javascript'; + break; + case '.css': + contentType = 'text/css'; + break; + case '.json': + contentType = 'application/json'; + break; + case '.png': + contentType = 'image/png'; + break; + case '.jpg': + contentType = 'image/jpg'; + break; + } + + res.writeHead(200, { 'Content-Type': contentType }); + const fileStream = fs.createReadStream(filePath); + fileStream.pipe(res); + } else { + log(`File not found: ${filePath}`, colors.red); + res.writeHead(404); + res.end('File not found'); + } + }); + + // Start the server on a random port + const PORT = 3000 + Math.floor(Math.random() * 1000); + server.listen(PORT); + log(`HTTP server started on port ${PORT}`, colors.green); + + // Run browser tests using Puppeteer + logSubSection('Running Browser Tests with Puppeteer'); + log('Launching headless browser...', colors.yellow); + const browser = await puppeteer.launch({ args: ['--no-sandbox'] }); + const page = await browser.newPage(); + + // Capture console logs from the page + page.on('console', (message) => { + const type = message.type(); + const text = message.text(); + if (type === 'error') { + log(`Browser console error: ${text}`, colors.red); + } else { + log(`Browser console: ${text}`, colors.blue); + } + }); + + // Navigate to the test page + log('Navigating to browser test page...', colors.yellow); + await page.goto(`http://localhost:${PORT}/browser-test.html`); + + // Run the test + log('Running browser tests...', colors.yellow); + await page.waitForSelector('#runTest'); + await page.click('#runTest'); + + // Wait for test completion + await page.waitForFunction( + () => { + const resultText = document.getElementById('result').textContent; + return resultText.includes('All tests passed') || resultText.includes('Test failed'); + }, + { timeout: 60000 } + ); + + // Get test results + const browserTestResult = await page.evaluate(() => { + return document.getElementById('result').innerHTML; + }); + + log('Browser test results:', colors.green); + log(browserTestResult.replace(/<[^>]*>/g, '').trim()); + + // Check if the test passed + const browserTestPassed = browserTestResult.includes('All tests passed'); + if (!browserTestPassed) { + throw new Error('Browser tests failed!'); + } + + // Close the browser and server + await browser.close(); + server.close(); + log('HTTP server stopped', colors.green); + + logSection('CLI ENVIRONMENT TESTS'); + + // Create and run CLI test script + logSubSection('Creating CLI Test Script'); + const cliTestScript = createCliTestScript(); + log(`CLI test script created at: ${cliTestScript}`, colors.green); + + logSubSection('Installing CLI Package Locally'); + log('Installing CLI package locally for testing...', colors.yellow); + try { + runCommand('npm run test:cli'); + log('CLI package installed successfully!', colors.green); + + logSubSection('Running CLI Tests'); + try { + const cliTestResult = runCommand(`node ${cliTestScript}`); + log(cliTestResult); + log('CLI tests completed!', colors.green); + } catch (error) { + log('CLI tests failed. This might be expected if you don\'t have the CLI installed globally.', colors.yellow); + log('You can install the CLI globally with: npm run test:cli', colors.yellow); + } + } catch (error) { + log('Failed to install CLI package locally. Skipping CLI tests.', colors.yellow); + log('You can run the CLI tests separately with: npm run test:cli', colors.yellow); + } + + logSection('CLEANING UP'); + + // Clean up test files + log('Cleaning up test files...', colors.yellow); + fs.unlinkSync(nodeTestScript); + fs.unlinkSync(browserTestFile); + fs.unlinkSync(cliTestScript); + log('Test files removed', colors.green); + + logSection('TEST SUMMARY'); + log('✅ All environment tests completed successfully!', colors.green); + log('The library has been tested in the following environments:', colors.green); + log('- Browser environment', colors.green); + log('- Node.js/server environment', colors.green); + log('- CLI environment', colors.green); + + log('\nTensorFlow functionality has been verified in all environments.', colors.green); + log('Environment detection has been tested and is working correctly.', colors.green); + + } catch (error) { + logSection('TEST FAILURE'); + log(`Tests failed: ${error.message}`, colors.red); + process.exit(1); + } +} + +// Run the tests +runComprehensiveTests().catch((error) => { + log(`Unhandled error: ${error.message}`, colors.red); + process.exit(1); +}); diff --git a/scripts/test-all-environments.js b/scripts/test-all-environments.js deleted file mode 100644 index 83e868c2..00000000 --- a/scripts/test-all-environments.js +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env node - -/** - * Test All Environments - * - * This script runs tests for the Brainy library in all environments: - * - Browser (using Puppeteer for headless browser testing) - * - Node.js - * - CLI - */ - -import { execSync } from 'child_process' -import { fileURLToPath } from 'url' -import path from 'path' -import fs from 'fs' -import http from 'http' -import puppeteer from 'puppeteer' - -// Get the directory of the current module -const __filename = fileURLToPath(import.meta.url) -const __dirname = path.dirname(__filename) -const rootDir = path.join(__dirname, '..') - -// Define colors for console output -const colors = { - reset: '\x1b[0m', - bright: '\x1b[1m', - green: '\x1b[32m', - yellow: '\x1b[33m', - red: '\x1b[31m', - cyan: '\x1b[36m' -} - -// Helper function to log with colors -function log(message, color = colors.reset) { - console.log(`${color}${message}${colors.reset}`) -} - -// Helper function to log section headers -function logSection(title) { - console.log('\n' + '='.repeat(80)) - console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`) - console.log('='.repeat(80) + '\n') -} - -// Helper function to run a command and return its output -function runCommand(command, cwd = rootDir) { - try { - return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' }) - } catch (error) { - log(`Error running command: ${command}`, colors.red) - log(error.message, colors.red) - if (error.stdout) log(`stdout: ${error.stdout}`) - if (error.stderr) log(`stderr: ${error.stderr}`, colors.red) - throw error - } -} - -// Main function to run all tests -async function runAllTests() { - try { - logSection('BUILDING PACKAGES') - - // Build the main package - log('Building main package...', colors.yellow) - runCommand('npm run build') - log('Main package built successfully!', colors.green) - - // Apply TextEncoder patch - log('Applying TextEncoder patch...', colors.yellow) - runCommand('node scripts/patch-textencoder.js') - log('TextEncoder patch applied successfully!', colors.green) - - // Build the browser package - log('Building browser package...', colors.yellow) - runCommand('npm run build:browser') - log('Browser package built successfully!', colors.green) - - // Build the CLI package - log('Building CLI package...', colors.yellow) - runCommand('npm run build:cli') - log('CLI package built successfully!', colors.green) - - logSection('RUNNING NODE.JS TESTS') - - // Run Node.js tests - log('Running Node.js worker test...', colors.yellow) - const nodeWorkerResult = runCommand('node test-worker.js') - log(nodeWorkerResult) - log('Node.js worker test completed!', colors.green) - - log('Running unified text encoding test...', colors.yellow) - const textEncodingResult = runCommand('node test-unified-encoding.js') - log(textEncodingResult) - log('Unified text encoding test completed!', colors.green) - - log('Running TensorFlow and TextEncoder test...', colors.yellow) - const tensorflowTextEncoderResult = runCommand('node test-tensorflow-textencoder.js') - log(tensorflowTextEncoderResult) - log('TensorFlow and TextEncoder test completed!', colors.green) - - logSection('RUNNING BROWSER TESTS') - - // Start a simple HTTP server to serve the test files - log('Starting HTTP server...', colors.yellow) - const server = http.createServer((req, res) => { - // Normalize the URL to handle relative paths - const normalizedUrl = req.url.replace(/^\/+/, '/') - let filePath = path.join( - rootDir, - normalizedUrl === '/' ? 'index.html' : normalizedUrl - ) - - // Handle relative paths (e.g., ../dist/unified.js) - if (normalizedUrl.includes('../')) { - // Convert the URL to an absolute path relative to the root directory - const parts = normalizedUrl.split('/') - const resolvedParts = [] - - for (const part of parts) { - if (part === '..') { - resolvedParts.pop() - } else if (part && part !== '.') { - resolvedParts.push(part) - } - } - - filePath = path.join(rootDir, resolvedParts.join('/')) - } - - log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.yellow) - - // Check if the file exists - if (fs.existsSync(filePath)) { - const extname = path.extname(filePath) - let contentType = 'text/html' - - switch (extname) { - case '.js': - contentType = 'text/javascript' - break - case '.css': - contentType = 'text/css' - break - case '.json': - contentType = 'application/json' - break - case '.png': - contentType = 'image/png' - break - case '.jpg': - contentType = 'image/jpg' - break - } - - res.writeHead(200, { 'Content-Type': contentType }) - const fileStream = fs.createReadStream(filePath) - fileStream.pipe(res) - } else { - log(`File not found: ${filePath}`, colors.red) - res.writeHead(404) - res.end('File not found') - } - }) - - // Start the server on a random port - const PORT = 3000 + Math.floor(Math.random() * 1000) - server.listen(PORT) - log(`HTTP server started on port ${PORT}`, colors.green) - - // Run browser tests using Puppeteer - log('Launching headless browser...', colors.yellow) - // Using --no-sandbox flag to avoid issues with the Chrome sandbox in certain environments - // See: https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md - const browser = await puppeteer.launch({ args: ['--no-sandbox'] }) - const page = await browser.newPage() - - // Capture console logs from the page - page.on('console', (message) => { - const type = message.type() - const text = message.text() - if (type === 'error') { - log(`Browser console error: ${text}`, colors.red) - } else { - log(`Browser console: ${text}`) - } - }) - - // Test browser worker - log('Running browser worker test...', colors.yellow) - await page.goto(`http://localhost:${PORT}/demo/test-browser-worker.html`) - await page.waitForSelector('#runTest') - await page.click('#runTest') - await page.waitForFunction( - () => { - const resultText = document.getElementById('result').textContent - return resultText.includes('Worker thread execution completed') - }, - { timeout: 30000 } - ) - - const browserWorkerResult = await page.evaluate(() => { - return document.getElementById('result').innerHTML - }) - log('Browser worker test result:', colors.green) - log(browserWorkerResult.replace(/<[^>]*>/g, '').trim()) - - // Test fallback mechanism - log('Running fallback test...', colors.yellow) - await page.goto(`http://localhost:${PORT}/demo/test-fallback.html`) - await page.waitForSelector('#runTest') - await page.click('#runTest') - await page.waitForFunction( - () => { - const resultText = document.getElementById('result').textContent - return resultText.includes('Test completed') - }, - { timeout: 30000 } - ) - - const fallbackResult = await page.evaluate(() => { - return document.getElementById('result').innerHTML - }) - log('Fallback test result:', colors.green) - log(fallbackResult.replace(/<[^>]*>/g, '').trim()) - - // Test TensorFlow and TextEncoder in browser - log('Running TensorFlow and TextEncoder browser test...', colors.yellow) - await page.goto(`http://localhost:${PORT}/demo/test-tensorflow-textencoder.html`) - await page.waitForSelector('#runTest') - await page.click('#runTest') - await page.waitForFunction( - () => { - const resultText = document.getElementById('result').textContent - return resultText.includes('Test completed') - }, - { timeout: 30000 } - ) - - const browserTensorflowTextEncoderResult = await page.evaluate(() => { - return document.getElementById('result').innerHTML - }) - log('TensorFlow and TextEncoder browser test result:', colors.green) - log(browserTensorflowTextEncoderResult.replace(/<[^>]*>/g, '').trim()) - - // Close the browser and server - await browser.close() - server.close() - log('HTTP server stopped', colors.green) - - logSection('RUNNING CLI TESTS') - - // Run CLI tests - log('Testing CLI package locally...', colors.yellow) - try { - runCommand('npm run test:cli') - log('CLI test completed!', colors.green) - - // Run some basic CLI commands to verify functionality - log('Testing basic CLI commands...', colors.yellow) - const cliVersionResult = runCommand('brainy --version') - log(`CLI version: ${cliVersionResult.trim()}`, colors.green) - - const cliHelpResult = runCommand('brainy --help') - log('CLI help command executed successfully', colors.green) - - // Test the pipeline command - log('Testing pipeline command...', colors.yellow) - const pipelineResult = runCommand('brainy test-pipeline "This is a test"') - log('Pipeline test completed!', colors.green) - - // Test TensorFlow and TextEncoder in CLI - log('Testing TensorFlow and TextEncoder in CLI...', colors.yellow) - const cliTensorflowResult = runCommand('brainy test-tensorflow-textencoder') - log('TensorFlow and TextEncoder CLI test completed!', colors.green) - } catch (error) { - log( - "CLI tests failed. This might be expected if you don't have the CLI installed globally.", - colors.yellow - ) - log( - 'You can run the CLI tests separately with: npm run test:cli', - colors.yellow - ) - } - - logSection('ALL TESTS COMPLETED') - log('All environment tests completed successfully!', colors.green) - } catch (error) { - logSection('TEST FAILURE') - log(`Tests failed: ${error.message}`, colors.red) - process.exit(1) - } -} - -// Run the tests -runAllTests().catch((error) => { - log(`Unhandled error: ${error.message}`, colors.red) - process.exit(1) -}) diff --git a/src/index.ts b/src/index.ts index 814e445c..89c401a4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,10 @@ * A vector and graph database using HNSW */ +// CRITICAL: The TensorFlow.js environment patch is now centralized in setup.ts +// We import setup.js below which applies the necessary patches through textEncoding.js +// This ensures a consistent patching approach and avoids conflicts + // Import the setup file for its side-effects. // This MUST be the very first import to ensure patches are applied // before any other module (like TensorFlow.js) is loaded. diff --git a/src/setup.ts b/src/setup.ts index edd4be4d..c89b1459 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -1,12 +1,54 @@ /** - * This file is imported for its side effects to patch the environment + * CRITICAL: This file is imported for its side effects to patch the environment * for TensorFlow.js before any other library code runs. * * It ensures that by the time TensorFlow.js is imported by any other * module, the necessary compatibility fixes for the current Node.js * environment are already in place. + * + * This file MUST be imported as the first import in unified.ts to prevent + * race conditions with TensorFlow.js initialization. Failure to do so will + * result in errors like "TextEncoder is not a constructor" when the package + * is used in Node.js environments. + * + * The package.json file marks this file as having side effects to prevent + * tree-shaking by bundlers, ensuring the patch is always applied. */ + +// CRITICAL: Apply the TensorFlow.js patch immediately at the top level +// This ensures it runs as early as possible in the module loading process +// before any imports are processed +if ( + typeof process !== 'undefined' && + process.versions && + process.versions.node +) { + try { + // For CommonJS environments, use require to ensure synchronous loading + if (typeof require === 'function') { + const textEncoding = require('./utils/textEncoding.js') + if ( + textEncoding && + typeof textEncoding.applyTensorFlowPatch === 'function' + ) { + textEncoding.applyTensorFlowPatch() + console.log( + 'Applied TensorFlow.js patch via CommonJS require in setup.ts' + ) + } + } + } catch (e) { + console.warn('Failed to apply TensorFlow.js patch via require:', e) + // Continue to the import-based approach + } +} + +// Also import normally for ES modules environments import { applyTensorFlowPatch } from './utils/textEncoding.js' // Apply the TensorFlow.js platform patch if needed +// This will be a no-op if the patch was already applied via require above applyTensorFlowPatch() +console.log( + 'Applied or verified TensorFlow.js patch via ES modules in setup.ts' +) diff --git a/src/unified.ts b/src/unified.ts index e00d8527..be00581f 100644 --- a/src/unified.ts +++ b/src/unified.ts @@ -4,6 +4,18 @@ * Environment detection is handled here and made available to all components */ +// CRITICAL: The TensorFlow.js environment patch is now centralized in setup.ts +// We import setup.ts below which applies the necessary patches + +// CRITICAL: Import setup.js first to ensure TensorFlow.js environment patching +// This MUST be the first import to prevent race conditions with TensorFlow.js initialization +// Moving or removing this import will cause errors like "TextEncoder is not a constructor" +// when the package is used in Node.js environments +// +// The setup.js file applies a patch that ensures TextEncoder/TextDecoder are properly +// available to TensorFlow.js before it initializes its platform detection +import './setup.js' + // Export environment information export const environment = { isBrowser: typeof window !== 'undefined', diff --git a/src/utils/distance.ts b/src/utils/distance.ts index 90917dd2..dc3a8e2c 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -145,14 +145,36 @@ export async function calculateDistancesBatch( // In worker context, use the importTensorFlow function tf = await self.importTensorFlow() } else { - // Dynamically import TensorFlow.js core module and backends - tf = await import('@tensorflow/tfjs-core') + // CRITICAL: First, directly import the setup module to ensure the TensorFlow.js patch is applied + // This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded + try { + // In Node.js environment, use require() which is synchronous + if (typeof require !== 'undefined') { + // First, require the setup module to apply the patch + require('../setup.js') - // Import CPU backend - await import('@tensorflow/tfjs-backend-cpu') + // Now load TensorFlow.js core module + tf = require('@tensorflow/tfjs-core') - // Set CPU as the backend - await tf.setBackend('cpu') + // Load CPU backend + require('@tensorflow/tfjs-backend-cpu') + + // Set CPU as the backend + tf.setBackend('cpu') + } else { + // In browser or other environments without require(), use dynamic imports + // First, dynamically import the setup module to apply the patch + await import('../setup.js') + + // Now load TensorFlow.js core module + tf = await import('@tensorflow/tfjs-core') + await import('@tensorflow/tfjs-backend-cpu') + await tf.setBackend('cpu') + } + } catch (error) { + console.error('Failed to initialize TensorFlow.js:', error) + throw error + } } // Convert vectors to tensors diff --git a/src/utils/embedding.ts b/src/utils/embedding.ts index b0ca71de..4648d35a 100644 --- a/src/utils/embedding.ts +++ b/src/utils/embedding.ts @@ -23,6 +23,10 @@ export class UniversalSentenceEncoder implements EmbeddingModel { /** * Add polyfills and patches for TensorFlow.js compatibility * This addresses issues with TensorFlow.js in Node.js environments + * + * Note: The main TensorFlow.js patching is now centralized in textEncoding.ts + * and applied through setup.ts. This method only adds additional utility functions + * that might be needed by TensorFlow.js. */ private addNodeCompatibilityPolyfills(): void { // Only apply in Node.js environment @@ -38,82 +42,30 @@ export class UniversalSentenceEncoder implements EmbeddingModel { // This fixes the "Cannot read properties of undefined (reading 'isFloat32Array')" error if (typeof global !== 'undefined') { try { - // Define a custom PlatformNode class - class PlatformNode { - util: any - textEncoder: TextEncoder - textDecoder: TextDecoder + // Ensure the util object exists + if (!global.util) { + global.util = {} + } - constructor() { - // Create a util object with necessary methods - this.util = { - // Add isFloat32Array and isTypedArray directly to util - isFloat32Array: (arr: any) => { - return !!( - arr instanceof Float32Array || - (arr && - Object.prototype.toString.call(arr) === - '[object Float32Array]') - ) - }, - isTypedArray: (arr: any) => { - return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)) - }, - // Use native TextEncoder and TextDecoder - TextEncoder: TextEncoder, - TextDecoder: TextDecoder - } - - // Initialize encoders using native constructors - this.textEncoder = new TextEncoder() - this.textDecoder = new TextDecoder() - } - - // Define isFloat32Array directly on the instance - isFloat32Array(arr: any) { + // Add isFloat32Array method if it doesn't exist + if (!global.util.isFloat32Array) { + global.util.isFloat32Array = (obj: any) => { return !!( - arr instanceof Float32Array || - (arr && - Object.prototype.toString.call(arr) === '[object Float32Array]') + obj instanceof Float32Array || + (obj && + Object.prototype.toString.call(obj) === '[object Float32Array]') ) } + } - // Define isTypedArray directly on the instance - isTypedArray(arr: any) { - return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)) + // Add isTypedArray method if it doesn't exist + if (!global.util.isTypedArray) { + global.util.isTypedArray = (obj: any) => { + return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView)) } } - - // Assign the PlatformNode class to the global object - ;(global as any).PlatformNode = PlatformNode - - // Also create an instance and assign it to global.platformNode - ;(global as any).platformNode = new PlatformNode() } catch (error) { - console.warn('Failed to define global PlatformNode class:', error) - } - - // Ensure the util object exists - if (!global.util) { - global.util = {} - } - - // Add isFloat32Array method if it doesn't exist - if (!global.util.isFloat32Array) { - global.util.isFloat32Array = (obj: any) => { - return !!( - obj instanceof Float32Array || - (obj && - Object.prototype.toString.call(obj) === '[object Float32Array]') - ) - } - } - - // Add isTypedArray method if it doesn't exist - if (!global.util.isTypedArray) { - global.util.isTypedArray = (obj: any) => { - return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView)) - } + console.warn('Failed to add utility polyfills:', error) } } } @@ -145,36 +97,95 @@ export class UniversalSentenceEncoder implements EmbeddingModel { // TensorFlow.js will use its default EPSILON value - // Dynamically import TensorFlow.js core module and backends - // Use type assertions to tell TypeScript these modules exist - this.tf = await import('@tensorflow/tfjs-core') - - // Import CPU backend (always needed as fallback) - await import('@tensorflow/tfjs-backend-cpu') - - // Try to import WebGL backend for GPU acceleration in browser environments + // CRITICAL: First, directly import the setup module to ensure the TensorFlow.js patch is applied + // This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded try { - if (typeof window !== 'undefined') { - await import('@tensorflow/tfjs-backend-webgl') - // Check if WebGL is available using setBackend instead of findBackend - try { - if (this.tf.setBackend) { - await this.tf.setBackend('webgl') - this.backend = 'webgl' - console.log('Using WebGL backend for TensorFlow.js') - } else { + // In Node.js environment, use require() which is synchronous + if (typeof require !== 'undefined') { + // First, require the setup module to apply the patch + require('../setup.js') + + // Now load TensorFlow.js core module + this.tf = require('@tensorflow/tfjs-core') + + // Load CPU backend (always needed as fallback) + require('@tensorflow/tfjs-backend-cpu') + + // Try to load WebGL backend for GPU acceleration in browser environments + if (typeof window !== 'undefined') { + try { + require('@tensorflow/tfjs-backend-webgl') + // Check if WebGL is available + if (this.tf.setBackend) { + this.tf.setBackend('webgl') + this.backend = 'webgl' + console.log('Using WebGL backend for TensorFlow.js') + } else { + console.warn( + 'tf.setBackend is not available, falling back to CPU' + ) + } + } catch (e) { console.warn( - 'tf.setBackend is not available, falling back to CPU' + 'WebGL backend not available, falling back to CPU:', + e ) + this.backend = 'cpu' } - } catch (e) { - console.warn('WebGL backend not available, falling back to CPU:', e) + } + + // Load Universal Sentence Encoder + this.use = require('@tensorflow-models/universal-sentence-encoder') + } else { + // In browser or other environments without require(), use dynamic imports + // First, dynamically import the setup module to apply the patch + await import('../setup.js') + + // Now load TensorFlow.js core module + this.tf = await import('@tensorflow/tfjs-core') + + // Import CPU backend (always needed as fallback) + await import('@tensorflow/tfjs-backend-cpu') + + // Try to import WebGL backend for GPU acceleration in browser environments + try { + if (typeof window !== 'undefined') { + await import('@tensorflow/tfjs-backend-webgl') + // Check if WebGL is available + try { + if (this.tf.setBackend) { + await this.tf.setBackend('webgl') + this.backend = 'webgl' + console.log('Using WebGL backend for TensorFlow.js') + } else { + console.warn( + 'tf.setBackend is not available, falling back to CPU' + ) + } + } catch (e) { + console.warn( + 'WebGL backend not available, falling back to CPU:', + e + ) + this.backend = 'cpu' + } + } + } catch (error) { + console.warn( + 'WebGL backend not available, falling back to CPU:', + error + ) this.backend = 'cpu' } + + // Load Universal Sentence Encoder + this.use = await import( + '@tensorflow-models/universal-sentence-encoder' + ) } } catch (error) { - console.warn('WebGL backend not available, falling back to CPU:', error) - this.backend = 'cpu' + console.error('Failed to initialize TensorFlow.js:', error) + throw error } // Set the backend @@ -182,8 +193,6 @@ export class UniversalSentenceEncoder implements EmbeddingModel { await this.tf.setBackend(this.backend) } - this.use = await import('@tensorflow-models/universal-sentence-encoder') - // Log the module structure to help with debugging console.log( 'Universal Sentence Encoder module structure in main thread:', diff --git a/src/utils/textEncoding.ts b/src/utils/textEncoding.ts index 3bab39cc..ff13d633 100644 --- a/src/utils/textEncoding.ts +++ b/src/utils/textEncoding.ts @@ -1,94 +1,224 @@ +// In: @soulcraft/brainy/src/utils/textEncoding.ts + /** - * Unified Text Encoding Utilities + * Checks if the code is running in a Node.js environment. + */ +function isNode(): boolean { + return ( + typeof process !== 'undefined' && + process.versions != null && + process.versions.node != null + ) +} + +/** + * Global flag to track if TensorFlow.js has been initialized + * This helps prevent multiple registrations of the same kernels + */ +const TENSORFLOW_INITIALIZED = Symbol('TENSORFLOW_INITIALIZED') + +/** + * Flag to track if the patch has been applied + * This prevents multiple applications of the patch + */ +let patchApplied = false + +/** + * CRITICAL: Applies a compatibility patch for TensorFlow.js when running in a modern + * Node.js ES Module environment. This must be called before any TensorFlow.js + * modules are imported. * - * This module provides a consistent way to handle text encoding/decoding across all environments - * using the native TextEncoder/TextDecoder APIs. - */ - -/** - * Get a text encoder that works in the current environment - * @returns A TextEncoder instance - */ -export function getTextEncoder(): TextEncoder { - return new TextEncoder() -} - -/** - * Get a text decoder that works in the current environment - * @returns A TextDecoder instance - */ -export function getTextDecoder(): TextDecoder { - return new TextDecoder() -} - -/** - * Apply the TensorFlow.js platform patch if needed - * This function patches the global object to provide a PlatformNode class - * that uses native TextEncoder/TextDecoder + * This function prevents the "TextEncoder is not a constructor" error by preemptively + * creating a compliant PlatformNode class with proper TextEncoder/TextDecoder support + * and placing it on the global object where TensorFlow.js expects to find it. + * + * The race condition occurs because TensorFlow.js's platform detection might run + * before the necessary global objects are properly initialized in certain Node.js + * environments, particularly when the package is being used by other applications. + * + * This function is called from setup.ts, which must be the first import in unified.ts + * to ensure the patch is applied before any TensorFlow.js code is executed. + * + * It also applies a patch to prevent duplicate kernel registrations when TensorFlow.js + * is imported multiple times. */ export function applyTensorFlowPatch(): void { - try { - // Define a custom Platform class that works in both Node.js and browser environments - class Platform { - util: any - textEncoder: TextEncoder - textDecoder: TextDecoder + // Prevent multiple applications of the patch + if (patchApplied) { + return + } - constructor() { - // Create a util object with necessary methods and constructors - this.util = { - // Use native TextEncoder and TextDecoder - TextEncoder: globalThis.TextEncoder || TextEncoder, - TextDecoder: globalThis.TextDecoder || TextDecoder + if (!isNode()) { + return // Patch is only for Node.js + } + + // In modern Node.js with ES Modules, TensorFlow.js can fail during its + // initial platform detection. This patch preempts that logic by creating + // a compliant "Platform" class that uses the standard global TextEncoder + // and placing it on the global object where TensorFlow.js expects to find it. + try { + // Ensure TextEncoder and TextDecoder are available + const nodeUtil = require('util') + const TextEncoderPolyfill = nodeUtil.TextEncoder || global.TextEncoder + const TextDecoderPolyfill = nodeUtil.TextDecoder || global.TextDecoder + + if (!TextEncoderPolyfill || !TextDecoderPolyfill) { + console.warn( + 'Brainy: TextEncoder or TextDecoder not available, attempting to polyfill' + ) + + // If still not available, try to use a simple polyfill + if (!TextEncoderPolyfill) { + class SimpleTextEncoder { + encode(input: string): Uint8Array { + const buf = Buffer.from(input, 'utf8') + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) + } } - // Initialize using native constructors directly - this.textEncoder = new (globalThis.TextEncoder || TextEncoder)() - this.textDecoder = new (globalThis.TextDecoder || TextDecoder)() + global.TextEncoder = SimpleTextEncoder } - // Define isFloat32Array directly on the instance - isFloat32Array(arr: any) { - return !!( + if (!TextDecoderPolyfill) { + class SimpleTextDecoder { + decode(input?: Uint8Array): string { + if (!input) return '' + return Buffer.from( + input.buffer, + input.byteOffset, + input.byteLength + ).toString('utf8') + } + } + + global.TextDecoder = SimpleTextDecoder + } + } else { + // Ensure they're available globally + global.TextEncoder = TextEncoderPolyfill + global.TextDecoder = TextDecoderPolyfill + } + + // Create a PlatformNode implementation that uses the polyfilled TextEncoder/TextDecoder + class BrainyPlatformNode { + // Use the polyfilled TextEncoder/TextDecoder + readonly util = { + TextEncoder: global.TextEncoder, + TextDecoder: global.TextDecoder, + + // Add utility functions that TensorFlow.js might need + isTypedArray: (arr: any): boolean => { + return ArrayBuffer.isView(arr) && !(arr instanceof DataView) + }, + + isFloat32Array: (arr: any): boolean => { + return ( + arr instanceof Float32Array || + (arr && + Object.prototype.toString.call(arr) === '[object Float32Array]') + ) + } + } + + // Create instances of the encoder/decoder + readonly textEncoder: any + readonly textDecoder: any + + constructor() { + try { + // Initialize encoders using constructors + this.textEncoder = new global.TextEncoder() + this.textDecoder = new global.TextDecoder() + } catch (e) { + console.warn( + 'Brainy: Error creating TextEncoder/TextDecoder instances:', + e + ) + // Provide fallback implementations if instantiation fails + this.textEncoder = { + encode: (input: string): Uint8Array => { + const buf = Buffer.from(input, 'utf8') + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) + } + } + this.textDecoder = { + decode: (input?: Uint8Array): string => { + if (!input) return '' + return Buffer.from( + input.buffer, + input.byteOffset, + input.byteLength + ).toString('utf8') + } + } + } + } + + isTypedArray(arr: any): arr is Float32Array | Int32Array | Uint8Array { + return ArrayBuffer.isView(arr) && !(arr instanceof DataView) + } + + isFloat32Array(arr: any): arr is Float32Array { + return ( arr instanceof Float32Array || (arr && Object.prototype.toString.call(arr) === '[object Float32Array]') ) } - - // Define isTypedArray directly on the instance - isTypedArray(arr: any) { - return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)) - } } - // Get the global object in a way that works in both Node.js and browser - const globalObj = - typeof global !== 'undefined' - ? global - : typeof window !== 'undefined' - ? window - : typeof self !== 'undefined' - ? self - : {} + // Assign the custom platform class to the global scope. + // TensorFlow.js specifically looks for `PlatformNode`. + global.PlatformNode = BrainyPlatformNode - // Only apply in Node.js environment - if ( - typeof process !== 'undefined' && - process.versions && - process.versions.node - ) { - // Assign the Platform class to the global object as PlatformNode for Node.js - ;(globalObj as any).PlatformNode = Platform - // Also create an instance and assign it to global.platformNode (lowercase p) - ;(globalObj as any).platformNode = new Platform() - } else if (typeof window !== 'undefined' || typeof self !== 'undefined') { - // In browser environments, we might need to provide similar functionality - // but we'll use a different name to avoid conflicts - ;(globalObj as any).PlatformBrowser = Platform - ;(globalObj as any).platformBrowser = new Platform() + // Also create an instance and assign it to global.platformNode (lowercase p) + // This is needed for some TensorFlow.js versions + global.platformNode = new BrainyPlatformNode() + + // Set up a global flag to track TensorFlow.js initialization + global[TENSORFLOW_INITIALIZED] = false + + // Monkey patch the registerKernel function to prevent duplicate registrations + // This will be applied when TensorFlow.js is imported + const originalRegisterKernel = global.registerKernel + if (!originalRegisterKernel) { + // Set up a handler to intercept the registerKernel function when it's defined + Object.defineProperty(global, 'registerKernel', { + set: function (newRegisterKernel) { + // Replace the setter with our patched version + Object.defineProperty(global, 'registerKernel', { + value: function (kernel: any) { + // Check if this kernel is already registered + const kernelName = kernel.kernelName + const backendName = kernel.backendName + const key = `${kernelName}_${backendName}` + + // Use a global registry to track registered kernels + if (!global.__REGISTERED_KERNELS__) { + global.__REGISTERED_KERNELS__ = new Set() + } + + // If this kernel is already registered, skip it + if (global.__REGISTERED_KERNELS__.has(key)) { + return + } + + // Otherwise, register it and add it to our registry + global.__REGISTERED_KERNELS__.add(key) + return newRegisterKernel(kernel) + }, + configurable: true, + writable: true + }) + }, + configurable: true + }) } + + // Mark the patch as applied + patchApplied = true + console.log('Brainy: Successfully applied TensorFlow.js platform patch') } catch (error) { - console.warn('Failed to apply TensorFlow.js platform patch:', error) + console.warn('Brainy: Failed to apply TensorFlow.js platform patch:', error) } } diff --git a/test-fallback-function.js b/test-fallback-function.js deleted file mode 100644 index 2eda4d84..00000000 --- a/test-fallback-function.js +++ /dev/null @@ -1,59 +0,0 @@ -// Test script to verify that the function string format works with the fallback mechanism -import { executeInThread } from './dist/unified.js' - -// Define a compute-intensive function using a named function declaration -// followed by a statement that returns the function -const computeIntensiveFunction = ` - // Define a named function - function computeTask(data) { - console.log('Worker/Fallback: Starting computation...'); - - // Simulate a compute-intensive task - const start = Date.now(); - let result = 0; - for (let i = 0; i < data.iterations; i++) { - result += Math.sqrt(i) * Math.sin(i); - } - - const duration = Date.now() - start; - console.log('Worker/Fallback: Computation completed in ' + duration + 'ms'); - - return { - result, - duration, - iterations: data.iterations - }; - } - - // Return the function - computeTask; - ` - -// Test with different environments -async function runTests() { - try { - console.log('Testing executeInThread with fallback...') - - // Disable Web Workers to force fallback - const originalWorker = globalThis.Worker - globalThis.Worker = function() { - throw new Error('Worker constructor disabled for testing') - } - - try { - // Execute the function in fallback mode - const result = await executeInThread(computeIntensiveFunction, { - iterations: 1000000 - }) - console.log('Fallback result:', result) - console.log('Test passed!') - } finally { - // Restore Web Workers - globalThis.Worker = originalWorker - } - } catch (error) { - console.error('Test failed:', error) - } -} - -runTests() diff --git a/test-fallback-simple.js b/test-fallback-simple.js deleted file mode 100644 index f54c2a30..00000000 --- a/test-fallback-simple.js +++ /dev/null @@ -1,52 +0,0 @@ -// Test script to verify that the function string format works with the fallback mechanism -import { executeInThread } from './dist/unified.js' - -// Define a compute-intensive function using a simple anonymous function expression -const computeIntensiveFunction = `function(data) { - console.log('Worker/Fallback: Starting computation...'); - - // Simulate a compute-intensive task - const start = Date.now(); - let result = 0; - for (let i = 0; i < data.iterations; i++) { - result += Math.sqrt(i) * Math.sin(i); - } - - const duration = Date.now() - start; - console.log('Worker/Fallback: Computation completed in ' + duration + 'ms'); - - return { - result, - duration, - iterations: data.iterations - }; -}` - -// Test with different environments -async function runTests() { - try { - console.log('Testing executeInThread with fallback...') - - // Disable Web Workers to force fallback - const originalWorker = globalThis.Worker - globalThis.Worker = function() { - throw new Error('Worker constructor disabled for testing') - } - - try { - // Execute the function in fallback mode - const result = await executeInThread(computeIntensiveFunction, { - iterations: 1000000 - }) - console.log('Fallback result:', result) - console.log('Test passed!') - } finally { - // Restore Web Workers - globalThis.Worker = originalWorker - } - } catch (error) { - console.error('Test failed:', error) - } -} - -runTests() diff --git a/test-fix.js b/test-fix.js deleted file mode 100644 index af000f6f..00000000 --- a/test-fix.js +++ /dev/null @@ -1,47 +0,0 @@ -// Test script to verify the TextEncoder fix -import { applyTensorFlowPatch } from './dist/utils/textEncoding.js' - -console.log('Testing TextEncoder fix...') - -// Apply the TensorFlow.js platform patch -applyTensorFlowPatch() - -// Check if PlatformNode is defined in the global object -if (typeof global.PlatformNode === 'function') { - console.log('PlatformNode is defined in the global object') - - // Create an instance of PlatformNode - try { - const platform = new global.PlatformNode() - console.log('Successfully created PlatformNode instance') - - // Check if textEncoder is defined - if (platform.textEncoder) { - console.log('textEncoder is defined') - - // Test encoding a string - const testString = 'Hello, world! 👋' - const encoded = platform.textEncoder.encode(testString) - console.log(`Successfully encoded string: ${testString}`) - console.log(`Encoded: [${encoded}]`) - - // Test decoding - const decoded = platform.textDecoder.decode(encoded) - console.log(`Successfully decoded back to: ${decoded}`) - - if (testString === decoded) { - console.log('✅ TextEncoder/TextDecoder test passed!') - } else { - console.error('❌ TextEncoder/TextDecoder test failed!') - } - } else { - console.error('textEncoder is not defined in the platform instance') - } - } catch (error) { - console.error('Error creating PlatformNode instance:', error) - } -} else { - console.error('PlatformNode is not defined in the global object') -} - -console.log('Test completed') diff --git a/test-tensorflow-textencoder.js b/test-tensorflow-textencoder.js deleted file mode 100644 index 498f2086..00000000 --- a/test-tensorflow-textencoder.js +++ /dev/null @@ -1,103 +0,0 @@ -// Test script to verify TensorFlow.js and TextEncoder functionality in Node.js environment -import * as tf from '@tensorflow/tfjs' -import '@tensorflow/tfjs-backend-cpu' -import { TextEncoder, TextDecoder } from 'util' - -// Implement the necessary functions directly -function applyTensorFlowPatch() { - // This is a simplified version of the patch - console.log('Applying TensorFlow patch directly in test file') - return true -} - -function getTextEncoder() { - return new TextEncoder() -} - -function getTextDecoder() { - return new TextDecoder() -} - -async function testTensorFlowAndTextEncoder() { - console.log('Testing TensorFlow.js and TextEncoder in Node.js environment...') - - try { - // Apply TensorFlow patch for TextEncoder compatibility - applyTensorFlowPatch() - console.log('TensorFlow patch applied successfully') - - // Test TextEncoder - console.log('\n--- Testing TextEncoder ---') - const encoder = getTextEncoder() - const decoder = getTextDecoder() - - const testString = 'Hello, world! 👋' - console.log(`Original string: "${testString}"`) - - const encoded = encoder.encode(testString) - console.log(`Encoded: [${encoded}]`) - - const decoded = decoder.decode(encoded) - console.log(`Decoded: "${decoded}"`) - - if (testString === decoded) { - console.log('✅ TextEncoder/TextDecoder test passed!') - } else { - console.error('❌ TextEncoder/TextDecoder test failed!') - return false - } - - // Test TensorFlow.js - console.log('\n--- Testing TensorFlow.js ---') - - // Create a simple tensor - const tensor = tf.tensor2d([ - [1, 2], - [3, 4] - ]) - console.log('Created tensor:') - tensor.print() - - // Perform a simple operation - const result = tensor.add(tf.scalar(1)) - console.log('Result of adding 1:') - result.print() - - // Check the values - const values = await result.array() - const expected = [ - [2, 3], - [4, 5] - ] - - console.log('Result values:', values) - console.log('Expected values:', expected) - - // Compare values - const match = JSON.stringify(values) === JSON.stringify(expected) - if (match) { - console.log('✅ TensorFlow.js test passed!') - } else { - console.error('❌ TensorFlow.js test failed!') - return false - } - - console.log('\nAll tests passed successfully!') - return true - } catch (error) { - console.error('Error during test:', error) - return false - } -} - -// Run the test -testTensorFlowAndTextEncoder().then((success) => { - if (success) { - console.log( - 'TensorFlow.js and TextEncoder verification completed successfully!' - ) - } else { - console.error('TensorFlow.js and TextEncoder verification failed!') - process.exit(1) - } -}) diff --git a/test-unified-encoding.js b/test-unified-encoding.js deleted file mode 100644 index 107151b8..00000000 --- a/test-unified-encoding.js +++ /dev/null @@ -1,39 +0,0 @@ -// Test script to verify the unified text encoding approach works correctly -import { BrainyData } from './dist/unified.js' - -async function testUnifiedEncoding() { - console.log( - 'Testing unified text encoding approach in Node.js environment...' - ) - - try { - // Initialize BrainyData which should trigger the PlatformNode constructor - console.log('Creating BrainyData instance...') - const db = new BrainyData() - - // Initialize the database - console.log('Initializing database...') - await db.init() - - console.log('Test successful! Unified text encoding is working correctly.') - - // Get database status to verify everything is working - const status = await db.status() - console.log('Database status:', status) - - return true - } catch (error) { - console.error('Error during test:', error) - return false - } -} - -// Run the test -testUnifiedEncoding().then((success) => { - if (success) { - console.log('Unified text encoding verification completed successfully!') - } else { - console.error('Unified text encoding verification failed!') - process.exit(1) - } -}) diff --git a/test-worker-utils.js b/test-worker-utils.js deleted file mode 100644 index 4405391a..00000000 --- a/test-worker-utils.js +++ /dev/null @@ -1,24 +0,0 @@ -// Test script to verify that the workerUtils functions work correctly after removing eval -import { executeInThread } from './dist/unified.js' - -// Test function to execute in a thread -const testFunction = `function(args) { - return "Hello from " + args.name; -}` - -// Test with different environments -async function runTests() { - try { - console.log('Testing executeInThread...') - const result = await executeInThread(testFunction, { - name: 'Worker Thread' - }) - console.log('Result:', result) - - console.log('All tests passed!') - } catch (error) { - console.error('Test failed:', error) - } -} - -runTests()