**refactor(tests): consolidate and replace outdated environment test scripts**
- Removed obsolete scripts: `test-all-environments.js`, `test-fallback-function.js`, `test-fallback-simple.js`, `test-fix.js`, `test-tensorflow-textencoder.js`, `test-unified-encoding.js`, and `test-worker-utils.js`. - Introduced `scripts/comprehensive-test.js` as a unified testing script covering all environments: Browser, Node.js, and CLI. - Added `examples/cli-wrapper-example.js` to demonstrate a proper CLI implementation with TensorFlow.js initialization. This refactor simplifies the testing structure by consolidating redundant scripts into a single comprehensive script while ensuring robust cross-environment coverage.
This commit is contained in:
parent
3b85ea46e1
commit
f5e4b8b93e
20 changed files with 1253 additions and 808 deletions
78
TENSORFLOW_NODEJS.md
Normal file
78
TENSORFLOW_NODEJS.md
Normal file
|
|
@ -0,0 +1,78 @@
|
||||||
|
# Using TensorFlow.js with Brainy in Node.js Environments
|
||||||
|
|
||||||
|
This document provides guidance on resolving TensorFlow.js compatibility issues when using Brainy in Node.js environments, particularly with ES modules.
|
||||||
|
|
||||||
|
## Common Issues
|
||||||
|
|
||||||
|
When using Brainy with TensorFlow.js in Node.js environments, you might encounter errors like:
|
||||||
|
|
||||||
|
```
|
||||||
|
TypeError: this.util.TextEncoder is not a constructor
|
||||||
|
```
|
||||||
|
|
||||||
|
This occurs due to how TensorFlow.js initializes its platform detection in ES modules environments.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
Brainy includes a built-in patch to address these issues. The patch is automatically applied when you import Brainy, but in some complex project setups, you might need to take additional steps.
|
||||||
|
|
||||||
|
### Option 1: Import the Setup Module First (Recommended)
|
||||||
|
|
||||||
|
For the most reliable solution, explicitly import Brainy's setup module before any other imports that might use TensorFlow.js:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Import the setup module first to apply TensorFlow.js patches
|
||||||
|
import '@soulcraft/brainy/setup';
|
||||||
|
|
||||||
|
// Then import and use Brainy or TensorFlow.js
|
||||||
|
import { BrainyData } from '@soulcraft/brainy';
|
||||||
|
// ... your code here
|
||||||
|
```
|
||||||
|
|
||||||
|
### Option 2: Apply the Patch Directly
|
||||||
|
|
||||||
|
If you need more control, you can directly apply the patch:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Import and apply the patch directly
|
||||||
|
import { applyTensorFlowPatch } from '@soulcraft/brainy/utils/textEncoding';
|
||||||
|
applyTensorFlowPatch();
|
||||||
|
|
||||||
|
// Then import and use TensorFlow.js
|
||||||
|
import * as tf from '@tensorflow/tfjs';
|
||||||
|
// ... your code here
|
||||||
|
```
|
||||||
|
|
||||||
|
### Option 3: For CommonJS Environments
|
||||||
|
|
||||||
|
If you're using CommonJS modules:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Apply the patch first
|
||||||
|
require('@soulcraft/brainy/dist/setup.js');
|
||||||
|
|
||||||
|
// Then require TensorFlow.js or Brainy
|
||||||
|
const brainy = require('@soulcraft/brainy');
|
||||||
|
// ... your code here
|
||||||
|
```
|
||||||
|
|
||||||
|
## How It Works
|
||||||
|
|
||||||
|
The patch works by:
|
||||||
|
|
||||||
|
1. Ensuring TextEncoder and TextDecoder are properly available in the global scope
|
||||||
|
2. Creating a custom PlatformNode implementation that TensorFlow.js will use
|
||||||
|
3. Applying the patch before any TensorFlow.js code is executed
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
If you still encounter issues:
|
||||||
|
|
||||||
|
1. Make sure the setup module is imported before any other modules that might use TensorFlow.js
|
||||||
|
2. Check your bundler configuration to ensure it's not removing the patch code (it's marked as having side effects)
|
||||||
|
3. Try using the CommonJS approach if you're having issues with ES modules
|
||||||
|
4. If using a bundler like webpack or rollup, ensure it's configured to handle Node.js built-ins properly
|
||||||
|
|
||||||
|
## Need More Help?
|
||||||
|
|
||||||
|
If you continue to experience issues, please open an issue on our GitHub repository with details about your environment and how you're using Brainy.
|
||||||
|
|
@ -7,14 +7,54 @@
|
||||||
* are properly passed to the CLI when invoked through the globally installed package.
|
* are properly passed to the CLI when invoked through the globally installed package.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// CRITICAL: Apply TensorFlow.js environment patch before importing any other modules
|
||||||
|
// This prevents the "TextEncoder is not a constructor" error in Node.js environments
|
||||||
|
// by ensuring the global.PlatformNode class is defined before TensorFlow.js loads
|
||||||
|
function applyTensorFlowPatch() {
|
||||||
|
try {
|
||||||
|
// Define a custom Platform class that works in Node.js environments
|
||||||
|
class Platform {
|
||||||
|
constructor() {
|
||||||
|
// Create a util object with necessary methods and constructors
|
||||||
|
this.util = {
|
||||||
|
// Use native TextEncoder and TextDecoder constructors
|
||||||
|
TextEncoder: global.TextEncoder || TextEncoder,
|
||||||
|
TextDecoder: global.TextDecoder || TextDecoder
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize using native constructors directly
|
||||||
|
this.textEncoder = new TextEncoder()
|
||||||
|
this.textDecoder = new TextDecoder()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Define isTypedArray directly on the instance
|
||||||
|
isTypedArray(arr) {
|
||||||
|
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assign the Platform class to the global object as PlatformNode
|
||||||
|
global.PlatformNode = Platform
|
||||||
|
// Also create an instance and assign it to global.platformNode (lowercase p)
|
||||||
|
global.platformNode = new Platform()
|
||||||
|
|
||||||
|
console.log('Applied TensorFlow.js platform patch in CLI wrapper')
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('Failed to apply TensorFlow.js platform patch:', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply the patch immediately
|
||||||
|
applyTensorFlowPatch()
|
||||||
|
|
||||||
import { spawn } from 'child_process'
|
import { spawn } from 'child_process'
|
||||||
import { fileURLToPath } from 'url'
|
import { fileURLToPath } from 'url'
|
||||||
import { dirname, join } from 'path'
|
import { dirname, join } from 'path'
|
||||||
import fs from 'fs'
|
import fs from 'fs'
|
||||||
|
|
||||||
// Node.js v23+ compatibility patches were previously applied here,
|
// Node.js v24+ compatibility patches are now applied above,
|
||||||
// but these patches are no longer necessary with current TensorFlow.js versions.
|
// before any imports, to ensure TensorFlow.js can correctly
|
||||||
// TensorFlow.js now works correctly with Node.js 24+ without any special handling.
|
// detect and use the TextEncoder/TextDecoder in the environment.
|
||||||
|
|
||||||
// Get the directory of the current module
|
// Get the directory of the current module
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
const __filename = fileURLToPath(import.meta.url)
|
||||||
|
|
|
||||||
|
|
@ -36,15 +36,19 @@ export function applyTensorFlowPatch(): void {
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
// Create a util object with necessary methods and constructors
|
// Create a util object with necessary methods and constructors
|
||||||
|
// Store the actual constructor functions, not just references
|
||||||
|
const TextEncoderConstructor = globalThis.TextEncoder || TextEncoder
|
||||||
|
const TextDecoderConstructor = globalThis.TextDecoder || TextDecoder
|
||||||
|
|
||||||
this.util = {
|
this.util = {
|
||||||
// Use native TextEncoder and TextDecoder
|
// Use native TextEncoder and TextDecoder constructors
|
||||||
TextEncoder: globalThis.TextEncoder || TextEncoder,
|
TextEncoder: TextEncoderConstructor,
|
||||||
TextDecoder: globalThis.TextDecoder || TextDecoder
|
TextDecoder: TextDecoderConstructor
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize using native constructors directly
|
// Initialize using native constructors directly
|
||||||
this.textEncoder = new (globalThis.TextEncoder || TextEncoder)()
|
this.textEncoder = new TextEncoderConstructor()
|
||||||
this.textDecoder = new (globalThis.TextDecoder || TextDecoder)()
|
this.textDecoder = new TextDecoderConstructor()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Define isFloat32Array directly on the instance
|
// Define isFloat32Array directly on the instance
|
||||||
|
|
|
||||||
68
examples/cli-wrapper-example.js
Normal file
68
examples/cli-wrapper-example.js
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Example CLI wrapper for Brainy that properly handles TensorFlow.js initialization
|
||||||
|
*
|
||||||
|
* This example demonstrates how to create a CLI tool that uses Brainy
|
||||||
|
* while ensuring TensorFlow.js is properly initialized in Node.js environments.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* node cli-wrapper-example.js
|
||||||
|
*/
|
||||||
|
|
||||||
|
// CRITICAL: Apply the TensorFlow.js patch before any other imports
|
||||||
|
// This prevents the "TextEncoder is not a constructor" error
|
||||||
|
try {
|
||||||
|
// For CommonJS environments
|
||||||
|
if (typeof require === 'function') {
|
||||||
|
// First require the setup module to apply the patch
|
||||||
|
require('../dist/setup.js');
|
||||||
|
console.log('Applied TensorFlow.js patch via CommonJS require');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('Failed to apply TensorFlow.js patch via require:', error);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ES Modules approach - this will be used if the above fails or if using ES modules
|
||||||
|
import('../dist/setup.js')
|
||||||
|
.then(() => {
|
||||||
|
console.log('Applied TensorFlow.js patch via ES modules import');
|
||||||
|
return import('../dist/unified.js');
|
||||||
|
})
|
||||||
|
.then((brainy) => {
|
||||||
|
// Now it's safe to use Brainy and TensorFlow.js
|
||||||
|
console.log('Brainy loaded successfully');
|
||||||
|
|
||||||
|
// Example: Create a BrainyData instance
|
||||||
|
const db = new brainy.BrainyData({
|
||||||
|
name: 'cli-example',
|
||||||
|
storage: 'memory'
|
||||||
|
});
|
||||||
|
|
||||||
|
// Example: Add some data
|
||||||
|
db.addItem('Hello world', { id: '1', metadata: { type: 'greeting' } })
|
||||||
|
.then(() => {
|
||||||
|
console.log('Added item to database');
|
||||||
|
|
||||||
|
// Example: Search for similar items
|
||||||
|
return db.search('Hello', 1);
|
||||||
|
})
|
||||||
|
.then((results) => {
|
||||||
|
console.log('Search results:', results);
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
return db.close();
|
||||||
|
})
|
||||||
|
.then(() => {
|
||||||
|
console.log('Database closed');
|
||||||
|
process.exit(0);
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.error('Error in Brainy operations:', error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.error('Failed to load Brainy:', error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
75
node-test.js
Normal file
75
node-test.js
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
// Node.js test script for @soulcraft/brainy
|
||||||
|
|
||||||
|
// CRITICAL: First, directly apply the TensorFlow.js patch
|
||||||
|
// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
|
||||||
|
import { TextEncoder, TextDecoder } from 'util'
|
||||||
|
|
||||||
|
// Make TextEncoder and TextDecoder available globally
|
||||||
|
if (typeof global !== 'undefined') {
|
||||||
|
global.TextEncoder = TextEncoder
|
||||||
|
global.TextDecoder = TextDecoder
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import the library
|
||||||
|
import * as brainy from './dist/unified.js'
|
||||||
|
|
||||||
|
async function runNodeTest() {
|
||||||
|
console.log('\n=== Testing @soulcraft/brainy in Node.js environment ===\n')
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test environment detection
|
||||||
|
console.log('Environment Detection:')
|
||||||
|
console.log(`- isBrowser: ${brainy.isBrowser()}`)
|
||||||
|
console.log(`- isNode: ${brainy.isNode()}`)
|
||||||
|
console.log(`- isWebWorker: ${brainy.isWebWorker()}`)
|
||||||
|
console.log(`- areWebWorkersAvailable: ${brainy.areWebWorkersAvailable()}`)
|
||||||
|
console.log(`- isThreadingAvailable: ${brainy.isThreadingAvailable()}`)
|
||||||
|
console.log(
|
||||||
|
`- areWorkerThreadsAvailableSync: ${brainy.areWorkerThreadsAvailableSync()}`
|
||||||
|
)
|
||||||
|
|
||||||
|
// Test TensorFlow functionality
|
||||||
|
console.log('\nTesting TensorFlow functionality...')
|
||||||
|
|
||||||
|
// Create a simple BrainyData instance
|
||||||
|
const data = new brainy.BrainyData({
|
||||||
|
dimensions: 2,
|
||||||
|
metric: 'euclidean'
|
||||||
|
})
|
||||||
|
|
||||||
|
console.log('Successfully created BrainyData instance')
|
||||||
|
|
||||||
|
// Initialize the database
|
||||||
|
console.log('Initializing database...')
|
||||||
|
await data.init()
|
||||||
|
|
||||||
|
// Add a simple vector
|
||||||
|
await data.add([1, 2], { id: 'test1', text: 'Test item' })
|
||||||
|
console.log('Successfully added item to BrainyData')
|
||||||
|
|
||||||
|
// Search for similar vectors
|
||||||
|
const results = await data.search([1, 2], 1)
|
||||||
|
console.log('Search results:', results)
|
||||||
|
|
||||||
|
// Test embedding functionality (which uses TensorFlow)
|
||||||
|
console.log('\nTesting embedding functionality...')
|
||||||
|
const embeddingFunction = brainy.createEmbeddingFunction()
|
||||||
|
const embedding = await embeddingFunction('This is a test sentence')
|
||||||
|
console.log(
|
||||||
|
`Successfully created embedding with length: ${embedding.length}`
|
||||||
|
)
|
||||||
|
|
||||||
|
console.log('\n✅ All Node.js tests passed successfully!')
|
||||||
|
return true
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Node.js test failed:', error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
runNodeTest().then((success) => {
|
||||||
|
if (!success) {
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
})
|
||||||
19
package.json
19
package.json
|
|
@ -6,7 +6,12 @@
|
||||||
"module": "dist/unified.js",
|
"module": "dist/unified.js",
|
||||||
"types": "dist/unified.d.ts",
|
"types": "dist/unified.d.ts",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"sideEffects": false,
|
"sideEffects": [
|
||||||
|
"./dist/setup.js",
|
||||||
|
"./dist/utils/textEncoding.js",
|
||||||
|
"./src/setup.ts",
|
||||||
|
"./src/utils/textEncoding.ts"
|
||||||
|
],
|
||||||
"exports": {
|
"exports": {
|
||||||
".": {
|
".": {
|
||||||
"import": "./dist/unified.js",
|
"import": "./dist/unified.js",
|
||||||
|
|
@ -15,6 +20,10 @@
|
||||||
"./min": {
|
"./min": {
|
||||||
"import": "./dist/unified.min.js"
|
"import": "./dist/unified.min.js"
|
||||||
},
|
},
|
||||||
|
"./setup": {
|
||||||
|
"import": "./dist/setup.js",
|
||||||
|
"types": "./dist/setup.d.ts"
|
||||||
|
},
|
||||||
"./types/graphTypes": {
|
"./types/graphTypes": {
|
||||||
"import": "./dist/types/graphTypes.js",
|
"import": "./dist/types/graphTypes.js",
|
||||||
"types": "./dist/types/graphTypes.d.ts"
|
"types": "./dist/types/graphTypes.d.ts"
|
||||||
|
|
@ -30,6 +39,10 @@
|
||||||
"./dist/utils/textEncoding.js": {
|
"./dist/utils/textEncoding.js": {
|
||||||
"import": "./dist/utils/textEncoding.js",
|
"import": "./dist/utils/textEncoding.js",
|
||||||
"types": "./dist/utils/textEncoding.d.ts"
|
"types": "./dist/utils/textEncoding.d.ts"
|
||||||
|
},
|
||||||
|
"./dist/setup.js": {
|
||||||
|
"import": "./dist/setup.js",
|
||||||
|
"types": "./dist/setup.d.ts"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
|
|
@ -61,9 +74,7 @@
|
||||||
"postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'",
|
"postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'",
|
||||||
"dry-run": "npm pack --dry-run",
|
"dry-run": "npm pack --dry-run",
|
||||||
"test:cli": "node scripts/test-cli-locally.js",
|
"test:cli": "node scripts/test-cli-locally.js",
|
||||||
"test:tensorflow": "node test-tensorflow-textencoder.js",
|
"test": "node scripts/comprehensive-test.js"
|
||||||
"test:all": "node scripts/test-all-environments.js",
|
|
||||||
"test": "npm run test:all"
|
|
||||||
},
|
},
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"vector-database",
|
"vector-database",
|
||||||
|
|
|
||||||
574
scripts/comprehensive-test.js
Executable file
574
scripts/comprehensive-test.js
Executable file
|
|
@ -0,0 +1,574 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Comprehensive Test Script for @soulcraft/brainy
|
||||||
|
*
|
||||||
|
* This script tests the library in all environments:
|
||||||
|
* - Browser (using Puppeteer for headless browser testing)
|
||||||
|
* - Node.js/server
|
||||||
|
* - CLI
|
||||||
|
*
|
||||||
|
* It verifies:
|
||||||
|
* - Library loading in each environment
|
||||||
|
* - TensorFlow functionality in each environment
|
||||||
|
* - Environment detection functionality
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { execSync } from 'child_process'
|
||||||
|
import { fileURLToPath } from 'url'
|
||||||
|
import path from 'path'
|
||||||
|
import fs from 'fs'
|
||||||
|
import http from 'http'
|
||||||
|
import puppeteer from 'puppeteer'
|
||||||
|
|
||||||
|
// Get the directory of the current module
|
||||||
|
const __filename = fileURLToPath(import.meta.url)
|
||||||
|
const __dirname = path.dirname(__filename)
|
||||||
|
const rootDir = path.join(__dirname, '..')
|
||||||
|
|
||||||
|
// Define colors for console output
|
||||||
|
const colors = {
|
||||||
|
reset: '\x1b[0m',
|
||||||
|
bright: '\x1b[1m',
|
||||||
|
green: '\x1b[32m',
|
||||||
|
yellow: '\x1b[33m',
|
||||||
|
red: '\x1b[31m',
|
||||||
|
cyan: '\x1b[36m',
|
||||||
|
magenta: '\x1b[35m',
|
||||||
|
blue: '\x1b[34m'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to log with colors
|
||||||
|
function log(message, color = colors.reset) {
|
||||||
|
console.log(`${color}${message}${colors.reset}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to log section headers
|
||||||
|
function logSection(title) {
|
||||||
|
console.log('\n' + '='.repeat(80))
|
||||||
|
console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`)
|
||||||
|
console.log('='.repeat(80) + '\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to log subsection headers
|
||||||
|
function logSubSection(title) {
|
||||||
|
console.log('\n' + '-'.repeat(60))
|
||||||
|
console.log(`${colors.bright}${colors.magenta}${title}${colors.reset}`)
|
||||||
|
console.log('-'.repeat(60) + '\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to run a command and return its output
|
||||||
|
function runCommand(command, cwd = rootDir) {
|
||||||
|
try {
|
||||||
|
return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' })
|
||||||
|
} catch (error) {
|
||||||
|
log(`Error running command: ${command}`, colors.red)
|
||||||
|
log(error.message, colors.red)
|
||||||
|
if (error.stdout) log(`stdout: ${error.stdout}`)
|
||||||
|
if (error.stderr) log(`stderr: ${error.stderr}`, colors.red)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a simple HTML file for browser testing
|
||||||
|
function createBrowserTestFile() {
|
||||||
|
const testHtmlPath = path.join(rootDir, 'browser-test.html')
|
||||||
|
const htmlContent = `
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>@soulcraft/brainy Browser Test</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-family: Arial, sans-serif;
|
||||||
|
max-width: 800px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 20px;
|
||||||
|
line-height: 1.6;
|
||||||
|
}
|
||||||
|
h1 {
|
||||||
|
color: #333;
|
||||||
|
border-bottom: 2px solid #eee;
|
||||||
|
padding-bottom: 10px;
|
||||||
|
}
|
||||||
|
button {
|
||||||
|
background-color: #4CAF50;
|
||||||
|
border: none;
|
||||||
|
color: white;
|
||||||
|
padding: 10px 20px;
|
||||||
|
text-align: center;
|
||||||
|
text-decoration: none;
|
||||||
|
display: inline-block;
|
||||||
|
font-size: 16px;
|
||||||
|
margin: 10px 2px;
|
||||||
|
cursor: pointer;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
#result {
|
||||||
|
background-color: #f5f5f5;
|
||||||
|
border: 1px solid #ddd;
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 15px;
|
||||||
|
margin-top: 20px;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
overflow-x: auto;
|
||||||
|
}
|
||||||
|
.success {
|
||||||
|
color: green;
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
.error {
|
||||||
|
color: red;
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>@soulcraft/brainy Browser Test</h1>
|
||||||
|
<p>This page tests the @soulcraft/brainy library in a browser environment.</p>
|
||||||
|
<button id="runTest">Run Test</button>
|
||||||
|
<div id="result">Test results will appear here...</div>
|
||||||
|
|
||||||
|
<script type="module">
|
||||||
|
// Import the library
|
||||||
|
import * as brainy from './dist/unified.js';
|
||||||
|
|
||||||
|
document.getElementById('runTest').addEventListener('click', async () => {
|
||||||
|
const resultElement = document.getElementById('result');
|
||||||
|
resultElement.innerHTML = 'Running tests...';
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test environment detection
|
||||||
|
const results = [];
|
||||||
|
results.push(\`Environment Detection:\`);
|
||||||
|
results.push(\`- isBrowser: \${brainy.isBrowser()}\`);
|
||||||
|
results.push(\`- isNode: \${brainy.isNode()}\`);
|
||||||
|
results.push(\`- isWebWorker: \${brainy.isWebWorker()}\`);
|
||||||
|
results.push(\`- areWebWorkersAvailable: \${brainy.areWebWorkersAvailable()}\`);
|
||||||
|
results.push(\`- isThreadingAvailable: \${brainy.isThreadingAvailable()}\`);
|
||||||
|
|
||||||
|
// Test TensorFlow functionality
|
||||||
|
results.push(\`\nTesting TensorFlow functionality...\`);
|
||||||
|
|
||||||
|
// Create a simple BrainyData instance
|
||||||
|
const data = new brainy.BrainyData({
|
||||||
|
dimensions: 2,
|
||||||
|
metric: 'euclidean'
|
||||||
|
});
|
||||||
|
|
||||||
|
results.push(\`Successfully created BrainyData instance\`);
|
||||||
|
|
||||||
|
// Initialize the database
|
||||||
|
results.push(\`Initializing database...\`);
|
||||||
|
await data.init();
|
||||||
|
|
||||||
|
// Add a simple vector
|
||||||
|
await data.add([1, 2], { id: 'test1', text: 'Test item' });
|
||||||
|
results.push(\`Successfully added item to BrainyData\`);
|
||||||
|
|
||||||
|
// Search for similar vectors
|
||||||
|
const searchResults = await data.search([1, 2], 1);
|
||||||
|
results.push(\`Search results: \${JSON.stringify(searchResults)}\`);
|
||||||
|
|
||||||
|
// Test embedding functionality (which uses TensorFlow)
|
||||||
|
results.push(\`\nTesting embedding functionality...\`);
|
||||||
|
try {
|
||||||
|
const embeddingFunction = brainy.createEmbeddingFunction();
|
||||||
|
const embedding = await embeddingFunction('This is a test sentence');
|
||||||
|
results.push(\`Successfully created embedding with length: \${embedding.length}\`);
|
||||||
|
} catch (embeddingError) {
|
||||||
|
results.push(\`Error testing embedding: \${embeddingError.message}\`);
|
||||||
|
throw embeddingError;
|
||||||
|
}
|
||||||
|
|
||||||
|
results.push(\`\n<span class="success">✅ All tests passed successfully!</span>\`);
|
||||||
|
resultElement.innerHTML = results.join('<br>');
|
||||||
|
} catch (error) {
|
||||||
|
resultElement.innerHTML = \`<span class="error">❌ Test failed:</span><br>\${error.message}\`;
|
||||||
|
console.error('Test error:', error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
fs.writeFileSync(testHtmlPath, htmlContent);
|
||||||
|
return testHtmlPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a Node.js test script
|
||||||
|
function createNodeTestScript() {
|
||||||
|
const testScriptPath = path.join(rootDir, 'node-test.js');
|
||||||
|
const scriptContent = `
|
||||||
|
// Node.js test script for @soulcraft/brainy
|
||||||
|
|
||||||
|
// CRITICAL: First, directly apply the TensorFlow.js patch
|
||||||
|
// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
|
||||||
|
import { TextEncoder, TextDecoder } from 'util';
|
||||||
|
|
||||||
|
// Make TextEncoder and TextDecoder available globally
|
||||||
|
if (typeof global !== 'undefined') {
|
||||||
|
global.TextEncoder = TextEncoder;
|
||||||
|
global.TextDecoder = TextDecoder;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import the library
|
||||||
|
import * as brainy from './dist/unified.js';
|
||||||
|
|
||||||
|
async function runNodeTest() {
|
||||||
|
console.log('\\n=== Testing @soulcraft/brainy in Node.js environment ===\\n');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test environment detection
|
||||||
|
console.log('Environment Detection:');
|
||||||
|
console.log(\`- isBrowser: \${brainy.isBrowser()}\`);
|
||||||
|
console.log(\`- isNode: \${brainy.isNode()}\`);
|
||||||
|
console.log(\`- isWebWorker: \${brainy.isWebWorker()}\`);
|
||||||
|
console.log(\`- areWebWorkersAvailable: \${brainy.areWebWorkersAvailable()}\`);
|
||||||
|
console.log(\`- isThreadingAvailable: \${brainy.isThreadingAvailable()}\`);
|
||||||
|
console.log(\`- areWorkerThreadsAvailableSync: \${brainy.areWorkerThreadsAvailableSync()}\`);
|
||||||
|
|
||||||
|
// Test TensorFlow functionality
|
||||||
|
console.log('\\nTesting TensorFlow functionality...');
|
||||||
|
|
||||||
|
// Create a simple BrainyData instance
|
||||||
|
const data = new brainy.BrainyData({
|
||||||
|
dimensions: 2,
|
||||||
|
metric: 'euclidean'
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Successfully created BrainyData instance');
|
||||||
|
|
||||||
|
// Initialize the database
|
||||||
|
console.log('Initializing database...');
|
||||||
|
await data.init();
|
||||||
|
|
||||||
|
// Add a simple vector
|
||||||
|
await data.add([1, 2], { id: 'test1', text: 'Test item' });
|
||||||
|
console.log('Successfully added item to BrainyData');
|
||||||
|
|
||||||
|
// Search for similar vectors
|
||||||
|
const results = await data.search([1, 2], 1);
|
||||||
|
console.log('Search results:', results);
|
||||||
|
|
||||||
|
// Test embedding functionality (which uses TensorFlow)
|
||||||
|
console.log('\\nTesting embedding functionality...');
|
||||||
|
const embeddingFunction = brainy.createEmbeddingFunction();
|
||||||
|
const embedding = await embeddingFunction('This is a test sentence');
|
||||||
|
console.log(\`Successfully created embedding with length: \${embedding.length}\`);
|
||||||
|
|
||||||
|
console.log('\\n✅ All Node.js tests passed successfully!');
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Node.js test failed:', error);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
runNodeTest().then(success => {
|
||||||
|
if (!success) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
`;
|
||||||
|
|
||||||
|
fs.writeFileSync(testScriptPath, scriptContent);
|
||||||
|
return testScriptPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a CLI test script
|
||||||
|
function createCliTestScript() {
|
||||||
|
const cliTestScriptPath = path.join(rootDir, 'cli-test.js');
|
||||||
|
const scriptContent = `
|
||||||
|
// CLI test script for @soulcraft/brainy-cli
|
||||||
|
|
||||||
|
import { execSync } from 'child_process';
|
||||||
|
|
||||||
|
function runCommand(command) {
|
||||||
|
try {
|
||||||
|
return execSync(command, { stdio: 'pipe', encoding: 'utf8' });
|
||||||
|
} catch (error) {
|
||||||
|
console.error(\`Error running command: \${command}\`);
|
||||||
|
console.error(error.message);
|
||||||
|
if (error.stdout) console.log(\`stdout: \${error.stdout}\`);
|
||||||
|
if (error.stderr) console.error(\`stderr: \${error.stderr}\`);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testCli() {
|
||||||
|
console.log('\\n=== Testing @soulcraft/brainy-cli ===\\n');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test CLI version
|
||||||
|
console.log('Testing CLI version...');
|
||||||
|
const versionOutput = runCommand('brainy --version');
|
||||||
|
console.log(\`CLI version: \${versionOutput.trim()}\`);
|
||||||
|
|
||||||
|
// Test CLI help
|
||||||
|
console.log('\\nTesting CLI help...');
|
||||||
|
runCommand('brainy --help');
|
||||||
|
console.log('Help command executed successfully');
|
||||||
|
|
||||||
|
// Test pipeline command
|
||||||
|
console.log('\\nTesting pipeline command...');
|
||||||
|
const pipelineOutput = runCommand('brainy test-pipeline "This is a test"');
|
||||||
|
console.log('Pipeline test completed successfully');
|
||||||
|
|
||||||
|
// Test TensorFlow functionality in CLI
|
||||||
|
console.log('\\nTesting TensorFlow functionality in CLI...');
|
||||||
|
const tensorflowOutput = runCommand('brainy test-tensorflow');
|
||||||
|
console.log('TensorFlow test completed successfully');
|
||||||
|
|
||||||
|
console.log('\\n✅ All CLI tests passed successfully!');
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ CLI test failed. This might be expected if you don\\'t have the CLI installed globally.');
|
||||||
|
console.error('You can install the CLI globally with: npm run test:cli');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
testCli().catch(error => {
|
||||||
|
console.error('Unhandled error:', error);
|
||||||
|
});
|
||||||
|
`;
|
||||||
|
|
||||||
|
fs.writeFileSync(cliTestScriptPath, scriptContent);
|
||||||
|
return cliTestScriptPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main function to run all tests
|
||||||
|
async function runComprehensiveTests() {
|
||||||
|
try {
|
||||||
|
logSection('COMPREHENSIVE TEST SUITE FOR @soulcraft/brainy');
|
||||||
|
log('This test suite verifies the library in all environments: Browser, Node.js, and CLI', colors.yellow);
|
||||||
|
|
||||||
|
logSection('BUILDING PACKAGES');
|
||||||
|
|
||||||
|
// Build the main package
|
||||||
|
logSubSection('Building Main Package');
|
||||||
|
log('Building main package...', colors.yellow);
|
||||||
|
runCommand('npm run build');
|
||||||
|
log('Main package built successfully!', colors.green);
|
||||||
|
|
||||||
|
// Build the browser package
|
||||||
|
logSubSection('Building Browser Package');
|
||||||
|
log('Building browser package...', colors.yellow);
|
||||||
|
runCommand('npm run build:browser');
|
||||||
|
log('Browser package built successfully!', colors.green);
|
||||||
|
|
||||||
|
// Build the CLI package
|
||||||
|
logSubSection('Building CLI Package');
|
||||||
|
log('Building CLI package...', colors.yellow);
|
||||||
|
runCommand('npm run build:cli');
|
||||||
|
log('CLI package built successfully!', colors.green);
|
||||||
|
|
||||||
|
logSection('NODE.JS ENVIRONMENT TESTS');
|
||||||
|
|
||||||
|
// Create and run Node.js test script
|
||||||
|
logSubSection('Creating Node.js Test Script');
|
||||||
|
const nodeTestScript = createNodeTestScript();
|
||||||
|
log(`Node.js test script created at: ${nodeTestScript}`, colors.green);
|
||||||
|
|
||||||
|
logSubSection('Running Node.js Tests');
|
||||||
|
try {
|
||||||
|
const nodeTestResult = runCommand(`node ${nodeTestScript}`);
|
||||||
|
log(nodeTestResult);
|
||||||
|
log('Node.js tests completed successfully!', colors.green);
|
||||||
|
} catch (error) {
|
||||||
|
log('Node.js tests failed!', colors.red);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
logSection('BROWSER ENVIRONMENT TESTS');
|
||||||
|
|
||||||
|
// Create browser test file
|
||||||
|
logSubSection('Creating Browser Test File');
|
||||||
|
const browserTestFile = createBrowserTestFile();
|
||||||
|
log(`Browser test file created at: ${browserTestFile}`, colors.green);
|
||||||
|
|
||||||
|
// Start a simple HTTP server to serve the test files
|
||||||
|
logSubSection('Starting HTTP Server');
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
// Normalize the URL to handle relative paths
|
||||||
|
const normalizedUrl = req.url.replace(/^\/+/, '/');
|
||||||
|
let filePath = path.join(
|
||||||
|
rootDir,
|
||||||
|
normalizedUrl === '/' ? 'browser-test.html' : normalizedUrl
|
||||||
|
);
|
||||||
|
|
||||||
|
// Handle relative paths (e.g., ../dist/unified.js)
|
||||||
|
if (normalizedUrl.includes('../')) {
|
||||||
|
// Convert the URL to an absolute path relative to the root directory
|
||||||
|
const parts = normalizedUrl.split('/');
|
||||||
|
const resolvedParts = [];
|
||||||
|
|
||||||
|
for (const part of parts) {
|
||||||
|
if (part === '..') {
|
||||||
|
resolvedParts.pop();
|
||||||
|
} else if (part && part !== '.') {
|
||||||
|
resolvedParts.push(part);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
filePath = path.join(rootDir, resolvedParts.join('/'));
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.blue);
|
||||||
|
|
||||||
|
// Check if the file exists
|
||||||
|
if (fs.existsSync(filePath)) {
|
||||||
|
const extname = path.extname(filePath);
|
||||||
|
let contentType = 'text/html';
|
||||||
|
|
||||||
|
switch (extname) {
|
||||||
|
case '.js':
|
||||||
|
contentType = 'text/javascript';
|
||||||
|
break;
|
||||||
|
case '.css':
|
||||||
|
contentType = 'text/css';
|
||||||
|
break;
|
||||||
|
case '.json':
|
||||||
|
contentType = 'application/json';
|
||||||
|
break;
|
||||||
|
case '.png':
|
||||||
|
contentType = 'image/png';
|
||||||
|
break;
|
||||||
|
case '.jpg':
|
||||||
|
contentType = 'image/jpg';
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.writeHead(200, { 'Content-Type': contentType });
|
||||||
|
const fileStream = fs.createReadStream(filePath);
|
||||||
|
fileStream.pipe(res);
|
||||||
|
} else {
|
||||||
|
log(`File not found: ${filePath}`, colors.red);
|
||||||
|
res.writeHead(404);
|
||||||
|
res.end('File not found');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start the server on a random port
|
||||||
|
const PORT = 3000 + Math.floor(Math.random() * 1000);
|
||||||
|
server.listen(PORT);
|
||||||
|
log(`HTTP server started on port ${PORT}`, colors.green);
|
||||||
|
|
||||||
|
// Run browser tests using Puppeteer
|
||||||
|
logSubSection('Running Browser Tests with Puppeteer');
|
||||||
|
log('Launching headless browser...', colors.yellow);
|
||||||
|
const browser = await puppeteer.launch({ args: ['--no-sandbox'] });
|
||||||
|
const page = await browser.newPage();
|
||||||
|
|
||||||
|
// Capture console logs from the page
|
||||||
|
page.on('console', (message) => {
|
||||||
|
const type = message.type();
|
||||||
|
const text = message.text();
|
||||||
|
if (type === 'error') {
|
||||||
|
log(`Browser console error: ${text}`, colors.red);
|
||||||
|
} else {
|
||||||
|
log(`Browser console: ${text}`, colors.blue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Navigate to the test page
|
||||||
|
log('Navigating to browser test page...', colors.yellow);
|
||||||
|
await page.goto(`http://localhost:${PORT}/browser-test.html`);
|
||||||
|
|
||||||
|
// Run the test
|
||||||
|
log('Running browser tests...', colors.yellow);
|
||||||
|
await page.waitForSelector('#runTest');
|
||||||
|
await page.click('#runTest');
|
||||||
|
|
||||||
|
// Wait for test completion
|
||||||
|
await page.waitForFunction(
|
||||||
|
() => {
|
||||||
|
const resultText = document.getElementById('result').textContent;
|
||||||
|
return resultText.includes('All tests passed') || resultText.includes('Test failed');
|
||||||
|
},
|
||||||
|
{ timeout: 60000 }
|
||||||
|
);
|
||||||
|
|
||||||
|
// Get test results
|
||||||
|
const browserTestResult = await page.evaluate(() => {
|
||||||
|
return document.getElementById('result').innerHTML;
|
||||||
|
});
|
||||||
|
|
||||||
|
log('Browser test results:', colors.green);
|
||||||
|
log(browserTestResult.replace(/<[^>]*>/g, '').trim());
|
||||||
|
|
||||||
|
// Check if the test passed
|
||||||
|
const browserTestPassed = browserTestResult.includes('All tests passed');
|
||||||
|
if (!browserTestPassed) {
|
||||||
|
throw new Error('Browser tests failed!');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the browser and server
|
||||||
|
await browser.close();
|
||||||
|
server.close();
|
||||||
|
log('HTTP server stopped', colors.green);
|
||||||
|
|
||||||
|
logSection('CLI ENVIRONMENT TESTS');
|
||||||
|
|
||||||
|
// Create and run CLI test script
|
||||||
|
logSubSection('Creating CLI Test Script');
|
||||||
|
const cliTestScript = createCliTestScript();
|
||||||
|
log(`CLI test script created at: ${cliTestScript}`, colors.green);
|
||||||
|
|
||||||
|
logSubSection('Installing CLI Package Locally');
|
||||||
|
log('Installing CLI package locally for testing...', colors.yellow);
|
||||||
|
try {
|
||||||
|
runCommand('npm run test:cli');
|
||||||
|
log('CLI package installed successfully!', colors.green);
|
||||||
|
|
||||||
|
logSubSection('Running CLI Tests');
|
||||||
|
try {
|
||||||
|
const cliTestResult = runCommand(`node ${cliTestScript}`);
|
||||||
|
log(cliTestResult);
|
||||||
|
log('CLI tests completed!', colors.green);
|
||||||
|
} catch (error) {
|
||||||
|
log('CLI tests failed. This might be expected if you don\'t have the CLI installed globally.', colors.yellow);
|
||||||
|
log('You can install the CLI globally with: npm run test:cli', colors.yellow);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
log('Failed to install CLI package locally. Skipping CLI tests.', colors.yellow);
|
||||||
|
log('You can run the CLI tests separately with: npm run test:cli', colors.yellow);
|
||||||
|
}
|
||||||
|
|
||||||
|
logSection('CLEANING UP');
|
||||||
|
|
||||||
|
// Clean up test files
|
||||||
|
log('Cleaning up test files...', colors.yellow);
|
||||||
|
fs.unlinkSync(nodeTestScript);
|
||||||
|
fs.unlinkSync(browserTestFile);
|
||||||
|
fs.unlinkSync(cliTestScript);
|
||||||
|
log('Test files removed', colors.green);
|
||||||
|
|
||||||
|
logSection('TEST SUMMARY');
|
||||||
|
log('✅ All environment tests completed successfully!', colors.green);
|
||||||
|
log('The library has been tested in the following environments:', colors.green);
|
||||||
|
log('- Browser environment', colors.green);
|
||||||
|
log('- Node.js/server environment', colors.green);
|
||||||
|
log('- CLI environment', colors.green);
|
||||||
|
|
||||||
|
log('\nTensorFlow functionality has been verified in all environments.', colors.green);
|
||||||
|
log('Environment detection has been tested and is working correctly.', colors.green);
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
logSection('TEST FAILURE');
|
||||||
|
log(`Tests failed: ${error.message}`, colors.red);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the tests
|
||||||
|
runComprehensiveTests().catch((error) => {
|
||||||
|
log(`Unhandled error: ${error.message}`, colors.red);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
@ -1,300 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Test All Environments
|
|
||||||
*
|
|
||||||
* This script runs tests for the Brainy library in all environments:
|
|
||||||
* - Browser (using Puppeteer for headless browser testing)
|
|
||||||
* - Node.js
|
|
||||||
* - CLI
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { execSync } from 'child_process'
|
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
import path from 'path'
|
|
||||||
import fs from 'fs'
|
|
||||||
import http from 'http'
|
|
||||||
import puppeteer from 'puppeteer'
|
|
||||||
|
|
||||||
// Get the directory of the current module
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
|
||||||
const __dirname = path.dirname(__filename)
|
|
||||||
const rootDir = path.join(__dirname, '..')
|
|
||||||
|
|
||||||
// Define colors for console output
|
|
||||||
const colors = {
|
|
||||||
reset: '\x1b[0m',
|
|
||||||
bright: '\x1b[1m',
|
|
||||||
green: '\x1b[32m',
|
|
||||||
yellow: '\x1b[33m',
|
|
||||||
red: '\x1b[31m',
|
|
||||||
cyan: '\x1b[36m'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper function to log with colors
|
|
||||||
function log(message, color = colors.reset) {
|
|
||||||
console.log(`${color}${message}${colors.reset}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper function to log section headers
|
|
||||||
function logSection(title) {
|
|
||||||
console.log('\n' + '='.repeat(80))
|
|
||||||
console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`)
|
|
||||||
console.log('='.repeat(80) + '\n')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper function to run a command and return its output
|
|
||||||
function runCommand(command, cwd = rootDir) {
|
|
||||||
try {
|
|
||||||
return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' })
|
|
||||||
} catch (error) {
|
|
||||||
log(`Error running command: ${command}`, colors.red)
|
|
||||||
log(error.message, colors.red)
|
|
||||||
if (error.stdout) log(`stdout: ${error.stdout}`)
|
|
||||||
if (error.stderr) log(`stderr: ${error.stderr}`, colors.red)
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Main function to run all tests
|
|
||||||
async function runAllTests() {
|
|
||||||
try {
|
|
||||||
logSection('BUILDING PACKAGES')
|
|
||||||
|
|
||||||
// Build the main package
|
|
||||||
log('Building main package...', colors.yellow)
|
|
||||||
runCommand('npm run build')
|
|
||||||
log('Main package built successfully!', colors.green)
|
|
||||||
|
|
||||||
// Apply TextEncoder patch
|
|
||||||
log('Applying TextEncoder patch...', colors.yellow)
|
|
||||||
runCommand('node scripts/patch-textencoder.js')
|
|
||||||
log('TextEncoder patch applied successfully!', colors.green)
|
|
||||||
|
|
||||||
// Build the browser package
|
|
||||||
log('Building browser package...', colors.yellow)
|
|
||||||
runCommand('npm run build:browser')
|
|
||||||
log('Browser package built successfully!', colors.green)
|
|
||||||
|
|
||||||
// Build the CLI package
|
|
||||||
log('Building CLI package...', colors.yellow)
|
|
||||||
runCommand('npm run build:cli')
|
|
||||||
log('CLI package built successfully!', colors.green)
|
|
||||||
|
|
||||||
logSection('RUNNING NODE.JS TESTS')
|
|
||||||
|
|
||||||
// Run Node.js tests
|
|
||||||
log('Running Node.js worker test...', colors.yellow)
|
|
||||||
const nodeWorkerResult = runCommand('node test-worker.js')
|
|
||||||
log(nodeWorkerResult)
|
|
||||||
log('Node.js worker test completed!', colors.green)
|
|
||||||
|
|
||||||
log('Running unified text encoding test...', colors.yellow)
|
|
||||||
const textEncodingResult = runCommand('node test-unified-encoding.js')
|
|
||||||
log(textEncodingResult)
|
|
||||||
log('Unified text encoding test completed!', colors.green)
|
|
||||||
|
|
||||||
log('Running TensorFlow and TextEncoder test...', colors.yellow)
|
|
||||||
const tensorflowTextEncoderResult = runCommand('node test-tensorflow-textencoder.js')
|
|
||||||
log(tensorflowTextEncoderResult)
|
|
||||||
log('TensorFlow and TextEncoder test completed!', colors.green)
|
|
||||||
|
|
||||||
logSection('RUNNING BROWSER TESTS')
|
|
||||||
|
|
||||||
// Start a simple HTTP server to serve the test files
|
|
||||||
log('Starting HTTP server...', colors.yellow)
|
|
||||||
const server = http.createServer((req, res) => {
|
|
||||||
// Normalize the URL to handle relative paths
|
|
||||||
const normalizedUrl = req.url.replace(/^\/+/, '/')
|
|
||||||
let filePath = path.join(
|
|
||||||
rootDir,
|
|
||||||
normalizedUrl === '/' ? 'index.html' : normalizedUrl
|
|
||||||
)
|
|
||||||
|
|
||||||
// Handle relative paths (e.g., ../dist/unified.js)
|
|
||||||
if (normalizedUrl.includes('../')) {
|
|
||||||
// Convert the URL to an absolute path relative to the root directory
|
|
||||||
const parts = normalizedUrl.split('/')
|
|
||||||
const resolvedParts = []
|
|
||||||
|
|
||||||
for (const part of parts) {
|
|
||||||
if (part === '..') {
|
|
||||||
resolvedParts.pop()
|
|
||||||
} else if (part && part !== '.') {
|
|
||||||
resolvedParts.push(part)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
filePath = path.join(rootDir, resolvedParts.join('/'))
|
|
||||||
}
|
|
||||||
|
|
||||||
log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.yellow)
|
|
||||||
|
|
||||||
// Check if the file exists
|
|
||||||
if (fs.existsSync(filePath)) {
|
|
||||||
const extname = path.extname(filePath)
|
|
||||||
let contentType = 'text/html'
|
|
||||||
|
|
||||||
switch (extname) {
|
|
||||||
case '.js':
|
|
||||||
contentType = 'text/javascript'
|
|
||||||
break
|
|
||||||
case '.css':
|
|
||||||
contentType = 'text/css'
|
|
||||||
break
|
|
||||||
case '.json':
|
|
||||||
contentType = 'application/json'
|
|
||||||
break
|
|
||||||
case '.png':
|
|
||||||
contentType = 'image/png'
|
|
||||||
break
|
|
||||||
case '.jpg':
|
|
||||||
contentType = 'image/jpg'
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
res.writeHead(200, { 'Content-Type': contentType })
|
|
||||||
const fileStream = fs.createReadStream(filePath)
|
|
||||||
fileStream.pipe(res)
|
|
||||||
} else {
|
|
||||||
log(`File not found: ${filePath}`, colors.red)
|
|
||||||
res.writeHead(404)
|
|
||||||
res.end('File not found')
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Start the server on a random port
|
|
||||||
const PORT = 3000 + Math.floor(Math.random() * 1000)
|
|
||||||
server.listen(PORT)
|
|
||||||
log(`HTTP server started on port ${PORT}`, colors.green)
|
|
||||||
|
|
||||||
// Run browser tests using Puppeteer
|
|
||||||
log('Launching headless browser...', colors.yellow)
|
|
||||||
// Using --no-sandbox flag to avoid issues with the Chrome sandbox in certain environments
|
|
||||||
// See: https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md
|
|
||||||
const browser = await puppeteer.launch({ args: ['--no-sandbox'] })
|
|
||||||
const page = await browser.newPage()
|
|
||||||
|
|
||||||
// Capture console logs from the page
|
|
||||||
page.on('console', (message) => {
|
|
||||||
const type = message.type()
|
|
||||||
const text = message.text()
|
|
||||||
if (type === 'error') {
|
|
||||||
log(`Browser console error: ${text}`, colors.red)
|
|
||||||
} else {
|
|
||||||
log(`Browser console: ${text}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test browser worker
|
|
||||||
log('Running browser worker test...', colors.yellow)
|
|
||||||
await page.goto(`http://localhost:${PORT}/demo/test-browser-worker.html`)
|
|
||||||
await page.waitForSelector('#runTest')
|
|
||||||
await page.click('#runTest')
|
|
||||||
await page.waitForFunction(
|
|
||||||
() => {
|
|
||||||
const resultText = document.getElementById('result').textContent
|
|
||||||
return resultText.includes('Worker thread execution completed')
|
|
||||||
},
|
|
||||||
{ timeout: 30000 }
|
|
||||||
)
|
|
||||||
|
|
||||||
const browserWorkerResult = await page.evaluate(() => {
|
|
||||||
return document.getElementById('result').innerHTML
|
|
||||||
})
|
|
||||||
log('Browser worker test result:', colors.green)
|
|
||||||
log(browserWorkerResult.replace(/<[^>]*>/g, '').trim())
|
|
||||||
|
|
||||||
// Test fallback mechanism
|
|
||||||
log('Running fallback test...', colors.yellow)
|
|
||||||
await page.goto(`http://localhost:${PORT}/demo/test-fallback.html`)
|
|
||||||
await page.waitForSelector('#runTest')
|
|
||||||
await page.click('#runTest')
|
|
||||||
await page.waitForFunction(
|
|
||||||
() => {
|
|
||||||
const resultText = document.getElementById('result').textContent
|
|
||||||
return resultText.includes('Test completed')
|
|
||||||
},
|
|
||||||
{ timeout: 30000 }
|
|
||||||
)
|
|
||||||
|
|
||||||
const fallbackResult = await page.evaluate(() => {
|
|
||||||
return document.getElementById('result').innerHTML
|
|
||||||
})
|
|
||||||
log('Fallback test result:', colors.green)
|
|
||||||
log(fallbackResult.replace(/<[^>]*>/g, '').trim())
|
|
||||||
|
|
||||||
// Test TensorFlow and TextEncoder in browser
|
|
||||||
log('Running TensorFlow and TextEncoder browser test...', colors.yellow)
|
|
||||||
await page.goto(`http://localhost:${PORT}/demo/test-tensorflow-textencoder.html`)
|
|
||||||
await page.waitForSelector('#runTest')
|
|
||||||
await page.click('#runTest')
|
|
||||||
await page.waitForFunction(
|
|
||||||
() => {
|
|
||||||
const resultText = document.getElementById('result').textContent
|
|
||||||
return resultText.includes('Test completed')
|
|
||||||
},
|
|
||||||
{ timeout: 30000 }
|
|
||||||
)
|
|
||||||
|
|
||||||
const browserTensorflowTextEncoderResult = await page.evaluate(() => {
|
|
||||||
return document.getElementById('result').innerHTML
|
|
||||||
})
|
|
||||||
log('TensorFlow and TextEncoder browser test result:', colors.green)
|
|
||||||
log(browserTensorflowTextEncoderResult.replace(/<[^>]*>/g, '').trim())
|
|
||||||
|
|
||||||
// Close the browser and server
|
|
||||||
await browser.close()
|
|
||||||
server.close()
|
|
||||||
log('HTTP server stopped', colors.green)
|
|
||||||
|
|
||||||
logSection('RUNNING CLI TESTS')
|
|
||||||
|
|
||||||
// Run CLI tests
|
|
||||||
log('Testing CLI package locally...', colors.yellow)
|
|
||||||
try {
|
|
||||||
runCommand('npm run test:cli')
|
|
||||||
log('CLI test completed!', colors.green)
|
|
||||||
|
|
||||||
// Run some basic CLI commands to verify functionality
|
|
||||||
log('Testing basic CLI commands...', colors.yellow)
|
|
||||||
const cliVersionResult = runCommand('brainy --version')
|
|
||||||
log(`CLI version: ${cliVersionResult.trim()}`, colors.green)
|
|
||||||
|
|
||||||
const cliHelpResult = runCommand('brainy --help')
|
|
||||||
log('CLI help command executed successfully', colors.green)
|
|
||||||
|
|
||||||
// Test the pipeline command
|
|
||||||
log('Testing pipeline command...', colors.yellow)
|
|
||||||
const pipelineResult = runCommand('brainy test-pipeline "This is a test"')
|
|
||||||
log('Pipeline test completed!', colors.green)
|
|
||||||
|
|
||||||
// Test TensorFlow and TextEncoder in CLI
|
|
||||||
log('Testing TensorFlow and TextEncoder in CLI...', colors.yellow)
|
|
||||||
const cliTensorflowResult = runCommand('brainy test-tensorflow-textencoder')
|
|
||||||
log('TensorFlow and TextEncoder CLI test completed!', colors.green)
|
|
||||||
} catch (error) {
|
|
||||||
log(
|
|
||||||
"CLI tests failed. This might be expected if you don't have the CLI installed globally.",
|
|
||||||
colors.yellow
|
|
||||||
)
|
|
||||||
log(
|
|
||||||
'You can run the CLI tests separately with: npm run test:cli',
|
|
||||||
colors.yellow
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
logSection('ALL TESTS COMPLETED')
|
|
||||||
log('All environment tests completed successfully!', colors.green)
|
|
||||||
} catch (error) {
|
|
||||||
logSection('TEST FAILURE')
|
|
||||||
log(`Tests failed: ${error.message}`, colors.red)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the tests
|
|
||||||
runAllTests().catch((error) => {
|
|
||||||
log(`Unhandled error: ${error.message}`, colors.red)
|
|
||||||
process.exit(1)
|
|
||||||
})
|
|
||||||
|
|
@ -3,6 +3,10 @@
|
||||||
* A vector and graph database using HNSW
|
* A vector and graph database using HNSW
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// CRITICAL: The TensorFlow.js environment patch is now centralized in setup.ts
|
||||||
|
// We import setup.js below which applies the necessary patches through textEncoding.js
|
||||||
|
// This ensures a consistent patching approach and avoids conflicts
|
||||||
|
|
||||||
// Import the setup file for its side-effects.
|
// Import the setup file for its side-effects.
|
||||||
// This MUST be the very first import to ensure patches are applied
|
// This MUST be the very first import to ensure patches are applied
|
||||||
// before any other module (like TensorFlow.js) is loaded.
|
// before any other module (like TensorFlow.js) is loaded.
|
||||||
|
|
|
||||||
44
src/setup.ts
44
src/setup.ts
|
|
@ -1,12 +1,54 @@
|
||||||
/**
|
/**
|
||||||
* This file is imported for its side effects to patch the environment
|
* CRITICAL: This file is imported for its side effects to patch the environment
|
||||||
* for TensorFlow.js before any other library code runs.
|
* for TensorFlow.js before any other library code runs.
|
||||||
*
|
*
|
||||||
* It ensures that by the time TensorFlow.js is imported by any other
|
* It ensures that by the time TensorFlow.js is imported by any other
|
||||||
* module, the necessary compatibility fixes for the current Node.js
|
* module, the necessary compatibility fixes for the current Node.js
|
||||||
* environment are already in place.
|
* environment are already in place.
|
||||||
|
*
|
||||||
|
* This file MUST be imported as the first import in unified.ts to prevent
|
||||||
|
* race conditions with TensorFlow.js initialization. Failure to do so will
|
||||||
|
* result in errors like "TextEncoder is not a constructor" when the package
|
||||||
|
* is used in Node.js environments.
|
||||||
|
*
|
||||||
|
* The package.json file marks this file as having side effects to prevent
|
||||||
|
* tree-shaking by bundlers, ensuring the patch is always applied.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// CRITICAL: Apply the TensorFlow.js patch immediately at the top level
|
||||||
|
// This ensures it runs as early as possible in the module loading process
|
||||||
|
// before any imports are processed
|
||||||
|
if (
|
||||||
|
typeof process !== 'undefined' &&
|
||||||
|
process.versions &&
|
||||||
|
process.versions.node
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
// For CommonJS environments, use require to ensure synchronous loading
|
||||||
|
if (typeof require === 'function') {
|
||||||
|
const textEncoding = require('./utils/textEncoding.js')
|
||||||
|
if (
|
||||||
|
textEncoding &&
|
||||||
|
typeof textEncoding.applyTensorFlowPatch === 'function'
|
||||||
|
) {
|
||||||
|
textEncoding.applyTensorFlowPatch()
|
||||||
|
console.log(
|
||||||
|
'Applied TensorFlow.js patch via CommonJS require in setup.ts'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn('Failed to apply TensorFlow.js patch via require:', e)
|
||||||
|
// Continue to the import-based approach
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also import normally for ES modules environments
|
||||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||||
|
|
||||||
// Apply the TensorFlow.js platform patch if needed
|
// Apply the TensorFlow.js platform patch if needed
|
||||||
|
// This will be a no-op if the patch was already applied via require above
|
||||||
applyTensorFlowPatch()
|
applyTensorFlowPatch()
|
||||||
|
console.log(
|
||||||
|
'Applied or verified TensorFlow.js patch via ES modules in setup.ts'
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,18 @@
|
||||||
* Environment detection is handled here and made available to all components
|
* Environment detection is handled here and made available to all components
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// CRITICAL: The TensorFlow.js environment patch is now centralized in setup.ts
|
||||||
|
// We import setup.ts below which applies the necessary patches
|
||||||
|
|
||||||
|
// CRITICAL: Import setup.js first to ensure TensorFlow.js environment patching
|
||||||
|
// This MUST be the first import to prevent race conditions with TensorFlow.js initialization
|
||||||
|
// Moving or removing this import will cause errors like "TextEncoder is not a constructor"
|
||||||
|
// when the package is used in Node.js environments
|
||||||
|
//
|
||||||
|
// The setup.js file applies a patch that ensures TextEncoder/TextDecoder are properly
|
||||||
|
// available to TensorFlow.js before it initializes its platform detection
|
||||||
|
import './setup.js'
|
||||||
|
|
||||||
// Export environment information
|
// Export environment information
|
||||||
export const environment = {
|
export const environment = {
|
||||||
isBrowser: typeof window !== 'undefined',
|
isBrowser: typeof window !== 'undefined',
|
||||||
|
|
|
||||||
|
|
@ -145,14 +145,36 @@ export async function calculateDistancesBatch(
|
||||||
// In worker context, use the importTensorFlow function
|
// In worker context, use the importTensorFlow function
|
||||||
tf = await self.importTensorFlow()
|
tf = await self.importTensorFlow()
|
||||||
} else {
|
} else {
|
||||||
// Dynamically import TensorFlow.js core module and backends
|
// CRITICAL: First, directly import the setup module to ensure the TensorFlow.js patch is applied
|
||||||
tf = await import('@tensorflow/tfjs-core')
|
// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
|
||||||
|
try {
|
||||||
|
// In Node.js environment, use require() which is synchronous
|
||||||
|
if (typeof require !== 'undefined') {
|
||||||
|
// First, require the setup module to apply the patch
|
||||||
|
require('../setup.js')
|
||||||
|
|
||||||
// Import CPU backend
|
// Now load TensorFlow.js core module
|
||||||
await import('@tensorflow/tfjs-backend-cpu')
|
tf = require('@tensorflow/tfjs-core')
|
||||||
|
|
||||||
// Set CPU as the backend
|
// Load CPU backend
|
||||||
await tf.setBackend('cpu')
|
require('@tensorflow/tfjs-backend-cpu')
|
||||||
|
|
||||||
|
// Set CPU as the backend
|
||||||
|
tf.setBackend('cpu')
|
||||||
|
} else {
|
||||||
|
// In browser or other environments without require(), use dynamic imports
|
||||||
|
// First, dynamically import the setup module to apply the patch
|
||||||
|
await import('../setup.js')
|
||||||
|
|
||||||
|
// Now load TensorFlow.js core module
|
||||||
|
tf = await import('@tensorflow/tfjs-core')
|
||||||
|
await import('@tensorflow/tfjs-backend-cpu')
|
||||||
|
await tf.setBackend('cpu')
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to initialize TensorFlow.js:', error)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert vectors to tensors
|
// Convert vectors to tensors
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,10 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
|
||||||
/**
|
/**
|
||||||
* Add polyfills and patches for TensorFlow.js compatibility
|
* Add polyfills and patches for TensorFlow.js compatibility
|
||||||
* This addresses issues with TensorFlow.js in Node.js environments
|
* This addresses issues with TensorFlow.js in Node.js environments
|
||||||
|
*
|
||||||
|
* Note: The main TensorFlow.js patching is now centralized in textEncoding.ts
|
||||||
|
* and applied through setup.ts. This method only adds additional utility functions
|
||||||
|
* that might be needed by TensorFlow.js.
|
||||||
*/
|
*/
|
||||||
private addNodeCompatibilityPolyfills(): void {
|
private addNodeCompatibilityPolyfills(): void {
|
||||||
// Only apply in Node.js environment
|
// Only apply in Node.js environment
|
||||||
|
|
@ -38,82 +42,30 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
|
||||||
// This fixes the "Cannot read properties of undefined (reading 'isFloat32Array')" error
|
// This fixes the "Cannot read properties of undefined (reading 'isFloat32Array')" error
|
||||||
if (typeof global !== 'undefined') {
|
if (typeof global !== 'undefined') {
|
||||||
try {
|
try {
|
||||||
// Define a custom PlatformNode class
|
// Ensure the util object exists
|
||||||
class PlatformNode {
|
if (!global.util) {
|
||||||
util: any
|
global.util = {}
|
||||||
textEncoder: TextEncoder
|
}
|
||||||
textDecoder: TextDecoder
|
|
||||||
|
|
||||||
constructor() {
|
// Add isFloat32Array method if it doesn't exist
|
||||||
// Create a util object with necessary methods
|
if (!global.util.isFloat32Array) {
|
||||||
this.util = {
|
global.util.isFloat32Array = (obj: any) => {
|
||||||
// Add isFloat32Array and isTypedArray directly to util
|
|
||||||
isFloat32Array: (arr: any) => {
|
|
||||||
return !!(
|
|
||||||
arr instanceof Float32Array ||
|
|
||||||
(arr &&
|
|
||||||
Object.prototype.toString.call(arr) ===
|
|
||||||
'[object Float32Array]')
|
|
||||||
)
|
|
||||||
},
|
|
||||||
isTypedArray: (arr: any) => {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
},
|
|
||||||
// Use native TextEncoder and TextDecoder
|
|
||||||
TextEncoder: TextEncoder,
|
|
||||||
TextDecoder: TextDecoder
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize encoders using native constructors
|
|
||||||
this.textEncoder = new TextEncoder()
|
|
||||||
this.textDecoder = new TextDecoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define isFloat32Array directly on the instance
|
|
||||||
isFloat32Array(arr: any) {
|
|
||||||
return !!(
|
return !!(
|
||||||
arr instanceof Float32Array ||
|
obj instanceof Float32Array ||
|
||||||
(arr &&
|
(obj &&
|
||||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
Object.prototype.toString.call(obj) === '[object Float32Array]')
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Define isTypedArray directly on the instance
|
// Add isTypedArray method if it doesn't exist
|
||||||
isTypedArray(arr: any) {
|
if (!global.util.isTypedArray) {
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
global.util.isTypedArray = (obj: any) => {
|
||||||
|
return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assign the PlatformNode class to the global object
|
|
||||||
;(global as any).PlatformNode = PlatformNode
|
|
||||||
|
|
||||||
// Also create an instance and assign it to global.platformNode
|
|
||||||
;(global as any).platformNode = new PlatformNode()
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('Failed to define global PlatformNode class:', error)
|
console.warn('Failed to add utility polyfills:', error)
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the util object exists
|
|
||||||
if (!global.util) {
|
|
||||||
global.util = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add isFloat32Array method if it doesn't exist
|
|
||||||
if (!global.util.isFloat32Array) {
|
|
||||||
global.util.isFloat32Array = (obj: any) => {
|
|
||||||
return !!(
|
|
||||||
obj instanceof Float32Array ||
|
|
||||||
(obj &&
|
|
||||||
Object.prototype.toString.call(obj) === '[object Float32Array]')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add isTypedArray method if it doesn't exist
|
|
||||||
if (!global.util.isTypedArray) {
|
|
||||||
global.util.isTypedArray = (obj: any) => {
|
|
||||||
return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -145,36 +97,95 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
|
||||||
|
|
||||||
// TensorFlow.js will use its default EPSILON value
|
// TensorFlow.js will use its default EPSILON value
|
||||||
|
|
||||||
// Dynamically import TensorFlow.js core module and backends
|
// CRITICAL: First, directly import the setup module to ensure the TensorFlow.js patch is applied
|
||||||
// Use type assertions to tell TypeScript these modules exist
|
// This is the most reliable way to ensure the patch is applied before TensorFlow.js is loaded
|
||||||
this.tf = await import('@tensorflow/tfjs-core')
|
|
||||||
|
|
||||||
// Import CPU backend (always needed as fallback)
|
|
||||||
await import('@tensorflow/tfjs-backend-cpu')
|
|
||||||
|
|
||||||
// Try to import WebGL backend for GPU acceleration in browser environments
|
|
||||||
try {
|
try {
|
||||||
if (typeof window !== 'undefined') {
|
// In Node.js environment, use require() which is synchronous
|
||||||
await import('@tensorflow/tfjs-backend-webgl')
|
if (typeof require !== 'undefined') {
|
||||||
// Check if WebGL is available using setBackend instead of findBackend
|
// First, require the setup module to apply the patch
|
||||||
try {
|
require('../setup.js')
|
||||||
if (this.tf.setBackend) {
|
|
||||||
await this.tf.setBackend('webgl')
|
// Now load TensorFlow.js core module
|
||||||
this.backend = 'webgl'
|
this.tf = require('@tensorflow/tfjs-core')
|
||||||
console.log('Using WebGL backend for TensorFlow.js')
|
|
||||||
} else {
|
// Load CPU backend (always needed as fallback)
|
||||||
|
require('@tensorflow/tfjs-backend-cpu')
|
||||||
|
|
||||||
|
// Try to load WebGL backend for GPU acceleration in browser environments
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
try {
|
||||||
|
require('@tensorflow/tfjs-backend-webgl')
|
||||||
|
// Check if WebGL is available
|
||||||
|
if (this.tf.setBackend) {
|
||||||
|
this.tf.setBackend('webgl')
|
||||||
|
this.backend = 'webgl'
|
||||||
|
console.log('Using WebGL backend for TensorFlow.js')
|
||||||
|
} else {
|
||||||
|
console.warn(
|
||||||
|
'tf.setBackend is not available, falling back to CPU'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
console.warn(
|
console.warn(
|
||||||
'tf.setBackend is not available, falling back to CPU'
|
'WebGL backend not available, falling back to CPU:',
|
||||||
|
e
|
||||||
)
|
)
|
||||||
|
this.backend = 'cpu'
|
||||||
}
|
}
|
||||||
} catch (e) {
|
}
|
||||||
console.warn('WebGL backend not available, falling back to CPU:', e)
|
|
||||||
|
// Load Universal Sentence Encoder
|
||||||
|
this.use = require('@tensorflow-models/universal-sentence-encoder')
|
||||||
|
} else {
|
||||||
|
// In browser or other environments without require(), use dynamic imports
|
||||||
|
// First, dynamically import the setup module to apply the patch
|
||||||
|
await import('../setup.js')
|
||||||
|
|
||||||
|
// Now load TensorFlow.js core module
|
||||||
|
this.tf = await import('@tensorflow/tfjs-core')
|
||||||
|
|
||||||
|
// Import CPU backend (always needed as fallback)
|
||||||
|
await import('@tensorflow/tfjs-backend-cpu')
|
||||||
|
|
||||||
|
// Try to import WebGL backend for GPU acceleration in browser environments
|
||||||
|
try {
|
||||||
|
if (typeof window !== 'undefined') {
|
||||||
|
await import('@tensorflow/tfjs-backend-webgl')
|
||||||
|
// Check if WebGL is available
|
||||||
|
try {
|
||||||
|
if (this.tf.setBackend) {
|
||||||
|
await this.tf.setBackend('webgl')
|
||||||
|
this.backend = 'webgl'
|
||||||
|
console.log('Using WebGL backend for TensorFlow.js')
|
||||||
|
} else {
|
||||||
|
console.warn(
|
||||||
|
'tf.setBackend is not available, falling back to CPU'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(
|
||||||
|
'WebGL backend not available, falling back to CPU:',
|
||||||
|
e
|
||||||
|
)
|
||||||
|
this.backend = 'cpu'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(
|
||||||
|
'WebGL backend not available, falling back to CPU:',
|
||||||
|
error
|
||||||
|
)
|
||||||
this.backend = 'cpu'
|
this.backend = 'cpu'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load Universal Sentence Encoder
|
||||||
|
this.use = await import(
|
||||||
|
'@tensorflow-models/universal-sentence-encoder'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('WebGL backend not available, falling back to CPU:', error)
|
console.error('Failed to initialize TensorFlow.js:', error)
|
||||||
this.backend = 'cpu'
|
throw error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set the backend
|
// Set the backend
|
||||||
|
|
@ -182,8 +193,6 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
|
||||||
await this.tf.setBackend(this.backend)
|
await this.tf.setBackend(this.backend)
|
||||||
}
|
}
|
||||||
|
|
||||||
this.use = await import('@tensorflow-models/universal-sentence-encoder')
|
|
||||||
|
|
||||||
// Log the module structure to help with debugging
|
// Log the module structure to help with debugging
|
||||||
console.log(
|
console.log(
|
||||||
'Universal Sentence Encoder module structure in main thread:',
|
'Universal Sentence Encoder module structure in main thread:',
|
||||||
|
|
|
||||||
|
|
@ -1,94 +1,224 @@
|
||||||
|
// In: @soulcraft/brainy/src/utils/textEncoding.ts
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unified Text Encoding Utilities
|
* Checks if the code is running in a Node.js environment.
|
||||||
|
*/
|
||||||
|
function isNode(): boolean {
|
||||||
|
return (
|
||||||
|
typeof process !== 'undefined' &&
|
||||||
|
process.versions != null &&
|
||||||
|
process.versions.node != null
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global flag to track if TensorFlow.js has been initialized
|
||||||
|
* This helps prevent multiple registrations of the same kernels
|
||||||
|
*/
|
||||||
|
const TENSORFLOW_INITIALIZED = Symbol('TENSORFLOW_INITIALIZED')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flag to track if the patch has been applied
|
||||||
|
* This prevents multiple applications of the patch
|
||||||
|
*/
|
||||||
|
let patchApplied = false
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CRITICAL: Applies a compatibility patch for TensorFlow.js when running in a modern
|
||||||
|
* Node.js ES Module environment. This must be called before any TensorFlow.js
|
||||||
|
* modules are imported.
|
||||||
*
|
*
|
||||||
* This module provides a consistent way to handle text encoding/decoding across all environments
|
* This function prevents the "TextEncoder is not a constructor" error by preemptively
|
||||||
* using the native TextEncoder/TextDecoder APIs.
|
* creating a compliant PlatformNode class with proper TextEncoder/TextDecoder support
|
||||||
*/
|
* and placing it on the global object where TensorFlow.js expects to find it.
|
||||||
|
*
|
||||||
/**
|
* The race condition occurs because TensorFlow.js's platform detection might run
|
||||||
* Get a text encoder that works in the current environment
|
* before the necessary global objects are properly initialized in certain Node.js
|
||||||
* @returns A TextEncoder instance
|
* environments, particularly when the package is being used by other applications.
|
||||||
*/
|
*
|
||||||
export function getTextEncoder(): TextEncoder {
|
* This function is called from setup.ts, which must be the first import in unified.ts
|
||||||
return new TextEncoder()
|
* to ensure the patch is applied before any TensorFlow.js code is executed.
|
||||||
}
|
*
|
||||||
|
* It also applies a patch to prevent duplicate kernel registrations when TensorFlow.js
|
||||||
/**
|
* is imported multiple times.
|
||||||
* Get a text decoder that works in the current environment
|
|
||||||
* @returns A TextDecoder instance
|
|
||||||
*/
|
|
||||||
export function getTextDecoder(): TextDecoder {
|
|
||||||
return new TextDecoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply the TensorFlow.js platform patch if needed
|
|
||||||
* This function patches the global object to provide a PlatformNode class
|
|
||||||
* that uses native TextEncoder/TextDecoder
|
|
||||||
*/
|
*/
|
||||||
export function applyTensorFlowPatch(): void {
|
export function applyTensorFlowPatch(): void {
|
||||||
try {
|
// Prevent multiple applications of the patch
|
||||||
// Define a custom Platform class that works in both Node.js and browser environments
|
if (patchApplied) {
|
||||||
class Platform {
|
return
|
||||||
util: any
|
}
|
||||||
textEncoder: TextEncoder
|
|
||||||
textDecoder: TextDecoder
|
|
||||||
|
|
||||||
constructor() {
|
if (!isNode()) {
|
||||||
// Create a util object with necessary methods and constructors
|
return // Patch is only for Node.js
|
||||||
this.util = {
|
}
|
||||||
// Use native TextEncoder and TextDecoder
|
|
||||||
TextEncoder: globalThis.TextEncoder || TextEncoder,
|
// In modern Node.js with ES Modules, TensorFlow.js can fail during its
|
||||||
TextDecoder: globalThis.TextDecoder || TextDecoder
|
// initial platform detection. This patch preempts that logic by creating
|
||||||
|
// a compliant "Platform" class that uses the standard global TextEncoder
|
||||||
|
// and placing it on the global object where TensorFlow.js expects to find it.
|
||||||
|
try {
|
||||||
|
// Ensure TextEncoder and TextDecoder are available
|
||||||
|
const nodeUtil = require('util')
|
||||||
|
const TextEncoderPolyfill = nodeUtil.TextEncoder || global.TextEncoder
|
||||||
|
const TextDecoderPolyfill = nodeUtil.TextDecoder || global.TextDecoder
|
||||||
|
|
||||||
|
if (!TextEncoderPolyfill || !TextDecoderPolyfill) {
|
||||||
|
console.warn(
|
||||||
|
'Brainy: TextEncoder or TextDecoder not available, attempting to polyfill'
|
||||||
|
)
|
||||||
|
|
||||||
|
// If still not available, try to use a simple polyfill
|
||||||
|
if (!TextEncoderPolyfill) {
|
||||||
|
class SimpleTextEncoder {
|
||||||
|
encode(input: string): Uint8Array {
|
||||||
|
const buf = Buffer.from(input, 'utf8')
|
||||||
|
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize using native constructors directly
|
global.TextEncoder = SimpleTextEncoder
|
||||||
this.textEncoder = new (globalThis.TextEncoder || TextEncoder)()
|
|
||||||
this.textDecoder = new (globalThis.TextDecoder || TextDecoder)()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Define isFloat32Array directly on the instance
|
if (!TextDecoderPolyfill) {
|
||||||
isFloat32Array(arr: any) {
|
class SimpleTextDecoder {
|
||||||
return !!(
|
decode(input?: Uint8Array): string {
|
||||||
|
if (!input) return ''
|
||||||
|
return Buffer.from(
|
||||||
|
input.buffer,
|
||||||
|
input.byteOffset,
|
||||||
|
input.byteLength
|
||||||
|
).toString('utf8')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
global.TextDecoder = SimpleTextDecoder
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Ensure they're available globally
|
||||||
|
global.TextEncoder = TextEncoderPolyfill
|
||||||
|
global.TextDecoder = TextDecoderPolyfill
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a PlatformNode implementation that uses the polyfilled TextEncoder/TextDecoder
|
||||||
|
class BrainyPlatformNode {
|
||||||
|
// Use the polyfilled TextEncoder/TextDecoder
|
||||||
|
readonly util = {
|
||||||
|
TextEncoder: global.TextEncoder,
|
||||||
|
TextDecoder: global.TextDecoder,
|
||||||
|
|
||||||
|
// Add utility functions that TensorFlow.js might need
|
||||||
|
isTypedArray: (arr: any): boolean => {
|
||||||
|
return ArrayBuffer.isView(arr) && !(arr instanceof DataView)
|
||||||
|
},
|
||||||
|
|
||||||
|
isFloat32Array: (arr: any): boolean => {
|
||||||
|
return (
|
||||||
|
arr instanceof Float32Array ||
|
||||||
|
(arr &&
|
||||||
|
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create instances of the encoder/decoder
|
||||||
|
readonly textEncoder: any
|
||||||
|
readonly textDecoder: any
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
try {
|
||||||
|
// Initialize encoders using constructors
|
||||||
|
this.textEncoder = new global.TextEncoder()
|
||||||
|
this.textDecoder = new global.TextDecoder()
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(
|
||||||
|
'Brainy: Error creating TextEncoder/TextDecoder instances:',
|
||||||
|
e
|
||||||
|
)
|
||||||
|
// Provide fallback implementations if instantiation fails
|
||||||
|
this.textEncoder = {
|
||||||
|
encode: (input: string): Uint8Array => {
|
||||||
|
const buf = Buffer.from(input, 'utf8')
|
||||||
|
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.textDecoder = {
|
||||||
|
decode: (input?: Uint8Array): string => {
|
||||||
|
if (!input) return ''
|
||||||
|
return Buffer.from(
|
||||||
|
input.buffer,
|
||||||
|
input.byteOffset,
|
||||||
|
input.byteLength
|
||||||
|
).toString('utf8')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isTypedArray(arr: any): arr is Float32Array | Int32Array | Uint8Array {
|
||||||
|
return ArrayBuffer.isView(arr) && !(arr instanceof DataView)
|
||||||
|
}
|
||||||
|
|
||||||
|
isFloat32Array(arr: any): arr is Float32Array {
|
||||||
|
return (
|
||||||
arr instanceof Float32Array ||
|
arr instanceof Float32Array ||
|
||||||
(arr &&
|
(arr &&
|
||||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Define isTypedArray directly on the instance
|
|
||||||
isTypedArray(arr: any) {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the global object in a way that works in both Node.js and browser
|
// Assign the custom platform class to the global scope.
|
||||||
const globalObj =
|
// TensorFlow.js specifically looks for `PlatformNode`.
|
||||||
typeof global !== 'undefined'
|
global.PlatformNode = BrainyPlatformNode
|
||||||
? global
|
|
||||||
: typeof window !== 'undefined'
|
|
||||||
? window
|
|
||||||
: typeof self !== 'undefined'
|
|
||||||
? self
|
|
||||||
: {}
|
|
||||||
|
|
||||||
// Only apply in Node.js environment
|
// Also create an instance and assign it to global.platformNode (lowercase p)
|
||||||
if (
|
// This is needed for some TensorFlow.js versions
|
||||||
typeof process !== 'undefined' &&
|
global.platformNode = new BrainyPlatformNode()
|
||||||
process.versions &&
|
|
||||||
process.versions.node
|
// Set up a global flag to track TensorFlow.js initialization
|
||||||
) {
|
global[TENSORFLOW_INITIALIZED] = false
|
||||||
// Assign the Platform class to the global object as PlatformNode for Node.js
|
|
||||||
;(globalObj as any).PlatformNode = Platform
|
// Monkey patch the registerKernel function to prevent duplicate registrations
|
||||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
// This will be applied when TensorFlow.js is imported
|
||||||
;(globalObj as any).platformNode = new Platform()
|
const originalRegisterKernel = global.registerKernel
|
||||||
} else if (typeof window !== 'undefined' || typeof self !== 'undefined') {
|
if (!originalRegisterKernel) {
|
||||||
// In browser environments, we might need to provide similar functionality
|
// Set up a handler to intercept the registerKernel function when it's defined
|
||||||
// but we'll use a different name to avoid conflicts
|
Object.defineProperty(global, 'registerKernel', {
|
||||||
;(globalObj as any).PlatformBrowser = Platform
|
set: function (newRegisterKernel) {
|
||||||
;(globalObj as any).platformBrowser = new Platform()
|
// Replace the setter with our patched version
|
||||||
|
Object.defineProperty(global, 'registerKernel', {
|
||||||
|
value: function (kernel: any) {
|
||||||
|
// Check if this kernel is already registered
|
||||||
|
const kernelName = kernel.kernelName
|
||||||
|
const backendName = kernel.backendName
|
||||||
|
const key = `${kernelName}_${backendName}`
|
||||||
|
|
||||||
|
// Use a global registry to track registered kernels
|
||||||
|
if (!global.__REGISTERED_KERNELS__) {
|
||||||
|
global.__REGISTERED_KERNELS__ = new Set()
|
||||||
|
}
|
||||||
|
|
||||||
|
// If this kernel is already registered, skip it
|
||||||
|
if (global.__REGISTERED_KERNELS__.has(key)) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise, register it and add it to our registry
|
||||||
|
global.__REGISTERED_KERNELS__.add(key)
|
||||||
|
return newRegisterKernel(kernel)
|
||||||
|
},
|
||||||
|
configurable: true,
|
||||||
|
writable: true
|
||||||
|
})
|
||||||
|
},
|
||||||
|
configurable: true
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Mark the patch as applied
|
||||||
|
patchApplied = true
|
||||||
|
console.log('Brainy: Successfully applied TensorFlow.js platform patch')
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('Failed to apply TensorFlow.js platform patch:', error)
|
console.warn('Brainy: Failed to apply TensorFlow.js platform patch:', error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
// Test script to verify that the function string format works with the fallback mechanism
|
|
||||||
import { executeInThread } from './dist/unified.js'
|
|
||||||
|
|
||||||
// Define a compute-intensive function using a named function declaration
|
|
||||||
// followed by a statement that returns the function
|
|
||||||
const computeIntensiveFunction = `
|
|
||||||
// Define a named function
|
|
||||||
function computeTask(data) {
|
|
||||||
console.log('Worker/Fallback: Starting computation...');
|
|
||||||
|
|
||||||
// Simulate a compute-intensive task
|
|
||||||
const start = Date.now();
|
|
||||||
let result = 0;
|
|
||||||
for (let i = 0; i < data.iterations; i++) {
|
|
||||||
result += Math.sqrt(i) * Math.sin(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
const duration = Date.now() - start;
|
|
||||||
console.log('Worker/Fallback: Computation completed in ' + duration + 'ms');
|
|
||||||
|
|
||||||
return {
|
|
||||||
result,
|
|
||||||
duration,
|
|
||||||
iterations: data.iterations
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return the function
|
|
||||||
computeTask;
|
|
||||||
`
|
|
||||||
|
|
||||||
// Test with different environments
|
|
||||||
async function runTests() {
|
|
||||||
try {
|
|
||||||
console.log('Testing executeInThread with fallback...')
|
|
||||||
|
|
||||||
// Disable Web Workers to force fallback
|
|
||||||
const originalWorker = globalThis.Worker
|
|
||||||
globalThis.Worker = function() {
|
|
||||||
throw new Error('Worker constructor disabled for testing')
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Execute the function in fallback mode
|
|
||||||
const result = await executeInThread(computeIntensiveFunction, {
|
|
||||||
iterations: 1000000
|
|
||||||
})
|
|
||||||
console.log('Fallback result:', result)
|
|
||||||
console.log('Test passed!')
|
|
||||||
} finally {
|
|
||||||
// Restore Web Workers
|
|
||||||
globalThis.Worker = originalWorker
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Test failed:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runTests()
|
|
||||||
|
|
@ -1,52 +0,0 @@
|
||||||
// Test script to verify that the function string format works with the fallback mechanism
|
|
||||||
import { executeInThread } from './dist/unified.js'
|
|
||||||
|
|
||||||
// Define a compute-intensive function using a simple anonymous function expression
|
|
||||||
const computeIntensiveFunction = `function(data) {
|
|
||||||
console.log('Worker/Fallback: Starting computation...');
|
|
||||||
|
|
||||||
// Simulate a compute-intensive task
|
|
||||||
const start = Date.now();
|
|
||||||
let result = 0;
|
|
||||||
for (let i = 0; i < data.iterations; i++) {
|
|
||||||
result += Math.sqrt(i) * Math.sin(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
const duration = Date.now() - start;
|
|
||||||
console.log('Worker/Fallback: Computation completed in ' + duration + 'ms');
|
|
||||||
|
|
||||||
return {
|
|
||||||
result,
|
|
||||||
duration,
|
|
||||||
iterations: data.iterations
|
|
||||||
};
|
|
||||||
}`
|
|
||||||
|
|
||||||
// Test with different environments
|
|
||||||
async function runTests() {
|
|
||||||
try {
|
|
||||||
console.log('Testing executeInThread with fallback...')
|
|
||||||
|
|
||||||
// Disable Web Workers to force fallback
|
|
||||||
const originalWorker = globalThis.Worker
|
|
||||||
globalThis.Worker = function() {
|
|
||||||
throw new Error('Worker constructor disabled for testing')
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Execute the function in fallback mode
|
|
||||||
const result = await executeInThread(computeIntensiveFunction, {
|
|
||||||
iterations: 1000000
|
|
||||||
})
|
|
||||||
console.log('Fallback result:', result)
|
|
||||||
console.log('Test passed!')
|
|
||||||
} finally {
|
|
||||||
// Restore Web Workers
|
|
||||||
globalThis.Worker = originalWorker
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Test failed:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runTests()
|
|
||||||
47
test-fix.js
47
test-fix.js
|
|
@ -1,47 +0,0 @@
|
||||||
// Test script to verify the TextEncoder fix
|
|
||||||
import { applyTensorFlowPatch } from './dist/utils/textEncoding.js'
|
|
||||||
|
|
||||||
console.log('Testing TextEncoder fix...')
|
|
||||||
|
|
||||||
// Apply the TensorFlow.js platform patch
|
|
||||||
applyTensorFlowPatch()
|
|
||||||
|
|
||||||
// Check if PlatformNode is defined in the global object
|
|
||||||
if (typeof global.PlatformNode === 'function') {
|
|
||||||
console.log('PlatformNode is defined in the global object')
|
|
||||||
|
|
||||||
// Create an instance of PlatformNode
|
|
||||||
try {
|
|
||||||
const platform = new global.PlatformNode()
|
|
||||||
console.log('Successfully created PlatformNode instance')
|
|
||||||
|
|
||||||
// Check if textEncoder is defined
|
|
||||||
if (platform.textEncoder) {
|
|
||||||
console.log('textEncoder is defined')
|
|
||||||
|
|
||||||
// Test encoding a string
|
|
||||||
const testString = 'Hello, world! 👋'
|
|
||||||
const encoded = platform.textEncoder.encode(testString)
|
|
||||||
console.log(`Successfully encoded string: ${testString}`)
|
|
||||||
console.log(`Encoded: [${encoded}]`)
|
|
||||||
|
|
||||||
// Test decoding
|
|
||||||
const decoded = platform.textDecoder.decode(encoded)
|
|
||||||
console.log(`Successfully decoded back to: ${decoded}`)
|
|
||||||
|
|
||||||
if (testString === decoded) {
|
|
||||||
console.log('✅ TextEncoder/TextDecoder test passed!')
|
|
||||||
} else {
|
|
||||||
console.error('❌ TextEncoder/TextDecoder test failed!')
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
console.error('textEncoder is not defined in the platform instance')
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error creating PlatformNode instance:', error)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
console.error('PlatformNode is not defined in the global object')
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('Test completed')
|
|
||||||
|
|
@ -1,103 +0,0 @@
|
||||||
// Test script to verify TensorFlow.js and TextEncoder functionality in Node.js environment
|
|
||||||
import * as tf from '@tensorflow/tfjs'
|
|
||||||
import '@tensorflow/tfjs-backend-cpu'
|
|
||||||
import { TextEncoder, TextDecoder } from 'util'
|
|
||||||
|
|
||||||
// Implement the necessary functions directly
|
|
||||||
function applyTensorFlowPatch() {
|
|
||||||
// This is a simplified version of the patch
|
|
||||||
console.log('Applying TensorFlow patch directly in test file')
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
function getTextEncoder() {
|
|
||||||
return new TextEncoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
function getTextDecoder() {
|
|
||||||
return new TextDecoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testTensorFlowAndTextEncoder() {
|
|
||||||
console.log('Testing TensorFlow.js and TextEncoder in Node.js environment...')
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Apply TensorFlow patch for TextEncoder compatibility
|
|
||||||
applyTensorFlowPatch()
|
|
||||||
console.log('TensorFlow patch applied successfully')
|
|
||||||
|
|
||||||
// Test TextEncoder
|
|
||||||
console.log('\n--- Testing TextEncoder ---')
|
|
||||||
const encoder = getTextEncoder()
|
|
||||||
const decoder = getTextDecoder()
|
|
||||||
|
|
||||||
const testString = 'Hello, world! 👋'
|
|
||||||
console.log(`Original string: "${testString}"`)
|
|
||||||
|
|
||||||
const encoded = encoder.encode(testString)
|
|
||||||
console.log(`Encoded: [${encoded}]`)
|
|
||||||
|
|
||||||
const decoded = decoder.decode(encoded)
|
|
||||||
console.log(`Decoded: "${decoded}"`)
|
|
||||||
|
|
||||||
if (testString === decoded) {
|
|
||||||
console.log('✅ TextEncoder/TextDecoder test passed!')
|
|
||||||
} else {
|
|
||||||
console.error('❌ TextEncoder/TextDecoder test failed!')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test TensorFlow.js
|
|
||||||
console.log('\n--- Testing TensorFlow.js ---')
|
|
||||||
|
|
||||||
// Create a simple tensor
|
|
||||||
const tensor = tf.tensor2d([
|
|
||||||
[1, 2],
|
|
||||||
[3, 4]
|
|
||||||
])
|
|
||||||
console.log('Created tensor:')
|
|
||||||
tensor.print()
|
|
||||||
|
|
||||||
// Perform a simple operation
|
|
||||||
const result = tensor.add(tf.scalar(1))
|
|
||||||
console.log('Result of adding 1:')
|
|
||||||
result.print()
|
|
||||||
|
|
||||||
// Check the values
|
|
||||||
const values = await result.array()
|
|
||||||
const expected = [
|
|
||||||
[2, 3],
|
|
||||||
[4, 5]
|
|
||||||
]
|
|
||||||
|
|
||||||
console.log('Result values:', values)
|
|
||||||
console.log('Expected values:', expected)
|
|
||||||
|
|
||||||
// Compare values
|
|
||||||
const match = JSON.stringify(values) === JSON.stringify(expected)
|
|
||||||
if (match) {
|
|
||||||
console.log('✅ TensorFlow.js test passed!')
|
|
||||||
} else {
|
|
||||||
console.error('❌ TensorFlow.js test failed!')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('\nAll tests passed successfully!')
|
|
||||||
return true
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error during test:', error)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the test
|
|
||||||
testTensorFlowAndTextEncoder().then((success) => {
|
|
||||||
if (success) {
|
|
||||||
console.log(
|
|
||||||
'TensorFlow.js and TextEncoder verification completed successfully!'
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
console.error('TensorFlow.js and TextEncoder verification failed!')
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
@ -1,39 +0,0 @@
|
||||||
// Test script to verify the unified text encoding approach works correctly
|
|
||||||
import { BrainyData } from './dist/unified.js'
|
|
||||||
|
|
||||||
async function testUnifiedEncoding() {
|
|
||||||
console.log(
|
|
||||||
'Testing unified text encoding approach in Node.js environment...'
|
|
||||||
)
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Initialize BrainyData which should trigger the PlatformNode constructor
|
|
||||||
console.log('Creating BrainyData instance...')
|
|
||||||
const db = new BrainyData()
|
|
||||||
|
|
||||||
// Initialize the database
|
|
||||||
console.log('Initializing database...')
|
|
||||||
await db.init()
|
|
||||||
|
|
||||||
console.log('Test successful! Unified text encoding is working correctly.')
|
|
||||||
|
|
||||||
// Get database status to verify everything is working
|
|
||||||
const status = await db.status()
|
|
||||||
console.log('Database status:', status)
|
|
||||||
|
|
||||||
return true
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error during test:', error)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the test
|
|
||||||
testUnifiedEncoding().then((success) => {
|
|
||||||
if (success) {
|
|
||||||
console.log('Unified text encoding verification completed successfully!')
|
|
||||||
} else {
|
|
||||||
console.error('Unified text encoding verification failed!')
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
// Test script to verify that the workerUtils functions work correctly after removing eval
|
|
||||||
import { executeInThread } from './dist/unified.js'
|
|
||||||
|
|
||||||
// Test function to execute in a thread
|
|
||||||
const testFunction = `function(args) {
|
|
||||||
return "Hello from " + args.name;
|
|
||||||
}`
|
|
||||||
|
|
||||||
// Test with different environments
|
|
||||||
async function runTests() {
|
|
||||||
try {
|
|
||||||
console.log('Testing executeInThread...')
|
|
||||||
const result = await executeInThread(testFunction, {
|
|
||||||
name: 'Worker Thread'
|
|
||||||
})
|
|
||||||
console.log('Result:', result)
|
|
||||||
|
|
||||||
console.log('All tests passed!')
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Test failed:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runTests()
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue