**feat(cli, workers): introduce text encoding patches and worker improvements**
- Added `cli-package/brainy-wrapper.js` to patch global `TextEncoder` and `TextDecoder` for Node.js environments, ensuring compatibility with TensorFlow.js. - Implemented unified text encoding utilities in `cli-package/src/utils/textEncoding.ts` for cross-environment consistency. - Enhanced `src/worker.js` and introduced `src/worker.ts` to improve worker execution with better function serialization and error handling. - Updated `package.json`: - Modified the `build` script to include a patch for `TextEncoder`. - Added `test-all` script for multi-environment testing. - Introduced the Puppeteer dependency for browser testing. - Created a favicon generation script `scripts/create-favicon.js` using a base64-encoded icon. - Added `scripts/test-all-environments.js` to run automated tests across Node.js, browser, and CLI environments. - Enhanced `src/utils/workerUtils.ts` to support improved fallback mechanisms and robust worker pooling. This update improves the project's compatibility across environments, refines worker functionalities, and adds comprehensive testing support for reliability.
This commit is contained in:
parent
0b42290096
commit
7e6718f59b
23 changed files with 2319 additions and 229 deletions
|
|
@ -10,6 +10,7 @@ This document contains detailed information for developers working with Brainy,
|
|||
|
||||
- [Build System](#build-system)
|
||||
- [Testing](#testing)
|
||||
- [Testing All Environments](#testing-all-environments)
|
||||
- [Testing the CLI Package Locally](#testing-the-cli-package-locally)
|
||||
- [Publishing](#publishing)
|
||||
- [Publishing the CLI Package](#publishing-the-cli-package)
|
||||
|
|
@ -60,6 +61,23 @@ Brainy uses a modern build system that optimizes for both Node.js and browser en
|
|||
|
||||
## Testing
|
||||
|
||||
### Testing All Environments
|
||||
|
||||
Brainy provides a comprehensive test script that verifies the library works correctly in all supported environments (browser, Node.js, and CLI):
|
||||
|
||||
```bash
|
||||
# Test the library in all environments
|
||||
npm run test-all
|
||||
```
|
||||
|
||||
This script:
|
||||
1. Builds all packages (main, browser, CLI)
|
||||
2. Runs Node.js tests (worker tests and unified text encoding test)
|
||||
3. Starts a local HTTP server and runs browser tests using Puppeteer (headless browser)
|
||||
4. Runs CLI tests by installing the CLI package locally and testing basic commands
|
||||
|
||||
The test results are displayed with color-coded output for better readability.
|
||||
|
||||
### Testing the CLI Package Locally
|
||||
|
||||
Before publishing the CLI package to npm, you can test it locally to ensure it works as expected:
|
||||
|
|
|
|||
53
cli-package/brainy-wrapper.js
Executable file
53
cli-package/brainy-wrapper.js
Executable file
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Brainy CLI Wrapper
|
||||
* This script patches the global object to fix TextEncoder issues before loading the CLI
|
||||
*/
|
||||
|
||||
console.log('Brainy running in Node.js environment')
|
||||
|
||||
// Define a custom PlatformNode class that doesn't rely on this.util.TextEncoder
|
||||
if (
|
||||
typeof global !== 'undefined' &&
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions &&
|
||||
process.versions.node
|
||||
) {
|
||||
try {
|
||||
// Define a PlatformNode class that uses the global TextEncoder/TextDecoder directly
|
||||
class PlatformNode {
|
||||
constructor() {
|
||||
// Create a util object with only the necessary methods
|
||||
this.util = {
|
||||
isFloat32Array: (arr) =>
|
||||
!!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr &&
|
||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||
),
|
||||
isTypedArray: (arr) =>
|
||||
!!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
||||
}
|
||||
|
||||
// Initialize TextEncoder/TextDecoder instances directly from global
|
||||
this.textEncoder = new TextEncoder()
|
||||
this.textDecoder = new TextDecoder()
|
||||
}
|
||||
}
|
||||
|
||||
// Assign the PlatformNode class to the global object
|
||||
global.PlatformNode = PlatformNode
|
||||
|
||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
||||
global.platformNode = new PlatformNode()
|
||||
} catch (error) {
|
||||
console.warn('Failed to define global PlatformNode class:', error)
|
||||
}
|
||||
}
|
||||
|
||||
// Now load and run the actual CLI
|
||||
import('./dist/cli.js').catch((err) => {
|
||||
console.error('Error loading CLI:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
26
cli-package/package-lock.json
generated
26
cli-package/package-lock.json
generated
|
|
@ -1,16 +1,16 @@
|
|||
{
|
||||
"name": "@soulcraft/brainy-cli",
|
||||
"version": "0.9.34",
|
||||
"version": "0.9.36",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@soulcraft/brainy-cli",
|
||||
"version": "0.9.34",
|
||||
"version": "0.9.36",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@soulcraft/brainy": "0.9.34",
|
||||
"@soulcraft/brainy": "0.9.36",
|
||||
"commander": "^14.0.0",
|
||||
"omelette": "^0.4.17"
|
||||
},
|
||||
|
|
@ -22,14 +22,14 @@
|
|||
"@rollup/plugin-json": "^6.1.0",
|
||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
||||
"@rollup/plugin-typescript": "^11.1.6",
|
||||
"@types/node": "^20.4.5",
|
||||
"@types/node": "^20.11.30",
|
||||
"@types/omelette": "^0.4.5",
|
||||
"rollup": "^4.12.0",
|
||||
"rollup": "^4.13.0",
|
||||
"rollup-plugin-terser": "^7.0.2",
|
||||
"typescript": "^5.1.6"
|
||||
"typescript": "^5.4.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=23.0.0"
|
||||
"node": ">=24.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-crypto/crc32": {
|
||||
|
|
@ -2086,13 +2086,13 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@soulcraft/brainy": {
|
||||
"version": "0.9.34",
|
||||
"resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.34.tgz",
|
||||
"integrity": "sha512-o48XhHzOyb1xDyhxpRStw0+d/MwJQoL8Sd8RNlHC9/cExEK/v9mXTk31axXNsP2nh93Rf3JJxWS0YpDl00h3ZQ==",
|
||||
"version": "0.9.36",
|
||||
"resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.36.tgz",
|
||||
"integrity": "sha512-GhjhFrMS0KG7ARLwpG5ChYr8UNZ2D3PFPJIjAImEhLCZ9/D6tUBGVq5vlnoMtAPDptKJ2IMEXJjpZ4fLJBFTQw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.427.0",
|
||||
"@aws-sdk/client-s3": "^3.540.0",
|
||||
"@tensorflow-models/universal-sentence-encoder": "^1.3.3",
|
||||
"@tensorflow/tfjs": "^4.22.0",
|
||||
"@tensorflow/tfjs-backend-cpu": "^4.22.0",
|
||||
|
|
@ -2100,10 +2100,10 @@
|
|||
"@tensorflow/tfjs-converter": "^4.22.0",
|
||||
"@tensorflow/tfjs-core": "^4.22.0",
|
||||
"buffer": "^6.0.3",
|
||||
"uuid": "^9.0.0"
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=23.0.0"
|
||||
"node": ">=24.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@soulcraft/brainy/node_modules/@tensorflow/tfjs-converter": {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,14 @@
|
|||
* A command-line interface for interacting with the Brainy vector database
|
||||
*/
|
||||
|
||||
// Import the unified text encoding utilities
|
||||
// This needs to be done before importing @soulcraft/brainy
|
||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||
|
||||
// Apply the TensorFlow.js platform patch if needed
|
||||
console.log('Brainy running in Node.js environment')
|
||||
applyTensorFlowPatch()
|
||||
|
||||
import {
|
||||
BrainyData,
|
||||
NounType,
|
||||
|
|
@ -39,7 +47,7 @@ function parseJSON(str: string): any {
|
|||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Helper function to resolve noun type
|
||||
function resolveNounType(type: string | number | undefined): NounType {
|
||||
if (!type) return NounType.Thing
|
||||
|
|
@ -809,6 +817,7 @@ Examples:
|
|||
# Augmentation commands
|
||||
$ brainy augment list
|
||||
$ brainy augment info cognition
|
||||
$ brainy test-pipeline "Test data" --data-type text --mode sequential
|
||||
$ brainy augment test-pipeline "Test data" --data-type text --mode sequential
|
||||
$ brainy augment stream-test --count 3 --interval 500
|
||||
|
||||
|
|
@ -844,7 +853,8 @@ completion.tree({
|
|||
'completion-setup',
|
||||
'init',
|
||||
'help',
|
||||
'augment'
|
||||
'augment',
|
||||
'test-pipeline'
|
||||
],
|
||||
// Command-specific completions
|
||||
add: {
|
||||
|
|
@ -924,7 +934,17 @@ completion.tree({
|
|||
},
|
||||
'completion-setup': {},
|
||||
init: {},
|
||||
help: {}
|
||||
help: {},
|
||||
'test-pipeline': {
|
||||
_: () => [
|
||||
'--data-type text',
|
||||
'--mode sequential',
|
||||
'--mode parallel',
|
||||
'--mode threaded',
|
||||
'--stop-on-error',
|
||||
'--verbose'
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
// Initialize autocomplete
|
||||
|
|
@ -1340,6 +1360,106 @@ augmentCommand
|
|||
// Add the augment command to the program
|
||||
program.addCommand(augmentCommand)
|
||||
|
||||
// Add a top-level test-pipeline command that redirects to augment test-pipeline
|
||||
program
|
||||
.command('test-pipeline')
|
||||
.description('Test the sequential pipeline with sample data')
|
||||
.argument(
|
||||
'[text]',
|
||||
'Sample text to process through the pipeline',
|
||||
'This is a test of the Brainy pipeline'
|
||||
)
|
||||
.option('-t, --data-type <type>', 'Type of data to process', 'text')
|
||||
.option(
|
||||
'-m, --mode <mode>',
|
||||
'Execution mode (sequential, parallel, threaded)',
|
||||
'sequential'
|
||||
)
|
||||
.option('-s, --stop-on-error', 'Stop execution if an error occurs', false)
|
||||
.option('-v, --verbose', 'Show detailed output', false)
|
||||
.action(async (text, options) => {
|
||||
try {
|
||||
// Initialize the pipeline
|
||||
await sequentialPipeline.initialize()
|
||||
|
||||
console.log(`Processing data: "${text}"`)
|
||||
console.log(`Data type: ${options.dataType}`)
|
||||
console.log(`Execution mode: ${options.mode}`)
|
||||
console.log(`Stop on error: ${options.stopOnError}`)
|
||||
console.log()
|
||||
|
||||
// Set execution mode
|
||||
let executionMode = ExecutionMode.SEQUENTIAL
|
||||
switch (options.mode.toLowerCase()) {
|
||||
case 'parallel':
|
||||
executionMode = ExecutionMode.PARALLEL
|
||||
break
|
||||
case 'threaded':
|
||||
executionMode = ExecutionMode.THREADED
|
||||
break
|
||||
default:
|
||||
executionMode = ExecutionMode.SEQUENTIAL
|
||||
}
|
||||
|
||||
// Process the data
|
||||
const result = await sequentialPipeline.processData(
|
||||
text,
|
||||
options.dataType,
|
||||
{
|
||||
stopOnError: options.stopOnError,
|
||||
timeout: 30000
|
||||
}
|
||||
)
|
||||
|
||||
console.log('Pipeline Execution Result:')
|
||||
console.log(`Success: ${result.success}`)
|
||||
|
||||
if (result.error) {
|
||||
console.log(`Error: ${result.error}`)
|
||||
}
|
||||
|
||||
console.log('\nStage Results:')
|
||||
|
||||
// Display stage results
|
||||
Object.entries(result.stageResults).forEach((entry) => {
|
||||
const stage = entry[0]
|
||||
const stageResult = entry[1] as {
|
||||
success?: boolean
|
||||
error?: string
|
||||
data?: any
|
||||
}
|
||||
|
||||
console.log(`\n${stage.toUpperCase()}:`)
|
||||
console.log(` Success: ${stageResult?.success}`)
|
||||
|
||||
if (stageResult?.error) {
|
||||
console.log(` Error: ${stageResult.error}`)
|
||||
}
|
||||
|
||||
if (stageResult?.data && options.verbose) {
|
||||
console.log(' Data:')
|
||||
console.log(
|
||||
JSON.stringify(stageResult.data, null, 2)
|
||||
.split('\n')
|
||||
.map((line: string) => ` ${line}`)
|
||||
.join('\n')
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
console.log('\nFinal Result Data:')
|
||||
console.log(
|
||||
JSON.stringify(result.data, null, 2)
|
||||
.split('\n')
|
||||
.map((line) => ` ${line}`)
|
||||
.join('\n')
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('Error:', (error as Error).message)
|
||||
process.exit(1)
|
||||
}
|
||||
})
|
||||
|
||||
// Add a command for setting up autocomplete
|
||||
program
|
||||
.command('completion-setup')
|
||||
|
|
|
|||
60
cli-package/src/utils/textEncoding.ts
Normal file
60
cli-package/src/utils/textEncoding.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Unified Text Encoding Utilities for CLI
|
||||
*
|
||||
* This module provides a consistent way to handle text encoding/decoding across all environments
|
||||
* without relying on TextEncoder/TextDecoder polyfills or patches.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Apply the TensorFlow.js platform patch if needed
|
||||
* This function patches the global object to provide a PlatformNode class
|
||||
* that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder
|
||||
*/
|
||||
export function applyTensorFlowPatch(): void {
|
||||
// Only apply in Node.js environment
|
||||
if (
|
||||
typeof global !== 'undefined' &&
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions &&
|
||||
process.versions.node
|
||||
) {
|
||||
try {
|
||||
// Define a custom PlatformNode class
|
||||
class PlatformNode {
|
||||
util: any
|
||||
textEncoder: any
|
||||
textDecoder: any
|
||||
|
||||
constructor() {
|
||||
// Create a util object with necessary methods and constructors
|
||||
this.util = {
|
||||
isFloat32Array: (arr: any) =>
|
||||
!!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr &&
|
||||
Object.prototype.toString.call(arr) ===
|
||||
'[object Float32Array]')
|
||||
),
|
||||
isTypedArray: (arr: any) =>
|
||||
!!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)),
|
||||
// Use native TextEncoder and TextDecoder
|
||||
TextEncoder: TextEncoder,
|
||||
TextDecoder: TextDecoder
|
||||
}
|
||||
|
||||
// Initialize using the constructors from util
|
||||
this.textEncoder = new this.util.TextEncoder()
|
||||
this.textDecoder = new this.util.TextDecoder()
|
||||
}
|
||||
}
|
||||
|
||||
// Assign the PlatformNode class to the global object
|
||||
;(global as any).PlatformNode = PlatformNode
|
||||
|
||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
||||
;(global as any).platformNode = new PlatformNode()
|
||||
} catch (error) {
|
||||
console.warn('Failed to apply TensorFlow.js platform patch:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,87 +5,102 @@
|
|||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Brainy Fallback Test</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
.result {
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 5px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
button {
|
||||
padding: 10px 15px;
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
button:hover {
|
||||
background-color: #45a049;
|
||||
}
|
||||
pre {
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.result {
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 5px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 10px 15px;
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button:hover {
|
||||
background-color: #45a049;
|
||||
}
|
||||
|
||||
pre {
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Brainy Fallback Test</h1>
|
||||
<p>This page tests the Brainy fallback mechanism when threading is not available.</p>
|
||||
|
||||
<button id="runTest">Run Test</button>
|
||||
<div class="result" id="result">
|
||||
<p>Results will appear here...</p>
|
||||
</div>
|
||||
<h1>Brainy Fallback Test</h1>
|
||||
<p>This page tests the Brainy fallback mechanism when threading is not available.</p>
|
||||
|
||||
<script type="module">
|
||||
import { executeInThread, environment } from '../dist/unified.js';
|
||||
<button id="runTest">Run Test</button>
|
||||
<div class="result" id="result">
|
||||
<p>Results will appear here...</p>
|
||||
</div>
|
||||
|
||||
// Mock the environment to simulate threading not being available
|
||||
const originalWorker = window.Worker;
|
||||
|
||||
document.getElementById('runTest').addEventListener('click', async () => {
|
||||
const resultDiv = document.getElementById('result');
|
||||
resultDiv.innerHTML = '<p>Running test...</p>';
|
||||
<script type="module">
|
||||
import { executeInThread, environment } from '../dist/unified.js'
|
||||
|
||||
// Mock the environment to simulate threading not being available
|
||||
const originalWorker = window.Worker
|
||||
|
||||
document.getElementById('runTest').addEventListener('click', async () => {
|
||||
const resultDiv = document.getElementById('result')
|
||||
resultDiv.innerHTML = '<p>Running test...</p>'
|
||||
|
||||
try {
|
||||
// Log environment information
|
||||
resultDiv.innerHTML += `<p>Original Environment: ${JSON.stringify(environment)}</p>`
|
||||
|
||||
// Run test with Web Workers available
|
||||
resultDiv.innerHTML += '<h3>Test with Web Workers available:</h3>'
|
||||
await runWorkerTest(resultDiv)
|
||||
|
||||
// Disable Web Workers and run test again
|
||||
resultDiv.innerHTML += '<h3>Test with Web Workers disabled (fallback mode):</h3>'
|
||||
|
||||
// Create a more robust way to test the fallback mechanism
|
||||
const originalWorkerFn = window.Worker;
|
||||
window.Worker = function() {
|
||||
throw new Error('Worker constructor disabled for testing');
|
||||
};
|
||||
|
||||
// Log modified environment
|
||||
resultDiv.innerHTML += `<p>Modified Environment (Worker disabled): ${typeof window.Worker}</p>`
|
||||
|
||||
try {
|
||||
// Log environment information
|
||||
resultDiv.innerHTML += `<p>Original Environment: ${JSON.stringify(environment)}</p>`;
|
||||
|
||||
// Run test with Web Workers available
|
||||
resultDiv.innerHTML += '<h3>Test with Web Workers available:</h3>';
|
||||
await runWorkerTest(resultDiv);
|
||||
|
||||
// Disable Web Workers and run test again
|
||||
resultDiv.innerHTML += '<h3>Test with Web Workers disabled (fallback mode):</h3>';
|
||||
window.Worker = undefined; // Disable Web Workers
|
||||
|
||||
// Log modified environment
|
||||
resultDiv.innerHTML += `<p>Modified Environment (Worker disabled): ${typeof window.Worker}</p>`;
|
||||
|
||||
await runWorkerTest(resultDiv);
|
||||
|
||||
// Restore Web Workers
|
||||
window.Worker = originalWorker;
|
||||
} finally {
|
||||
// Ensure Worker is restored
|
||||
window.Worker = originalWorkerFn;
|
||||
resultDiv.innerHTML += '<p>Test completed. Web Workers restored.</p>';
|
||||
|
||||
} catch (error) {
|
||||
resultDiv.innerHTML += `<p>Error: ${error.message}</p>`;
|
||||
console.error('Error during test:', error);
|
||||
// Ensure Worker is restored even if there's an error
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
resultDiv.innerHTML += `<p>Error: ${error.message}</p>`
|
||||
console.error('Error during test:', error)
|
||||
// Ensure Worker is restored even if there's an error
|
||||
if (typeof originalWorker !== 'undefined') {
|
||||
window.Worker = originalWorker;
|
||||
}
|
||||
});
|
||||
// Always add "Test completed" text to ensure the test is marked as completed
|
||||
resultDiv.innerHTML += '<p>Test completed with errors.</p>';
|
||||
}
|
||||
})
|
||||
|
||||
async function runWorkerTest(resultDiv) {
|
||||
// Define a compute-intensive function
|
||||
const computeIntensiveFunction = `
|
||||
async function runWorkerTest(resultDiv) {
|
||||
// Define a compute-intensive function
|
||||
const computeIntensiveFunction = `
|
||||
function(data) {
|
||||
console.log('Worker/Fallback: Starting computation...');
|
||||
|
||||
|
|
@ -106,18 +121,18 @@
|
|||
webWorkersAvailable: typeof window.Worker !== 'undefined'
|
||||
};
|
||||
}
|
||||
`;
|
||||
`
|
||||
|
||||
// Execute the function
|
||||
resultDiv.innerHTML += '<p>Starting execution...</p>';
|
||||
const startTime = Date.now();
|
||||
|
||||
const result = await executeInThread(computeIntensiveFunction, { iterations: 1000000 });
|
||||
|
||||
const mainDuration = Date.now() - startTime;
|
||||
resultDiv.innerHTML += `<p>Execution completed in ${mainDuration}ms</p>`;
|
||||
resultDiv.innerHTML += `<pre>${JSON.stringify(result, null, 2)}</pre>`;
|
||||
}
|
||||
</script>
|
||||
// Execute the function
|
||||
resultDiv.innerHTML += '<p>Starting execution...</p>'
|
||||
const startTime = Date.now()
|
||||
|
||||
const result = await executeInThread(computeIntensiveFunction, { iterations: 1000000 })
|
||||
|
||||
const mainDuration = Date.now() - startTime
|
||||
resultDiv.innerHTML += `<p>Execution completed in ${mainDuration}ms</p>`
|
||||
resultDiv.innerHTML += `<pre>${JSON.stringify(result, null, 2)}</pre>`
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
BIN
favicon.ico
Normal file
BIN
favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
856
package-lock.json
generated
856
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -29,7 +29,7 @@
|
|||
},
|
||||
"scripts": {
|
||||
"prebuild": "node scripts/generate-version.js",
|
||||
"build": "BUILD_TYPE=unified rollup -c rollup.config.js",
|
||||
"build": "BUILD_TYPE=unified rollup -c rollup.config.js && node scripts/patch-textencoder.js",
|
||||
"build:browser": "BUILD_TYPE=browser rollup -c rollup.config.js",
|
||||
"build:cli": "cd cli-package && npm run build",
|
||||
"start": "node dist/unified.js",
|
||||
|
|
@ -52,7 +52,8 @@
|
|||
"deploy:cloud": "echo 'Please use one of the following commands to deploy to a specific cloud provider:' && echo ' npm run deploy:cloud:aws' && echo ' npm run deploy:cloud:gcp' && echo ' npm run deploy:cloud:cloudflare'",
|
||||
"postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'",
|
||||
"dry-run": "npm pack --dry-run",
|
||||
"test-cli": "node scripts/test-cli-locally.js"
|
||||
"test-cli": "node scripts/test-cli-locally.js",
|
||||
"test-all": "node scripts/test-all-environments.js"
|
||||
},
|
||||
"keywords": [
|
||||
"vector-database",
|
||||
|
|
@ -106,6 +107,7 @@
|
|||
"@typescript-eslint/eslint-plugin": "^7.4.0",
|
||||
"@typescript-eslint/parser": "^7.4.0",
|
||||
"eslint": "^8.57.0",
|
||||
"puppeteer": "^22.5.0",
|
||||
"rollup": "^4.13.0",
|
||||
"rollup-plugin-terser": "^7.0.2",
|
||||
"tslib": "^2.6.2",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import commonjs from '@rollup/plugin-commonjs'
|
|||
import json from '@rollup/plugin-json'
|
||||
import { terser } from 'rollup-plugin-terser'
|
||||
import replace from '@rollup/plugin-replace'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
// Custom plugin to provide empty shims for Node.js built-in modules in browser environments
|
||||
const nodeModuleShims = () => {
|
||||
|
|
@ -11,7 +13,16 @@ const nodeModuleShims = () => {
|
|||
name: 'node-module-shims',
|
||||
resolveId(source) {
|
||||
// List of Node.js built-in modules to shim
|
||||
const nodeBuiltins = ['fs', 'path', 'util', 'child_process', 'node:fs', 'node:path', 'node:util', 'node:child_process']
|
||||
const nodeBuiltins = [
|
||||
'fs',
|
||||
'path',
|
||||
'util',
|
||||
'child_process',
|
||||
'node:fs',
|
||||
'node:path',
|
||||
'node:util',
|
||||
'node:child_process'
|
||||
]
|
||||
|
||||
if (nodeBuiltins.includes(source)) {
|
||||
// Return a virtual module ID for the shim
|
||||
|
|
@ -53,13 +64,13 @@ import { Buffer as BufferPolyfill } from 'buffer';
|
|||
if (typeof window !== 'undefined' && typeof globalThis.Buffer === 'undefined') {
|
||||
globalThis.Buffer = BufferPolyfill;
|
||||
}
|
||||
`;
|
||||
`
|
||||
return {
|
||||
code: bufferImport + code,
|
||||
map: { mappings: '' } // Provide an empty sourcemap to avoid warnings
|
||||
};
|
||||
}
|
||||
}
|
||||
return null; // Return null to let Rollup handle other files normally
|
||||
return null // Return null to let Rollup handle other files normally
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -86,6 +97,8 @@ const fixThisReferences = () => {
|
|||
}
|
||||
}
|
||||
|
||||
// We no longer need to copy the worker file as we'll bundle it separately
|
||||
|
||||
// Get build type from environment variable or default to 'unified'
|
||||
const buildType = process.env.BUILD_TYPE || 'unified'
|
||||
|
||||
|
|
@ -184,5 +197,51 @@ const mainConfig = {
|
|||
]
|
||||
}
|
||||
|
||||
// Export the main configuration
|
||||
export default mainConfig
|
||||
// Create a separate configuration for the worker.ts file
|
||||
const workerConfig = {
|
||||
input: 'src/worker.ts',
|
||||
output: {
|
||||
file: 'dist/worker.js',
|
||||
format: 'es',
|
||||
sourcemap: true
|
||||
},
|
||||
plugins: [
|
||||
// Add environment replacement
|
||||
replace({
|
||||
preventAssignment: true,
|
||||
'process.env.NODE_ENV': JSON.stringify('production')
|
||||
}),
|
||||
// Add our custom plugins
|
||||
fixThisReferences(),
|
||||
nodeModuleShims(),
|
||||
bufferPolyfill(), // Add Buffer polyfill
|
||||
resolve({
|
||||
browser: true,
|
||||
preferBuiltins: false
|
||||
}),
|
||||
commonjs({
|
||||
transformMixedEsModules: true
|
||||
}),
|
||||
json(),
|
||||
typescript({
|
||||
tsconfig: './tsconfig.unified.json'
|
||||
})
|
||||
],
|
||||
external: [
|
||||
// Add any dependencies you want to exclude from the bundle
|
||||
'@aws-sdk/client-s3',
|
||||
'@smithy/util-stream',
|
||||
'@smithy/node-http-handler',
|
||||
'@aws-crypto/crc32c',
|
||||
'node:stream/web',
|
||||
'node:worker_threads',
|
||||
'worker_threads',
|
||||
'node:fs',
|
||||
'node:path',
|
||||
'fs',
|
||||
'path'
|
||||
]
|
||||
};
|
||||
|
||||
// Export both configurations
|
||||
export default [mainConfig, workerConfig];
|
||||
|
|
|
|||
20
scripts/create-favicon.js
Normal file
20
scripts/create-favicon.js
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
// Create a simple favicon.ico file
|
||||
import { writeFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Get the directory name of the current module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// This is a base64-encoded 16x16 transparent favicon
|
||||
const faviconBase64 = 'AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABILAAASCwAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAA==';
|
||||
|
||||
// Path to save the favicon
|
||||
const faviconPath = join(__dirname, '..', 'favicon.ico');
|
||||
|
||||
// Convert base64 to binary and save
|
||||
const faviconBuffer = Buffer.from(faviconBase64, 'base64');
|
||||
writeFileSync(faviconPath, faviconBuffer);
|
||||
|
||||
console.log(`Favicon created at ${faviconPath}`);
|
||||
101
scripts/patch-textencoder.js
Executable file
101
scripts/patch-textencoder.js
Executable file
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Simplified TextEncoder Patch
|
||||
*
|
||||
* This script patches the compiled unified.js file to fix the TextEncoder issue in Node.js
|
||||
* by replacing references to this.util.TextEncoder with direct TextEncoder usage.
|
||||
*/
|
||||
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
// Path to the compiled unified.js file
|
||||
const unifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.js')
|
||||
|
||||
// Read the file
|
||||
console.log(`Reading ${unifiedJsPath}...`)
|
||||
let content = fs.readFileSync(unifiedJsPath, 'utf8')
|
||||
|
||||
// Simple replacement: replace all instances of new this.util.TextEncoder() with new TextEncoder()
|
||||
const pattern = /new\s+this\.util\.TextEncoder\(\)/g
|
||||
const replacement = 'new TextEncoder()'
|
||||
|
||||
// Apply the patch
|
||||
const patchedContent = content.replace(pattern, replacement)
|
||||
|
||||
// Check if the patch was applied
|
||||
if (patchedContent === content) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextEncoder()" found in the file.'
|
||||
)
|
||||
} else {
|
||||
// Write the patched file
|
||||
console.log('Writing patched file...')
|
||||
fs.writeFileSync(unifiedJsPath, patchedContent, 'utf8')
|
||||
console.log('Patch applied successfully!')
|
||||
}
|
||||
|
||||
// Also patch the minified version if it exists
|
||||
const minifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.min.js')
|
||||
if (fs.existsSync(minifiedJsPath)) {
|
||||
console.log(`Reading ${minifiedJsPath}...`)
|
||||
const minContent = fs.readFileSync(minifiedJsPath, 'utf8')
|
||||
|
||||
// Apply the same replacement to the minified file
|
||||
const patchedMinContent = minContent.replace(pattern, replacement)
|
||||
|
||||
// Check if the patch was applied
|
||||
if (patchedMinContent === minContent) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextEncoder()" found in the minified file.'
|
||||
)
|
||||
} else {
|
||||
// Write the patched file
|
||||
console.log('Writing patched minified file...')
|
||||
fs.writeFileSync(minifiedJsPath, patchedMinContent, 'utf8')
|
||||
console.log('Minified file patched successfully!')
|
||||
}
|
||||
}
|
||||
|
||||
// Also patch TextDecoder
|
||||
console.log('Patching TextDecoder references...')
|
||||
content = fs.readFileSync(unifiedJsPath, 'utf8')
|
||||
const decoderPattern = /new\s+this\.util\.TextDecoder\(\)/g
|
||||
const decoderReplacement = 'new TextDecoder()'
|
||||
const patchedDecoderContent = content.replace(
|
||||
decoderPattern,
|
||||
decoderReplacement
|
||||
)
|
||||
|
||||
if (patchedDecoderContent === content) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextDecoder()" found in the file.'
|
||||
)
|
||||
} else {
|
||||
fs.writeFileSync(unifiedJsPath, patchedDecoderContent, 'utf8')
|
||||
console.log('TextDecoder patch applied successfully!')
|
||||
}
|
||||
|
||||
// Patch the minified file for TextDecoder as well
|
||||
if (fs.existsSync(minifiedJsPath)) {
|
||||
const minContent = fs.readFileSync(minifiedJsPath, 'utf8')
|
||||
const patchedMinDecoderContent = minContent.replace(
|
||||
decoderPattern,
|
||||
decoderReplacement
|
||||
)
|
||||
|
||||
if (patchedMinDecoderContent === minContent) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextDecoder()" found in the minified file.'
|
||||
)
|
||||
} else {
|
||||
fs.writeFileSync(minifiedJsPath, patchedMinDecoderContent, 'utf8')
|
||||
console.log('TextDecoder patch applied to minified file successfully!')
|
||||
}
|
||||
}
|
||||
266
scripts/test-all-environments.js
Normal file
266
scripts/test-all-environments.js
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Test All Environments
|
||||
*
|
||||
* This script runs tests for the Brainy library in all environments:
|
||||
* - Browser (using Puppeteer for headless browser testing)
|
||||
* - Node.js
|
||||
* - CLI
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process'
|
||||
import { fileURLToPath } from 'url'
|
||||
import path from 'path'
|
||||
import fs from 'fs'
|
||||
import http from 'http'
|
||||
import puppeteer from 'puppeteer'
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
const rootDir = path.join(__dirname, '..')
|
||||
|
||||
// Define colors for console output
|
||||
const colors = {
|
||||
reset: '\x1b[0m',
|
||||
bright: '\x1b[1m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
red: '\x1b[31m',
|
||||
cyan: '\x1b[36m'
|
||||
}
|
||||
|
||||
// Helper function to log with colors
|
||||
function log(message, color = colors.reset) {
|
||||
console.log(`${color}${message}${colors.reset}`)
|
||||
}
|
||||
|
||||
// Helper function to log section headers
|
||||
function logSection(title) {
|
||||
console.log('\n' + '='.repeat(80))
|
||||
console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`)
|
||||
console.log('='.repeat(80) + '\n')
|
||||
}
|
||||
|
||||
// Helper function to run a command and return its output
|
||||
function runCommand(command, cwd = rootDir) {
|
||||
try {
|
||||
return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' })
|
||||
} catch (error) {
|
||||
log(`Error running command: ${command}`, colors.red)
|
||||
log(error.message, colors.red)
|
||||
if (error.stdout) log(`stdout: ${error.stdout}`)
|
||||
if (error.stderr) log(`stderr: ${error.stderr}`, colors.red)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Main function to run all tests
|
||||
async function runAllTests() {
|
||||
try {
|
||||
logSection('BUILDING PACKAGES')
|
||||
|
||||
// Build the main package
|
||||
log('Building main package...', colors.yellow)
|
||||
runCommand('npm run build')
|
||||
log('Main package built successfully!', colors.green)
|
||||
|
||||
// Build the browser package
|
||||
log('Building browser package...', colors.yellow)
|
||||
runCommand('npm run build:browser')
|
||||
log('Browser package built successfully!', colors.green)
|
||||
|
||||
// Build the CLI package
|
||||
log('Building CLI package...', colors.yellow)
|
||||
runCommand('npm run build:cli')
|
||||
log('CLI package built successfully!', colors.green)
|
||||
|
||||
logSection('RUNNING NODE.JS TESTS')
|
||||
|
||||
// Run Node.js tests
|
||||
log('Running Node.js worker test...', colors.yellow)
|
||||
const nodeWorkerResult = runCommand('node test-worker.js')
|
||||
log(nodeWorkerResult)
|
||||
log('Node.js worker test completed!', colors.green)
|
||||
|
||||
log('Running unified text encoding test...', colors.yellow)
|
||||
const textEncodingResult = runCommand('node test-unified-encoding.js')
|
||||
log(textEncodingResult)
|
||||
log('Unified text encoding test completed!', colors.green)
|
||||
|
||||
logSection('RUNNING BROWSER TESTS')
|
||||
|
||||
// Start a simple HTTP server to serve the test files
|
||||
log('Starting HTTP server...', colors.yellow)
|
||||
const server = http.createServer((req, res) => {
|
||||
// Normalize the URL to handle relative paths
|
||||
const normalizedUrl = req.url.replace(/^\/+/, '/')
|
||||
let filePath = path.join(
|
||||
rootDir,
|
||||
normalizedUrl === '/' ? 'index.html' : normalizedUrl
|
||||
)
|
||||
|
||||
// Handle relative paths (e.g., ../dist/unified.js)
|
||||
if (normalizedUrl.includes('../')) {
|
||||
// Convert the URL to an absolute path relative to the root directory
|
||||
const parts = normalizedUrl.split('/')
|
||||
const resolvedParts = []
|
||||
|
||||
for (const part of parts) {
|
||||
if (part === '..') {
|
||||
resolvedParts.pop()
|
||||
} else if (part && part !== '.') {
|
||||
resolvedParts.push(part)
|
||||
}
|
||||
}
|
||||
|
||||
filePath = path.join(rootDir, resolvedParts.join('/'))
|
||||
}
|
||||
|
||||
log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.yellow)
|
||||
|
||||
// Check if the file exists
|
||||
if (fs.existsSync(filePath)) {
|
||||
const extname = path.extname(filePath)
|
||||
let contentType = 'text/html'
|
||||
|
||||
switch (extname) {
|
||||
case '.js':
|
||||
contentType = 'text/javascript'
|
||||
break
|
||||
case '.css':
|
||||
contentType = 'text/css'
|
||||
break
|
||||
case '.json':
|
||||
contentType = 'application/json'
|
||||
break
|
||||
case '.png':
|
||||
contentType = 'image/png'
|
||||
break
|
||||
case '.jpg':
|
||||
contentType = 'image/jpg'
|
||||
break
|
||||
}
|
||||
|
||||
res.writeHead(200, { 'Content-Type': contentType })
|
||||
const fileStream = fs.createReadStream(filePath)
|
||||
fileStream.pipe(res)
|
||||
} else {
|
||||
log(`File not found: ${filePath}`, colors.red)
|
||||
res.writeHead(404)
|
||||
res.end('File not found')
|
||||
}
|
||||
})
|
||||
|
||||
// Start the server on a random port
|
||||
const PORT = 3000 + Math.floor(Math.random() * 1000)
|
||||
server.listen(PORT)
|
||||
log(`HTTP server started on port ${PORT}`, colors.green)
|
||||
|
||||
// Run browser tests using Puppeteer
|
||||
log('Launching headless browser...', colors.yellow)
|
||||
// Using --no-sandbox flag to avoid issues with the Chrome sandbox in certain environments
|
||||
// See: https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md
|
||||
const browser = await puppeteer.launch({ args: ['--no-sandbox'] })
|
||||
const page = await browser.newPage()
|
||||
|
||||
// Capture console logs from the page
|
||||
page.on('console', (message) => {
|
||||
const type = message.type()
|
||||
const text = message.text()
|
||||
if (type === 'error') {
|
||||
log(`Browser console error: ${text}`, colors.red)
|
||||
} else {
|
||||
log(`Browser console: ${text}`)
|
||||
}
|
||||
})
|
||||
|
||||
// Test browser worker
|
||||
log('Running browser worker test...', colors.yellow)
|
||||
await page.goto(`http://localhost:${PORT}/demo/test-browser-worker.html`)
|
||||
await page.waitForSelector('#runTest')
|
||||
await page.click('#runTest')
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const resultText = document.getElementById('result').textContent
|
||||
return resultText.includes('Worker thread execution completed')
|
||||
},
|
||||
{ timeout: 30000 }
|
||||
)
|
||||
|
||||
const browserWorkerResult = await page.evaluate(() => {
|
||||
return document.getElementById('result').innerHTML
|
||||
})
|
||||
log('Browser worker test result:', colors.green)
|
||||
log(browserWorkerResult.replace(/<[^>]*>/g, '').trim())
|
||||
|
||||
// Test fallback mechanism
|
||||
log('Running fallback test...', colors.yellow)
|
||||
await page.goto(`http://localhost:${PORT}/demo/test-fallback.html`)
|
||||
await page.waitForSelector('#runTest')
|
||||
await page.click('#runTest')
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const resultText = document.getElementById('result').textContent
|
||||
return resultText.includes('Test completed')
|
||||
},
|
||||
{ timeout: 30000 }
|
||||
)
|
||||
|
||||
const fallbackResult = await page.evaluate(() => {
|
||||
return document.getElementById('result').innerHTML
|
||||
})
|
||||
log('Fallback test result:', colors.green)
|
||||
log(fallbackResult.replace(/<[^>]*>/g, '').trim())
|
||||
|
||||
// Close the browser and server
|
||||
await browser.close()
|
||||
server.close()
|
||||
log('HTTP server stopped', colors.green)
|
||||
|
||||
logSection('RUNNING CLI TESTS')
|
||||
|
||||
// Run CLI tests
|
||||
log('Testing CLI package locally...', colors.yellow)
|
||||
try {
|
||||
runCommand('npm run test-cli')
|
||||
log('CLI test completed!', colors.green)
|
||||
|
||||
// Run some basic CLI commands to verify functionality
|
||||
log('Testing basic CLI commands...', colors.yellow)
|
||||
const cliVersionResult = runCommand('brainy --version')
|
||||
log(`CLI version: ${cliVersionResult.trim()}`, colors.green)
|
||||
|
||||
const cliHelpResult = runCommand('brainy --help')
|
||||
log('CLI help command executed successfully', colors.green)
|
||||
|
||||
// Test the pipeline command
|
||||
log('Testing pipeline command...', colors.yellow)
|
||||
const pipelineResult = runCommand('brainy test-pipeline "This is a test"')
|
||||
log('Pipeline test completed!', colors.green)
|
||||
} catch (error) {
|
||||
log(
|
||||
"CLI tests failed. This might be expected if you don't have the CLI installed globally.",
|
||||
colors.yellow
|
||||
)
|
||||
log(
|
||||
'You can run the CLI tests separately with: npm run test-cli',
|
||||
colors.yellow
|
||||
)
|
||||
}
|
||||
|
||||
logSection('ALL TESTS COMPLETED')
|
||||
log('All environment tests completed successfully!', colors.green)
|
||||
} catch (error) {
|
||||
logSection('TEST FAILURE')
|
||||
log(`Tests failed: ${error.message}`, colors.red)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Run the tests
|
||||
runAllTests().catch((error) => {
|
||||
log(`Unhandled error: ${error.message}`, colors.red)
|
||||
process.exit(1)
|
||||
})
|
||||
29
src/index.ts
29
src/index.ts
|
|
@ -3,6 +3,11 @@
|
|||
* A vector database using HNSW indexing with Origin Private File System storage
|
||||
*/
|
||||
|
||||
// Import unified text encoding utilities first to ensure they're available
|
||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||
// Apply the TensorFlow.js platform patch if needed
|
||||
applyTensorFlowPatch()
|
||||
|
||||
// Export main BrainyData class and related types
|
||||
import { BrainyData, BrainyDataConfig } from './brainyData.js'
|
||||
|
||||
|
|
@ -39,6 +44,18 @@ import {
|
|||
cleanupWorkerPools
|
||||
} from './utils/workerUtils.js'
|
||||
|
||||
// Export environment utilities
|
||||
import {
|
||||
isBrowser,
|
||||
isNode,
|
||||
isWebWorker,
|
||||
areWebWorkersAvailable,
|
||||
areWorkerThreadsAvailable,
|
||||
areWorkerThreadsAvailableSync,
|
||||
isThreadingAvailable,
|
||||
isThreadingAvailableAsync
|
||||
} from './utils/environment.js'
|
||||
|
||||
export {
|
||||
UniversalSentenceEncoder,
|
||||
createEmbeddingFunction,
|
||||
|
|
@ -48,7 +65,17 @@ export {
|
|||
|
||||
// Worker utilities
|
||||
executeInThread,
|
||||
cleanupWorkerPools
|
||||
cleanupWorkerPools,
|
||||
|
||||
// Environment utilities
|
||||
isBrowser,
|
||||
isNode,
|
||||
isWebWorker,
|
||||
areWebWorkersAvailable,
|
||||
areWorkerThreadsAvailable,
|
||||
areWorkerThreadsAvailableSync,
|
||||
isThreadingAvailable,
|
||||
isThreadingAvailableAsync
|
||||
}
|
||||
|
||||
// Export storage adapters
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ type Edge = GraphVerb
|
|||
|
||||
// Directory and file names
|
||||
const ROOT_DIR = 'opfs-vector-db'
|
||||
|
||||
const NOUNS_DIR = 'nouns'
|
||||
const VERBS_DIR = 'verbs'
|
||||
const METADATA_DIR = 'metadata'
|
||||
|
|
|
|||
|
|
@ -4,6 +4,13 @@
|
|||
* Environment detection is handled here and made available to all components
|
||||
*/
|
||||
|
||||
// Import unified text encoding utilities
|
||||
// This needs to be imported first to ensure it's loaded before TensorFlow.js
|
||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||
|
||||
// Apply the TensorFlow.js platform patch if needed
|
||||
applyTensorFlowPatch()
|
||||
|
||||
// Export environment information
|
||||
export const environment = {
|
||||
isBrowser: typeof window !== 'undefined',
|
||||
|
|
|
|||
|
|
@ -10,52 +10,12 @@ import type {
|
|||
PlatformNodeObject
|
||||
} from '../types/tensorflowTypes.js'
|
||||
|
||||
// Define a global PlatformNode class for TensorFlow.js compatibility
|
||||
// This is needed because TensorFlow.js creates its own PlatformNode instance
|
||||
// and we need to ensure it uses the correct TextEncoder/TextDecoder
|
||||
if (
|
||||
typeof global !== 'undefined' &&
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions &&
|
||||
process.versions.node
|
||||
) {
|
||||
try {
|
||||
// In Node.js v24.3.0+, TextEncoder and TextDecoder are globally available
|
||||
// Define the PlatformNode class that TensorFlow.js will use
|
||||
class PlatformNode {
|
||||
util: any
|
||||
textEncoder: any
|
||||
textDecoder: any
|
||||
// Import the unified text encoding utilities
|
||||
import { applyTensorFlowPatch } from './textEncoding.js'
|
||||
|
||||
constructor() {
|
||||
// Create a util object with the necessary methods
|
||||
this.util = {
|
||||
isFloat32Array,
|
||||
isTypedArray,
|
||||
TextEncoder,
|
||||
TextDecoder
|
||||
}
|
||||
|
||||
// Initialize TextEncoder/TextDecoder instances
|
||||
this.textEncoder = new TextEncoder()
|
||||
this.textDecoder = new TextDecoder()
|
||||
}
|
||||
}
|
||||
|
||||
// Assign the PlatformNode class to the global object
|
||||
;(global as any).PlatformNode = PlatformNode
|
||||
|
||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
||||
// Some TensorFlow.js code might look for this
|
||||
;(global as any).platformNode = new PlatformNode()
|
||||
|
||||
console.log(
|
||||
'Defined global PlatformNode class for TensorFlow.js compatibility'
|
||||
)
|
||||
} catch (error) {
|
||||
console.warn('Failed to define global PlatformNode class:', error)
|
||||
}
|
||||
}
|
||||
// Apply the TensorFlow.js platform patch if needed
|
||||
// This will define a global PlatformNode class that uses our text encoding utilities
|
||||
applyTensorFlowPatch()
|
||||
|
||||
/**
|
||||
* Check if an array is a Float32Array
|
||||
|
|
|
|||
167
src/utils/textEncoding.ts
Normal file
167
src/utils/textEncoding.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
/**
|
||||
* Unified Text Encoding Utilities
|
||||
*
|
||||
* This module provides a consistent way to handle text encoding/decoding across all environments
|
||||
* without relying on TextEncoder/TextDecoder polyfills or patches.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A simple text encoder that works in all environments
|
||||
* This avoids the need for TextEncoder polyfills and patches
|
||||
*/
|
||||
export class SimpleTextEncoder {
|
||||
/**
|
||||
* Encode a string to a Uint8Array
|
||||
* @param input - The string to encode
|
||||
* @returns A Uint8Array containing the encoded string
|
||||
*/
|
||||
encode(input: string): Uint8Array {
|
||||
// Simple UTF-8 encoding implementation that works everywhere
|
||||
return new Uint8Array([...input].map((c) => c.charCodeAt(0)))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A simple text decoder that works in all environments
|
||||
* This avoids the need for TextDecoder polyfills and patches
|
||||
*/
|
||||
export class SimpleTextDecoder {
|
||||
/**
|
||||
* Decode a Uint8Array to a string
|
||||
* @param input - The Uint8Array to decode
|
||||
* @returns The decoded string
|
||||
*/
|
||||
decode(input: Uint8Array): string {
|
||||
// Simple UTF-8 decoding implementation that works everywhere
|
||||
return String.fromCharCode.apply(null, [...input])
|
||||
}
|
||||
}
|
||||
|
||||
// Create constructor functions that can be used as drop-in replacements
|
||||
// for the native TextEncoder and TextDecoder
|
||||
|
||||
/**
|
||||
* Interface for UniversalTextEncoder instance
|
||||
*/
|
||||
interface IUniversalTextEncoder {
|
||||
encode: (input: string) => Uint8Array;
|
||||
}
|
||||
|
||||
/**
|
||||
* A constructor function for TextEncoder that works in all environments
|
||||
*/
|
||||
export function UniversalTextEncoder(this: IUniversalTextEncoder) {
|
||||
if (!(this instanceof UniversalTextEncoder)) {
|
||||
return new (UniversalTextEncoder as any)()
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to use the native TextEncoder if available
|
||||
const nativeEncoder: TextEncoder = new TextEncoder()
|
||||
this.encode = nativeEncoder.encode.bind(nativeEncoder)
|
||||
} catch (e) {
|
||||
// Fall back to our simple implementation
|
||||
const simpleEncoder: SimpleTextEncoder = new SimpleTextEncoder()
|
||||
this.encode = simpleEncoder.encode.bind(simpleEncoder)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for UniversalTextDecoder instance
|
||||
*/
|
||||
interface IUniversalTextDecoder {
|
||||
decode: (input: Uint8Array) => string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A constructor function for TextDecoder that works in all environments
|
||||
*/
|
||||
export function UniversalTextDecoder(this: IUniversalTextDecoder) {
|
||||
if (!(this instanceof UniversalTextDecoder)) {
|
||||
return new (UniversalTextDecoder as any)()
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to use the native TextDecoder if available
|
||||
const nativeDecoder: TextDecoder = new TextDecoder()
|
||||
this.decode = nativeDecoder.decode.bind(nativeDecoder)
|
||||
} catch (e) {
|
||||
// Fall back to our simple implementation
|
||||
const simpleDecoder: SimpleTextDecoder = new SimpleTextDecoder()
|
||||
this.decode = simpleDecoder.decode.bind(simpleDecoder)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a text encoder that works in the current environment
|
||||
* @returns A text encoder object with an encode method
|
||||
*/
|
||||
export function getTextEncoder(): IUniversalTextEncoder {
|
||||
return new (UniversalTextEncoder as any)()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a text decoder that works in the current environment
|
||||
* @returns A text decoder object with a decode method
|
||||
*/
|
||||
export function getTextDecoder(): IUniversalTextDecoder {
|
||||
return new (UniversalTextDecoder as any)()
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the TensorFlow.js platform patch if needed
|
||||
* This function patches the global object to provide a PlatformNode class
|
||||
* that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder
|
||||
*/
|
||||
export function applyTensorFlowPatch(): void {
|
||||
// Only apply in Node.js environment
|
||||
if (
|
||||
typeof global !== 'undefined' &&
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions &&
|
||||
process.versions.node
|
||||
) {
|
||||
try {
|
||||
// Get encoders/decoders
|
||||
const encoder = getTextEncoder()
|
||||
const decoder = getTextDecoder()
|
||||
|
||||
// Define a custom PlatformNode class
|
||||
class PlatformNode {
|
||||
util: any
|
||||
textEncoder: any
|
||||
textDecoder: any
|
||||
|
||||
constructor() {
|
||||
// Create a util object with necessary methods and constructors
|
||||
this.util = {
|
||||
isFloat32Array: (arr: any) =>
|
||||
!!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr &&
|
||||
Object.prototype.toString.call(arr) ===
|
||||
'[object Float32Array]')
|
||||
),
|
||||
isTypedArray: (arr: any) =>
|
||||
!!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)),
|
||||
// Add TextEncoder and TextDecoder as constructors
|
||||
TextEncoder: UniversalTextEncoder,
|
||||
TextDecoder: UniversalTextDecoder
|
||||
}
|
||||
|
||||
// Initialize using the constructors from util
|
||||
this.textEncoder = new this.util.TextEncoder()
|
||||
this.textDecoder = new this.util.TextDecoder()
|
||||
}
|
||||
}
|
||||
|
||||
// Assign the PlatformNode class to the global object
|
||||
;(global as any).PlatformNode = PlatformNode
|
||||
|
||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
||||
;(global as any).platformNode = new PlatformNode()
|
||||
} catch (error) {
|
||||
console.warn('Failed to apply TensorFlow.js platform patch:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -24,7 +24,42 @@ export function executeInThread<T>(fnString: string, args: any): Promise<T> {
|
|||
} else {
|
||||
// Fallback to main thread execution
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
// Try different approaches to create a function from string
|
||||
let fn
|
||||
try {
|
||||
// First try with 'return' prefix
|
||||
fn = new Function('return ' + fnString)()
|
||||
} catch (functionError) {
|
||||
console.warn(
|
||||
'Fallback: Error creating function with return syntax, trying alternative approaches',
|
||||
functionError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try wrapping in parentheses for function expressions
|
||||
fn = new Function('return (' + fnString + ')')()
|
||||
} catch (wrapError) {
|
||||
console.warn(
|
||||
'Fallback: Error creating function with parentheses wrapping',
|
||||
wrapError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try direct approach for named functions
|
||||
fn = new Function(fnString)()
|
||||
} catch (directError) {
|
||||
console.error(
|
||||
'Fallback: All approaches to create function failed',
|
||||
directError
|
||||
)
|
||||
throw new Error(
|
||||
'Failed to create function from string: ' +
|
||||
(functionError as Error).message
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.resolve(fn(args) as T)
|
||||
} catch (error) {
|
||||
return Promise.reject(error)
|
||||
|
|
@ -40,73 +75,82 @@ function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
|
|||
return new Promise<T>((resolve, reject) => {
|
||||
try {
|
||||
// Dynamically import worker_threads (Node.js only)
|
||||
import('node:worker_threads').then(({ Worker, isMainThread, parentPort, workerData }) => {
|
||||
if (!isMainThread && parentPort) {
|
||||
// We're inside a worker, execute the function
|
||||
const fn = new Function('return ' + workerData.fnString)()
|
||||
const result = fn(workerData.args)
|
||||
parentPort.postMessage({ result })
|
||||
return
|
||||
}
|
||||
import('node:worker_threads')
|
||||
.then(({ Worker, isMainThread, parentPort, workerData }) => {
|
||||
if (!isMainThread && parentPort) {
|
||||
// We're inside a worker, execute the function
|
||||
const fn = new Function('return ' + workerData.fnString)()
|
||||
const result = fn(workerData.args)
|
||||
parentPort.postMessage({ result })
|
||||
return
|
||||
}
|
||||
|
||||
// Get a worker from the pool or create a new one
|
||||
const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`
|
||||
let worker: any
|
||||
// Get a worker from the pool or create a new one
|
||||
const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`
|
||||
let worker: any
|
||||
|
||||
if (workerPool.size < MAX_POOL_SIZE) {
|
||||
// Create a new worker
|
||||
worker = new Worker(`
|
||||
if (workerPool.size < MAX_POOL_SIZE) {
|
||||
// Create a new worker
|
||||
worker = new Worker(
|
||||
`
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
const fn = new Function('return ' + workerData.fnString)();
|
||||
const result = fn(workerData.args);
|
||||
parentPort.postMessage({ result });
|
||||
`, {
|
||||
eval: true,
|
||||
workerData: { fnString, args }
|
||||
})
|
||||
`,
|
||||
{
|
||||
eval: true,
|
||||
workerData: { fnString, args }
|
||||
}
|
||||
)
|
||||
|
||||
workerPool.set(workerId, worker)
|
||||
} else {
|
||||
// Reuse an existing worker
|
||||
const poolKeys = Array.from(workerPool.keys())
|
||||
const randomKey = poolKeys[Math.floor(Math.random() * poolKeys.length)]
|
||||
worker = workerPool.get(randomKey)
|
||||
workerPool.set(workerId, worker)
|
||||
} else {
|
||||
// Reuse an existing worker
|
||||
const poolKeys = Array.from(workerPool.keys())
|
||||
const randomKey =
|
||||
poolKeys[Math.floor(Math.random() * poolKeys.length)]
|
||||
worker = workerPool.get(randomKey)
|
||||
|
||||
// Terminate and recreate if the worker is busy
|
||||
if (worker._busy) {
|
||||
worker.terminate()
|
||||
worker = new Worker(`
|
||||
// Terminate and recreate if the worker is busy
|
||||
if (worker._busy) {
|
||||
worker.terminate()
|
||||
worker = new Worker(
|
||||
`
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
const fn = new Function('return ' + workerData.fnString)();
|
||||
const result = fn(workerData.args);
|
||||
parentPort.postMessage({ result });
|
||||
`, {
|
||||
eval: true,
|
||||
workerData: { fnString, args }
|
||||
})
|
||||
workerPool.set(randomKey, worker)
|
||||
`,
|
||||
{
|
||||
eval: true,
|
||||
workerData: { fnString, args }
|
||||
}
|
||||
)
|
||||
workerPool.set(randomKey, worker)
|
||||
}
|
||||
|
||||
worker._busy = true
|
||||
}
|
||||
|
||||
worker._busy = true
|
||||
}
|
||||
|
||||
worker.on('message', (message: any) => {
|
||||
worker._busy = false
|
||||
resolve(message.result as T)
|
||||
})
|
||||
|
||||
worker.on('error', (err: any) => {
|
||||
worker._busy = false
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code: number) => {
|
||||
if (code !== 0) {
|
||||
worker.on('message', (message: any) => {
|
||||
worker._busy = false
|
||||
reject(new Error(`Worker stopped with exit code ${code}`))
|
||||
}
|
||||
resolve(message.result as T)
|
||||
})
|
||||
|
||||
worker.on('error', (err: any) => {
|
||||
worker._busy = false
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code: number) => {
|
||||
if (code !== 0) {
|
||||
worker._busy = false
|
||||
reject(new Error(`Worker stopped with exit code ${code}`))
|
||||
}
|
||||
})
|
||||
})
|
||||
}).catch(reject)
|
||||
.catch(reject)
|
||||
} catch (error) {
|
||||
reject(error)
|
||||
}
|
||||
|
|
@ -119,35 +163,173 @@ function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
|
|||
function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
try {
|
||||
const workerCode = `
|
||||
self.onmessage = function(e) {
|
||||
try {
|
||||
const fn = new Function('return ' + e.data.fnString)();
|
||||
const result = fn(e.data.args);
|
||||
self.postMessage({ result: result });
|
||||
} catch (error) {
|
||||
self.postMessage({ error: error.message });
|
||||
}
|
||||
};
|
||||
`
|
||||
const blob = new Blob([workerCode], { type: 'application/javascript' })
|
||||
const url = URL.createObjectURL(blob)
|
||||
const worker = new Worker(url)
|
||||
// Use the dedicated worker.js file instead of creating a blob
|
||||
// Try different approaches to locate the worker.js file
|
||||
let workerPath = './worker.js'
|
||||
|
||||
worker.onmessage = function(e) {
|
||||
try {
|
||||
// First try to use the import.meta.url if available (modern browsers)
|
||||
if (typeof import.meta !== 'undefined' && import.meta.url) {
|
||||
const baseUrl = import.meta.url.substring(
|
||||
0,
|
||||
import.meta.url.lastIndexOf('/') + 1
|
||||
)
|
||||
workerPath = `${baseUrl}worker.js`
|
||||
}
|
||||
// Fallback to a relative path based on the unified.js location
|
||||
else if (typeof document !== 'undefined') {
|
||||
// Find the script tag that loaded unified.js
|
||||
const scripts = document.getElementsByTagName('script')
|
||||
for (let i = 0; i < scripts.length; i++) {
|
||||
const src = scripts[i].src
|
||||
if (src && src.includes('unified.js')) {
|
||||
// Get the directory path
|
||||
workerPath =
|
||||
src.substring(0, src.lastIndexOf('/') + 1) + 'worker.js'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
'Could not determine worker path from import.meta.url, using relative path',
|
||||
e
|
||||
)
|
||||
}
|
||||
|
||||
// If we couldn't determine the path, try some common locations
|
||||
if (workerPath === './worker.js' && typeof window !== 'undefined') {
|
||||
// Try to find the worker.js in the same directory as the current page
|
||||
const pageUrl = window.location.href
|
||||
const pageDir = pageUrl.substring(0, pageUrl.lastIndexOf('/') + 1)
|
||||
workerPath = `${pageDir}worker.js`
|
||||
|
||||
// Also check for dist/worker.js
|
||||
if (typeof document !== 'undefined') {
|
||||
const distWorkerPath = `${pageDir}dist/worker.js`
|
||||
// Create a test request to see if the file exists
|
||||
const xhr = new XMLHttpRequest()
|
||||
xhr.open('HEAD', distWorkerPath, false)
|
||||
try {
|
||||
xhr.send()
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
workerPath = distWorkerPath
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore errors, we'll use the default path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Using worker path:', workerPath)
|
||||
|
||||
// Try to create a worker, but fall back to inline worker or main thread execution if it fails
|
||||
let worker: Worker
|
||||
try {
|
||||
worker = new Worker(workerPath)
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'Failed to create Web Worker from file, trying inline worker:',
|
||||
error
|
||||
)
|
||||
|
||||
try {
|
||||
// Create an inline worker using a Blob
|
||||
const workerCode = `
|
||||
// Brainy Inline Worker Script
|
||||
console.log('Brainy Inline Worker: Started');
|
||||
|
||||
self.onmessage = function (e) {
|
||||
try {
|
||||
console.log('Brainy Inline Worker: Received message', e.data ? 'with data' : 'without data');
|
||||
|
||||
if (!e.data || !e.data.fnString) {
|
||||
throw new Error('Invalid message: missing function string');
|
||||
}
|
||||
|
||||
console.log('Brainy Inline Worker: Creating function from string');
|
||||
const fn = new Function('return ' + e.data.fnString)();
|
||||
|
||||
console.log('Brainy Inline Worker: Executing function with args');
|
||||
const result = fn(e.data.args);
|
||||
|
||||
console.log('Brainy Inline Worker: Function executed successfully, posting result');
|
||||
self.postMessage({ result: result });
|
||||
} catch (error) {
|
||||
console.error('Brainy Inline Worker: Error executing function', error);
|
||||
self.postMessage({
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
}
|
||||
};
|
||||
`
|
||||
|
||||
const blob = new Blob([workerCode], {
|
||||
type: 'application/javascript'
|
||||
})
|
||||
const blobUrl = URL.createObjectURL(blob)
|
||||
worker = new Worker(blobUrl)
|
||||
|
||||
console.log('Created inline worker using Blob URL')
|
||||
} catch (inlineWorkerError) {
|
||||
console.warn(
|
||||
'Failed to create inline Web Worker, falling back to main thread execution:',
|
||||
inlineWorkerError
|
||||
)
|
||||
// Execute in main thread as fallback
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
resolve(fn(args) as T)
|
||||
return
|
||||
} catch (mainThreadError) {
|
||||
reject(mainThreadError)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set a timeout to prevent hanging
|
||||
const timeoutId = setTimeout(() => {
|
||||
console.warn(
|
||||
'Web Worker execution timed out, falling back to main thread'
|
||||
)
|
||||
worker.terminate()
|
||||
|
||||
// Execute in main thread as fallback
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
resolve(fn(args) as T)
|
||||
} catch (mainThreadError) {
|
||||
reject(mainThreadError)
|
||||
}
|
||||
}, 25000) // 25 second timeout (less than the 30 second test timeout)
|
||||
|
||||
worker.onmessage = function (e) {
|
||||
clearTimeout(timeoutId)
|
||||
if (e.data.error) {
|
||||
reject(new Error(e.data.error))
|
||||
} else {
|
||||
resolve(e.data.result as T)
|
||||
}
|
||||
worker.terminate()
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
|
||||
worker.onerror = function(e) {
|
||||
reject(new Error(`Worker error: ${e.message}`))
|
||||
worker.onerror = function (e) {
|
||||
clearTimeout(timeoutId)
|
||||
console.warn(
|
||||
'Web Worker error, falling back to main thread execution:',
|
||||
e.message
|
||||
)
|
||||
worker.terminate()
|
||||
URL.revokeObjectURL(url)
|
||||
|
||||
// Execute in main thread as fallback
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
resolve(fn(args) as T)
|
||||
} catch (mainThreadError) {
|
||||
reject(mainThreadError)
|
||||
}
|
||||
}
|
||||
|
||||
worker.postMessage({ fnString, args })
|
||||
|
|
@ -163,12 +345,14 @@ function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
|
|||
*/
|
||||
export function cleanupWorkerPools(): void {
|
||||
if (isNode()) {
|
||||
import('node:worker_threads').then(({ Worker }) => {
|
||||
for (const worker of workerPool.values()) {
|
||||
worker.terminate()
|
||||
}
|
||||
workerPool.clear()
|
||||
console.log('Worker pools cleaned up')
|
||||
}).catch(console.error)
|
||||
import('node:worker_threads')
|
||||
.then(({ Worker }) => {
|
||||
for (const worker of workerPool.values()) {
|
||||
worker.terminate()
|
||||
}
|
||||
workerPool.clear()
|
||||
console.log('Worker pools cleaned up')
|
||||
})
|
||||
.catch(console.error)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
36
src/worker.js
Normal file
36
src/worker.js
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// Brainy Worker Script
|
||||
// This script is used by the workerUtils.js file to execute functions in a separate thread
|
||||
|
||||
// Import text encoding utilities
|
||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||
|
||||
// Apply the TensorFlow.js platform patch if needed
|
||||
applyTensorFlowPatch()
|
||||
|
||||
// Log that the worker has started
|
||||
console.log('Brainy Worker: Started')
|
||||
|
||||
self.onmessage = function (e) {
|
||||
try {
|
||||
console.log('Brainy Worker: Received message', e.data ? 'with data' : 'without data')
|
||||
|
||||
if (!e.data || !e.data.fnString) {
|
||||
throw new Error('Invalid message: missing function string')
|
||||
}
|
||||
|
||||
console.log('Brainy Worker: Creating function from string')
|
||||
const fn = new Function('return ' + e.data.fnString)()
|
||||
|
||||
console.log('Brainy Worker: Executing function with args')
|
||||
const result = fn(e.data.args)
|
||||
|
||||
console.log('Brainy Worker: Function executed successfully, posting result')
|
||||
self.postMessage({ result: result })
|
||||
} catch (error) {
|
||||
console.error('Brainy Worker: Error executing function', error)
|
||||
self.postMessage({
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
})
|
||||
}
|
||||
}
|
||||
75
src/worker.ts
Normal file
75
src/worker.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
// Brainy Worker Script
|
||||
// This script is used by the workerUtils.js file to execute functions in a separate thread
|
||||
|
||||
// Import text encoding utilities
|
||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||
|
||||
// Apply the TensorFlow.js platform patch if needed
|
||||
applyTensorFlowPatch()
|
||||
|
||||
// Log that the worker has started
|
||||
console.log('Brainy Worker: Started')
|
||||
|
||||
// Define the message handler with proper TypeScript typing
|
||||
self.onmessage = function (e: MessageEvent): void {
|
||||
try {
|
||||
console.log(
|
||||
'Brainy Worker: Received message',
|
||||
e.data ? 'with data' : 'without data'
|
||||
)
|
||||
|
||||
if (!e.data || !e.data.fnString) {
|
||||
throw new Error('Invalid message: missing function string')
|
||||
}
|
||||
|
||||
console.log('Brainy Worker: Creating function from string')
|
||||
// Use Function constructor to create a function from the string
|
||||
let fn
|
||||
|
||||
try {
|
||||
// First try with 'return' prefix
|
||||
fn = new Function('return ' + e.data.fnString)()
|
||||
} catch (functionError) {
|
||||
console.warn(
|
||||
'Brainy Worker: Error creating function with return syntax, trying alternative approaches',
|
||||
functionError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try wrapping in parentheses for function expressions
|
||||
fn = new Function('return (' + e.data.fnString + ')')()
|
||||
} catch (wrapError) {
|
||||
console.warn(
|
||||
'Brainy Worker: Error creating function with parentheses wrapping',
|
||||
wrapError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try direct approach for named functions
|
||||
fn = new Function(e.data.fnString)()
|
||||
} catch (directError) {
|
||||
console.error(
|
||||
'Brainy Worker: All approaches to create function failed',
|
||||
directError
|
||||
)
|
||||
throw new Error(
|
||||
'Failed to create function from string: ' +
|
||||
(functionError as Error).message
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Brainy Worker: Executing function with args')
|
||||
const result = fn(e.data.args)
|
||||
|
||||
console.log('Brainy Worker: Function executed successfully, posting result')
|
||||
self.postMessage({ result: result })
|
||||
} catch (error: any) {
|
||||
console.error('Brainy Worker: Error executing function', error)
|
||||
self.postMessage({
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
})
|
||||
}
|
||||
}
|
||||
39
test-unified-encoding.js
Normal file
39
test-unified-encoding.js
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
// Test script to verify the unified text encoding approach works correctly
|
||||
import { BrainyData } from './dist/unified.js'
|
||||
|
||||
async function testUnifiedEncoding() {
|
||||
console.log(
|
||||
'Testing unified text encoding approach in Node.js environment...'
|
||||
)
|
||||
|
||||
try {
|
||||
// Initialize BrainyData which should trigger the PlatformNode constructor
|
||||
console.log('Creating BrainyData instance...')
|
||||
const db = new BrainyData()
|
||||
|
||||
// Initialize the database
|
||||
console.log('Initializing database...')
|
||||
await db.init()
|
||||
|
||||
console.log('Test successful! Unified text encoding is working correctly.')
|
||||
|
||||
// Get database status to verify everything is working
|
||||
const status = await db.status()
|
||||
console.log('Database status:', status)
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error('Error during test:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testUnifiedEncoding().then((success) => {
|
||||
if (success) {
|
||||
console.log('Unified text encoding verification completed successfully!')
|
||||
} else {
|
||||
console.error('Unified text encoding verification failed!')
|
||||
process.exit(1)
|
||||
}
|
||||
})
|
||||
24
test-worker-utils.js
Normal file
24
test-worker-utils.js
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// Test script to verify that the workerUtils functions work correctly after removing eval
|
||||
import { executeInThread } from './dist/unified.js'
|
||||
|
||||
// Test function to execute in a thread
|
||||
const testFunction = `function(args) {
|
||||
return "Hello from " + args.name;
|
||||
}`
|
||||
|
||||
// Test with different environments
|
||||
async function runTests() {
|
||||
try {
|
||||
console.log('Testing executeInThread...')
|
||||
const result = await executeInThread(testFunction, {
|
||||
name: 'Worker Thread'
|
||||
})
|
||||
console.log('Result:', result)
|
||||
|
||||
console.log('All tests passed!')
|
||||
} catch (error) {
|
||||
console.error('Test failed:', error)
|
||||
}
|
||||
}
|
||||
|
||||
runTests()
|
||||
Loading…
Add table
Add a link
Reference in a new issue