From da760a34edbc822d4d35a4a9afdd0602142198c0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 4 Jul 2025 14:42:33 -0700 Subject: [PATCH] **feat(cli, workers): introduce text encoding patches and worker improvements** - Added `cli-package/brainy-wrapper.js` to patch global `TextEncoder` and `TextDecoder` for Node.js environments, ensuring compatibility with TensorFlow.js. - Implemented unified text encoding utilities in `cli-package/src/utils/textEncoding.ts` for cross-environment consistency. - Enhanced `src/worker.js` and introduced `src/worker.ts` to improve worker execution with better function serialization and error handling. - Updated `package.json`: - Modified the `build` script to include a patch for `TextEncoder`. - Added `test-all` script for multi-environment testing. - Introduced the Puppeteer dependency for browser testing. - Created a favicon generation script `scripts/create-favicon.js` using a base64-encoded icon. - Added `scripts/test-all-environments.js` to run automated tests across Node.js, browser, and CLI environments. - Enhanced `src/utils/workerUtils.ts` to support improved fallback mechanisms and robust worker pooling. This update improves the project's compatibility across environments, refines worker functionalities, and adds comprehensive testing support for reliability. --- DEVELOPERS.md | 18 + cli-package/brainy-wrapper.js | 53 ++ cli-package/package-lock.json | 26 +- cli-package/src/cli.ts | 126 +++- cli-package/src/utils/textEncoding.ts | 60 ++ demo/test-fallback.html | 177 +++--- favicon.ico | Bin 0 -> 1486 bytes package-lock.json | 856 ++++++++++++++++++++++++++ package.json | 6 +- rollup.config.js | 71 ++- scripts/create-favicon.js | 20 + scripts/patch-textencoder.js | 101 +++ scripts/test-all-environments.js | 266 ++++++++ src/index.ts | 29 +- src/storage/opfsStorage.ts | 1 + src/unified.ts | 7 + src/utils/tensorflowUtils.ts | 50 +- src/utils/textEncoding.ts | 167 +++++ src/utils/workerUtils.ts | 340 +++++++--- src/worker.js | 36 ++ src/worker.ts | 75 +++ test-unified-encoding.js | 39 ++ test-worker-utils.js | 24 + 23 files changed, 2319 insertions(+), 229 deletions(-) create mode 100755 cli-package/brainy-wrapper.js create mode 100644 cli-package/src/utils/textEncoding.ts create mode 100644 favicon.ico create mode 100644 scripts/create-favicon.js create mode 100755 scripts/patch-textencoder.js create mode 100644 scripts/test-all-environments.js create mode 100644 src/utils/textEncoding.ts create mode 100644 src/worker.js create mode 100644 src/worker.ts create mode 100644 test-unified-encoding.js create mode 100644 test-worker-utils.js diff --git a/DEVELOPERS.md b/DEVELOPERS.md index 39c5cf9e..30dd9424 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -10,6 +10,7 @@ This document contains detailed information for developers working with Brainy, - [Build System](#build-system) - [Testing](#testing) + - [Testing All Environments](#testing-all-environments) - [Testing the CLI Package Locally](#testing-the-cli-package-locally) - [Publishing](#publishing) - [Publishing the CLI Package](#publishing-the-cli-package) @@ -60,6 +61,23 @@ Brainy uses a modern build system that optimizes for both Node.js and browser en ## Testing +### Testing All Environments + +Brainy provides a comprehensive test script that verifies the library works correctly in all supported environments (browser, Node.js, and CLI): + +```bash +# Test the library in all environments +npm run test-all +``` + +This script: +1. Builds all packages (main, browser, CLI) +2. Runs Node.js tests (worker tests and unified text encoding test) +3. Starts a local HTTP server and runs browser tests using Puppeteer (headless browser) +4. Runs CLI tests by installing the CLI package locally and testing basic commands + +The test results are displayed with color-coded output for better readability. + ### Testing the CLI Package Locally Before publishing the CLI package to npm, you can test it locally to ensure it works as expected: diff --git a/cli-package/brainy-wrapper.js b/cli-package/brainy-wrapper.js new file mode 100755 index 00000000..b2309c92 --- /dev/null +++ b/cli-package/brainy-wrapper.js @@ -0,0 +1,53 @@ +#!/usr/bin/env node + +/** + * Brainy CLI Wrapper + * This script patches the global object to fix TextEncoder issues before loading the CLI + */ + +console.log('Brainy running in Node.js environment') + +// Define a custom PlatformNode class that doesn't rely on this.util.TextEncoder +if ( + typeof global !== 'undefined' && + typeof process !== 'undefined' && + process.versions && + process.versions.node +) { + try { + // Define a PlatformNode class that uses the global TextEncoder/TextDecoder directly + class PlatformNode { + constructor() { + // Create a util object with only the necessary methods + this.util = { + isFloat32Array: (arr) => + !!( + arr instanceof Float32Array || + (arr && + Object.prototype.toString.call(arr) === '[object Float32Array]') + ), + isTypedArray: (arr) => + !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)) + } + + // Initialize TextEncoder/TextDecoder instances directly from global + this.textEncoder = new TextEncoder() + this.textDecoder = new TextDecoder() + } + } + + // Assign the PlatformNode class to the global object + global.PlatformNode = PlatformNode + + // Also create an instance and assign it to global.platformNode (lowercase p) + global.platformNode = new PlatformNode() + } catch (error) { + console.warn('Failed to define global PlatformNode class:', error) + } +} + +// Now load and run the actual CLI +import('./dist/cli.js').catch((err) => { + console.error('Error loading CLI:', err) + process.exit(1) +}) diff --git a/cli-package/package-lock.json b/cli-package/package-lock.json index 50953116..623dcb8e 100644 --- a/cli-package/package-lock.json +++ b/cli-package/package-lock.json @@ -1,16 +1,16 @@ { "name": "@soulcraft/brainy-cli", - "version": "0.9.34", + "version": "0.9.36", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy-cli", - "version": "0.9.34", + "version": "0.9.36", "hasInstallScript": true, "license": "MIT", "dependencies": { - "@soulcraft/brainy": "0.9.34", + "@soulcraft/brainy": "0.9.36", "commander": "^14.0.0", "omelette": "^0.4.17" }, @@ -22,14 +22,14 @@ "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-typescript": "^11.1.6", - "@types/node": "^20.4.5", + "@types/node": "^20.11.30", "@types/omelette": "^0.4.5", - "rollup": "^4.12.0", + "rollup": "^4.13.0", "rollup-plugin-terser": "^7.0.2", - "typescript": "^5.1.6" + "typescript": "^5.4.5" }, "engines": { - "node": ">=23.0.0" + "node": ">=24.3.0" } }, "node_modules/@aws-crypto/crc32": { @@ -2086,13 +2086,13 @@ } }, "node_modules/@soulcraft/brainy": { - "version": "0.9.34", - "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.34.tgz", - "integrity": "sha512-o48XhHzOyb1xDyhxpRStw0+d/MwJQoL8Sd8RNlHC9/cExEK/v9mXTk31axXNsP2nh93Rf3JJxWS0YpDl00h3ZQ==", + "version": "0.9.36", + "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.9.36.tgz", + "integrity": "sha512-GhjhFrMS0KG7ARLwpG5ChYr8UNZ2D3PFPJIjAImEhLCZ9/D6tUBGVq5vlnoMtAPDptKJ2IMEXJjpZ4fLJBFTQw==", "hasInstallScript": true, "license": "MIT", "dependencies": { - "@aws-sdk/client-s3": "^3.427.0", + "@aws-sdk/client-s3": "^3.540.0", "@tensorflow-models/universal-sentence-encoder": "^1.3.3", "@tensorflow/tfjs": "^4.22.0", "@tensorflow/tfjs-backend-cpu": "^4.22.0", @@ -2100,10 +2100,10 @@ "@tensorflow/tfjs-converter": "^4.22.0", "@tensorflow/tfjs-core": "^4.22.0", "buffer": "^6.0.3", - "uuid": "^9.0.0" + "uuid": "^9.0.1" }, "engines": { - "node": ">=23.0.0" + "node": ">=24.3.0" } }, "node_modules/@soulcraft/brainy/node_modules/@tensorflow/tfjs-converter": { diff --git a/cli-package/src/cli.ts b/cli-package/src/cli.ts index 6a3a0b48..4d14e021 100644 --- a/cli-package/src/cli.ts +++ b/cli-package/src/cli.ts @@ -5,6 +5,14 @@ * A command-line interface for interacting with the Brainy vector database */ +// Import the unified text encoding utilities +// This needs to be done before importing @soulcraft/brainy +import { applyTensorFlowPatch } from './utils/textEncoding.js' + +// Apply the TensorFlow.js platform patch if needed +console.log('Brainy running in Node.js environment') +applyTensorFlowPatch() + import { BrainyData, NounType, @@ -39,7 +47,7 @@ function parseJSON(str: string): any { return {} } } - + // Helper function to resolve noun type function resolveNounType(type: string | number | undefined): NounType { if (!type) return NounType.Thing @@ -809,6 +817,7 @@ Examples: # Augmentation commands $ brainy augment list $ brainy augment info cognition + $ brainy test-pipeline "Test data" --data-type text --mode sequential $ brainy augment test-pipeline "Test data" --data-type text --mode sequential $ brainy augment stream-test --count 3 --interval 500 @@ -844,7 +853,8 @@ completion.tree({ 'completion-setup', 'init', 'help', - 'augment' + 'augment', + 'test-pipeline' ], // Command-specific completions add: { @@ -924,7 +934,17 @@ completion.tree({ }, 'completion-setup': {}, init: {}, - help: {} + help: {}, + 'test-pipeline': { + _: () => [ + '--data-type text', + '--mode sequential', + '--mode parallel', + '--mode threaded', + '--stop-on-error', + '--verbose' + ] + } }) // Initialize autocomplete @@ -1340,6 +1360,106 @@ augmentCommand // Add the augment command to the program program.addCommand(augmentCommand) +// Add a top-level test-pipeline command that redirects to augment test-pipeline +program + .command('test-pipeline') + .description('Test the sequential pipeline with sample data') + .argument( + '[text]', + 'Sample text to process through the pipeline', + 'This is a test of the Brainy pipeline' + ) + .option('-t, --data-type ', 'Type of data to process', 'text') + .option( + '-m, --mode ', + 'Execution mode (sequential, parallel, threaded)', + 'sequential' + ) + .option('-s, --stop-on-error', 'Stop execution if an error occurs', false) + .option('-v, --verbose', 'Show detailed output', false) + .action(async (text, options) => { + try { + // Initialize the pipeline + await sequentialPipeline.initialize() + + console.log(`Processing data: "${text}"`) + console.log(`Data type: ${options.dataType}`) + console.log(`Execution mode: ${options.mode}`) + console.log(`Stop on error: ${options.stopOnError}`) + console.log() + + // Set execution mode + let executionMode = ExecutionMode.SEQUENTIAL + switch (options.mode.toLowerCase()) { + case 'parallel': + executionMode = ExecutionMode.PARALLEL + break + case 'threaded': + executionMode = ExecutionMode.THREADED + break + default: + executionMode = ExecutionMode.SEQUENTIAL + } + + // Process the data + const result = await sequentialPipeline.processData( + text, + options.dataType, + { + stopOnError: options.stopOnError, + timeout: 30000 + } + ) + + console.log('Pipeline Execution Result:') + console.log(`Success: ${result.success}`) + + if (result.error) { + console.log(`Error: ${result.error}`) + } + + console.log('\nStage Results:') + + // Display stage results + Object.entries(result.stageResults).forEach((entry) => { + const stage = entry[0] + const stageResult = entry[1] as { + success?: boolean + error?: string + data?: any + } + + console.log(`\n${stage.toUpperCase()}:`) + console.log(` Success: ${stageResult?.success}`) + + if (stageResult?.error) { + console.log(` Error: ${stageResult.error}`) + } + + if (stageResult?.data && options.verbose) { + console.log(' Data:') + console.log( + JSON.stringify(stageResult.data, null, 2) + .split('\n') + .map((line: string) => ` ${line}`) + .join('\n') + ) + } + }) + + console.log('\nFinal Result Data:') + console.log( + JSON.stringify(result.data, null, 2) + .split('\n') + .map((line) => ` ${line}`) + .join('\n') + ) + } catch (error) { + console.error('Error:', (error as Error).message) + process.exit(1) + } + }) + // Add a command for setting up autocomplete program .command('completion-setup') diff --git a/cli-package/src/utils/textEncoding.ts b/cli-package/src/utils/textEncoding.ts new file mode 100644 index 00000000..578c0800 --- /dev/null +++ b/cli-package/src/utils/textEncoding.ts @@ -0,0 +1,60 @@ +/** + * Unified Text Encoding Utilities for CLI + * + * This module provides a consistent way to handle text encoding/decoding across all environments + * without relying on TextEncoder/TextDecoder polyfills or patches. + */ + +/** + * Apply the TensorFlow.js platform patch if needed + * This function patches the global object to provide a PlatformNode class + * that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder + */ +export function applyTensorFlowPatch(): void { + // Only apply in Node.js environment + if ( + typeof global !== 'undefined' && + typeof process !== 'undefined' && + process.versions && + process.versions.node + ) { + try { + // Define a custom PlatformNode class + class PlatformNode { + util: any + textEncoder: any + textDecoder: any + + constructor() { + // Create a util object with necessary methods and constructors + this.util = { + isFloat32Array: (arr: any) => + !!( + arr instanceof Float32Array || + (arr && + Object.prototype.toString.call(arr) === + '[object Float32Array]') + ), + isTypedArray: (arr: any) => + !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)), + // Use native TextEncoder and TextDecoder + TextEncoder: TextEncoder, + TextDecoder: TextDecoder + } + + // Initialize using the constructors from util + this.textEncoder = new this.util.TextEncoder() + this.textDecoder = new this.util.TextDecoder() + } + } + + // Assign the PlatformNode class to the global object + ;(global as any).PlatformNode = PlatformNode + + // Also create an instance and assign it to global.platformNode (lowercase p) + ;(global as any).platformNode = new PlatformNode() + } catch (error) { + console.warn('Failed to apply TensorFlow.js platform patch:', error) + } + } +} diff --git a/demo/test-fallback.html b/demo/test-fallback.html index d65e7b36..d5d2c545 100644 --- a/demo/test-fallback.html +++ b/demo/test-fallback.html @@ -5,87 +5,102 @@ Brainy Fallback Test -

Brainy Fallback Test

-

This page tests the Brainy fallback mechanism when threading is not available.

- - -
-

Results will appear here...

-
+

Brainy Fallback Test

+

This page tests the Brainy fallback mechanism when threading is not available.

- + // Execute the function + resultDiv.innerHTML += '

Starting execution...

' + const startTime = Date.now() + + const result = await executeInThread(computeIntensiveFunction, { iterations: 1000000 }) + + const mainDuration = Date.now() - startTime + resultDiv.innerHTML += `

Execution completed in ${mainDuration}ms

` + resultDiv.innerHTML += `
${JSON.stringify(result, null, 2)}
` + } + diff --git a/favicon.ico b/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4c741ab02e6dd99520dda0ae9b356110f833ad59 GIT binary patch literal 1486 zcmZQzU<5(|0R|wcz>vYhz#zuJz@P!dKp~(AL>x#lFaYI*xFHzK2NM7P{~yJpVKABo RM$^D(8lXxV0F|laV*nl`8Jz$C literal 0 HcmV?d00001 diff --git a/package-lock.json b/package-lock.json index 4539ef8b..f519834d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "@typescript-eslint/eslint-plugin": "^7.4.0", "@typescript-eslint/parser": "^7.4.0", "eslint": "^8.57.0", + "puppeteer": "^22.5.0", "rollup": "^4.13.0", "rollup-plugin-terser": "^7.0.2", "tslib": "^2.6.2", @@ -1165,6 +1166,73 @@ "node": ">= 8" } }, + "node_modules/@puppeteer/browsers": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.3.0.tgz", + "integrity": "sha512-ioXoq9gPxkss4MYhD+SFaU9p1IHFUX0ILAWFPyjGaBdjLsYAlZw6j1iLA0N/m12uVHLFDfSYNF7EQccjinIMDA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.3.5", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.4.0", + "semver": "^7.6.3", + "tar-fs": "^3.0.6", + "unbzip2-stream": "^1.4.3", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/@rollup/plugin-commonjs": { "version": "25.0.8", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-25.0.8.tgz", @@ -2431,6 +2499,13 @@ "@tensorflow/tfjs-core": "4.22.0" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2489,6 +2564,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.18.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", @@ -2718,6 +2804,16 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2778,12 +2874,32 @@ "node": ">=8" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2791,6 +2907,83 @@ "dev": true, "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "dev": true, + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.1.6.tgz", + "integrity": "sha512-25RsLF33BqooOEFNdMcEhMpJy8EoR88zSMrnOQOaM3USnOK2VmaJ1uaQEwPA6AQjrv1lXChScosN6CzbwbO9OQ==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.1.tgz", + "integrity": "sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -2811,6 +3004,16 @@ ], "license": "MIT" }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -2864,6 +3067,16 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -2910,6 +3123,21 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chromium-bidi": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.6.3.tgz", + "integrity": "sha512-qXlsCmpCZJAnoTYI83Iu6EdYQpMYdVkCfq08KDh2pmlVqK5t5IA9mGs4/LwCwp4fqisSOMXZxP3HIh8w8aRn0A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mitt": "3.0.1", + "urlpattern-polyfill": "10.0.0", + "zod": "3.23.8" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -2983,6 +3211,33 @@ "url": "https://opencollective.com/core-js" } }, + "node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2998,6 +3253,16 @@ "node": ">= 8" } }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/debug": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", @@ -3033,6 +3298,21 @@ "node": ">=0.10.0" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -3042,6 +3322,13 @@ "node": ">=0.4.0" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1312386", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1312386.tgz", + "integrity": "sha512-DPnhUXvmvKT2dFA/j7B+riVLUt9Q6RKJlcppojL5CoRywJJKLDYnRlw0gTFKfgDPHP5E04UoB71SxoJlVZy8FA==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -3088,6 +3375,36 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -3155,6 +3472,28 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, "node_modules/eslint": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", @@ -3284,6 +3623,20 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -3337,6 +3690,27 @@ "node": ">=0.10.0" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -3344,6 +3718,13 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -3420,6 +3801,16 @@ "reusify": "^1.0.4" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -3578,6 +3969,37 @@ "node": ">= 0.4" } }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", + "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/glob": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", @@ -3729,6 +4151,34 @@ "node": ">= 0.4" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -3805,6 +4255,34 @@ "dev": true, "license": "ISC" }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", @@ -3939,6 +4417,13 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "dev": true, + "license": "MIT" + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -3946,6 +4431,13 @@ "dev": true, "license": "MIT" }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -3984,6 +4476,13 @@ "node": ">= 0.8.0" } }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -4013,6 +4512,16 @@ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", "license": "Apache-2.0" }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.17", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", @@ -4113,6 +4622,13 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "dev": true, + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4127,6 +4643,16 @@ "dev": true, "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-fetch": { "version": "2.6.13", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.13.tgz", @@ -4207,6 +4733,40 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dev": true, + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -4220,6 +4780,25 @@ "node": ">=6" } }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -4267,6 +4846,13 @@ "node": ">=8" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -4297,6 +4883,54 @@ "node": ">= 0.8.0" } }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true, + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -4307,6 +4941,43 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "22.15.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.15.0.tgz", + "integrity": "sha512-XjCY1SiSEi1T7iSYuxS82ft85kwDJUS7wj1Z0eGVXKdtr5g4xnVcbjwxhq5xBnpK/E7x1VZZoJDxpjAOasHT4Q==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.3.0", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1312386", + "puppeteer-core": "22.15.0" + }, + "bin": { + "puppeteer": "lib/esm/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "22.15.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.15.0.tgz", + "integrity": "sha512-cHArnywCiAAVXa3t4GGL2vttNxh7GqXtIYGym99egkNJ3oG//wL9LkvO4WE8W1TJe95t1F1ocu9X4xWaGsOKOA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.3.0", + "chromium-bidi": "0.6.3", + "debug": "^4.3.6", + "devtools-protocol": "0.0.1312386", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -4621,6 +5292,47 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.5.tgz", + "integrity": "sha512-iF+tNDQla22geJdTyJB1wM/qrX9DMRwWrciEPwWLPRWAUEM8sQiyxgckLxWT1f7+9VabJS0jTGGr4QgBuvi6Ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -4648,6 +5360,20 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "license": "BSD-3-Clause" }, + "node_modules/streamx": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.1.tgz", + "integrity": "sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -4733,6 +5459,33 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar-fs": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.0.tgz", + "integrity": "sha512-5Mty5y/sOF1YWj1J6GiBodjlDc05CUR8PKXrsnFAiSG0xA+GHeWLovaZPYUDXkH/1iKRf2+M5+OrRgzC7O9b7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/terser": { "version": "5.43.1", "resolved": "https://registry.npmjs.org/terser/-/terser-5.43.1.tgz", @@ -4752,6 +5505,16 @@ "node": ">=10" } }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -4759,6 +5522,13 @@ "dev": true, "license": "MIT" }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true, + "license": "MIT" + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -4837,6 +5607,42 @@ "node": ">=14.17" } }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, + "node_modules/unbzip2-stream/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -4853,6 +5659,13 @@ "punycode": "^2.1.0" } }, + "node_modules/urlpattern-polyfill": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", + "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==", + "dev": true, + "license": "MIT" + }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -4932,6 +5745,28 @@ "dev": true, "license": "ISC" }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -4968,6 +5803,17 @@ "node": ">=10" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -4980,6 +5826,16 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index e1056d81..6b5a8fce 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "scripts": { "prebuild": "node scripts/generate-version.js", - "build": "BUILD_TYPE=unified rollup -c rollup.config.js", + "build": "BUILD_TYPE=unified rollup -c rollup.config.js && node scripts/patch-textencoder.js", "build:browser": "BUILD_TYPE=browser rollup -c rollup.config.js", "build:cli": "cd cli-package && npm run build", "start": "node dist/unified.js", @@ -52,7 +52,8 @@ "deploy:cloud": "echo 'Please use one of the following commands to deploy to a specific cloud provider:' && echo ' npm run deploy:cloud:aws' && echo ' npm run deploy:cloud:gcp' && echo ' npm run deploy:cloud:cloudflare'", "postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'", "dry-run": "npm pack --dry-run", - "test-cli": "node scripts/test-cli-locally.js" + "test-cli": "node scripts/test-cli-locally.js", + "test-all": "node scripts/test-all-environments.js" }, "keywords": [ "vector-database", @@ -106,6 +107,7 @@ "@typescript-eslint/eslint-plugin": "^7.4.0", "@typescript-eslint/parser": "^7.4.0", "eslint": "^8.57.0", + "puppeteer": "^22.5.0", "rollup": "^4.13.0", "rollup-plugin-terser": "^7.0.2", "tslib": "^2.6.2", diff --git a/rollup.config.js b/rollup.config.js index a06fc65e..43a39f52 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -4,6 +4,8 @@ import commonjs from '@rollup/plugin-commonjs' import json from '@rollup/plugin-json' import { terser } from 'rollup-plugin-terser' import replace from '@rollup/plugin-replace' +import fs from 'fs' +import path from 'path' // Custom plugin to provide empty shims for Node.js built-in modules in browser environments const nodeModuleShims = () => { @@ -11,7 +13,16 @@ const nodeModuleShims = () => { name: 'node-module-shims', resolveId(source) { // List of Node.js built-in modules to shim - const nodeBuiltins = ['fs', 'path', 'util', 'child_process', 'node:fs', 'node:path', 'node:util', 'node:child_process'] + const nodeBuiltins = [ + 'fs', + 'path', + 'util', + 'child_process', + 'node:fs', + 'node:path', + 'node:util', + 'node:child_process' + ] if (nodeBuiltins.includes(source)) { // Return a virtual module ID for the shim @@ -53,13 +64,13 @@ import { Buffer as BufferPolyfill } from 'buffer'; if (typeof window !== 'undefined' && typeof globalThis.Buffer === 'undefined') { globalThis.Buffer = BufferPolyfill; } -`; +` return { code: bufferImport + code, map: { mappings: '' } // Provide an empty sourcemap to avoid warnings - }; + } } - return null; // Return null to let Rollup handle other files normally + return null // Return null to let Rollup handle other files normally } } } @@ -86,6 +97,8 @@ const fixThisReferences = () => { } } +// We no longer need to copy the worker file as we'll bundle it separately + // Get build type from environment variable or default to 'unified' const buildType = process.env.BUILD_TYPE || 'unified' @@ -184,5 +197,51 @@ const mainConfig = { ] } -// Export the main configuration -export default mainConfig +// Create a separate configuration for the worker.ts file +const workerConfig = { + input: 'src/worker.ts', + output: { + file: 'dist/worker.js', + format: 'es', + sourcemap: true + }, + plugins: [ + // Add environment replacement + replace({ + preventAssignment: true, + 'process.env.NODE_ENV': JSON.stringify('production') + }), + // Add our custom plugins + fixThisReferences(), + nodeModuleShims(), + bufferPolyfill(), // Add Buffer polyfill + resolve({ + browser: true, + preferBuiltins: false + }), + commonjs({ + transformMixedEsModules: true + }), + json(), + typescript({ + tsconfig: './tsconfig.unified.json' + }) + ], + external: [ + // Add any dependencies you want to exclude from the bundle + '@aws-sdk/client-s3', + '@smithy/util-stream', + '@smithy/node-http-handler', + '@aws-crypto/crc32c', + 'node:stream/web', + 'node:worker_threads', + 'worker_threads', + 'node:fs', + 'node:path', + 'fs', + 'path' + ] +}; + +// Export both configurations +export default [mainConfig, workerConfig]; diff --git a/scripts/create-favicon.js b/scripts/create-favicon.js new file mode 100644 index 00000000..9c05b359 --- /dev/null +++ b/scripts/create-favicon.js @@ -0,0 +1,20 @@ +// Create a simple favicon.ico file +import { writeFileSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +// Get the directory name of the current module +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// This is a base64-encoded 16x16 transparent favicon +const faviconBase64 = 'AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABILAAASCwAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAA=='; + +// Path to save the favicon +const faviconPath = join(__dirname, '..', 'favicon.ico'); + +// Convert base64 to binary and save +const faviconBuffer = Buffer.from(faviconBase64, 'base64'); +writeFileSync(faviconPath, faviconBuffer); + +console.log(`Favicon created at ${faviconPath}`); diff --git a/scripts/patch-textencoder.js b/scripts/patch-textencoder.js new file mode 100755 index 00000000..2de4caa3 --- /dev/null +++ b/scripts/patch-textencoder.js @@ -0,0 +1,101 @@ +#!/usr/bin/env node + +/** + * Simplified TextEncoder Patch + * + * This script patches the compiled unified.js file to fix the TextEncoder issue in Node.js + * by replacing references to this.util.TextEncoder with direct TextEncoder usage. + */ + +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +// Get the directory of the current module +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +// Path to the compiled unified.js file +const unifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.js') + +// Read the file +console.log(`Reading ${unifiedJsPath}...`) +let content = fs.readFileSync(unifiedJsPath, 'utf8') + +// Simple replacement: replace all instances of new this.util.TextEncoder() with new TextEncoder() +const pattern = /new\s+this\.util\.TextEncoder\(\)/g +const replacement = 'new TextEncoder()' + +// Apply the patch +const patchedContent = content.replace(pattern, replacement) + +// Check if the patch was applied +if (patchedContent === content) { + console.warn( + 'No instances of "new this.util.TextEncoder()" found in the file.' + ) +} else { + // Write the patched file + console.log('Writing patched file...') + fs.writeFileSync(unifiedJsPath, patchedContent, 'utf8') + console.log('Patch applied successfully!') +} + +// Also patch the minified version if it exists +const minifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.min.js') +if (fs.existsSync(minifiedJsPath)) { + console.log(`Reading ${minifiedJsPath}...`) + const minContent = fs.readFileSync(minifiedJsPath, 'utf8') + + // Apply the same replacement to the minified file + const patchedMinContent = minContent.replace(pattern, replacement) + + // Check if the patch was applied + if (patchedMinContent === minContent) { + console.warn( + 'No instances of "new this.util.TextEncoder()" found in the minified file.' + ) + } else { + // Write the patched file + console.log('Writing patched minified file...') + fs.writeFileSync(minifiedJsPath, patchedMinContent, 'utf8') + console.log('Minified file patched successfully!') + } +} + +// Also patch TextDecoder +console.log('Patching TextDecoder references...') +content = fs.readFileSync(unifiedJsPath, 'utf8') +const decoderPattern = /new\s+this\.util\.TextDecoder\(\)/g +const decoderReplacement = 'new TextDecoder()' +const patchedDecoderContent = content.replace( + decoderPattern, + decoderReplacement +) + +if (patchedDecoderContent === content) { + console.warn( + 'No instances of "new this.util.TextDecoder()" found in the file.' + ) +} else { + fs.writeFileSync(unifiedJsPath, patchedDecoderContent, 'utf8') + console.log('TextDecoder patch applied successfully!') +} + +// Patch the minified file for TextDecoder as well +if (fs.existsSync(minifiedJsPath)) { + const minContent = fs.readFileSync(minifiedJsPath, 'utf8') + const patchedMinDecoderContent = minContent.replace( + decoderPattern, + decoderReplacement + ) + + if (patchedMinDecoderContent === minContent) { + console.warn( + 'No instances of "new this.util.TextDecoder()" found in the minified file.' + ) + } else { + fs.writeFileSync(minifiedJsPath, patchedMinDecoderContent, 'utf8') + console.log('TextDecoder patch applied to minified file successfully!') + } +} diff --git a/scripts/test-all-environments.js b/scripts/test-all-environments.js new file mode 100644 index 00000000..8687e067 --- /dev/null +++ b/scripts/test-all-environments.js @@ -0,0 +1,266 @@ +#!/usr/bin/env node + +/** + * Test All Environments + * + * This script runs tests for the Brainy library in all environments: + * - Browser (using Puppeteer for headless browser testing) + * - Node.js + * - CLI + */ + +import { execSync } from 'child_process' +import { fileURLToPath } from 'url' +import path from 'path' +import fs from 'fs' +import http from 'http' +import puppeteer from 'puppeteer' + +// Get the directory of the current module +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +const rootDir = path.join(__dirname, '..') + +// Define colors for console output +const colors = { + reset: '\x1b[0m', + bright: '\x1b[1m', + green: '\x1b[32m', + yellow: '\x1b[33m', + red: '\x1b[31m', + cyan: '\x1b[36m' +} + +// Helper function to log with colors +function log(message, color = colors.reset) { + console.log(`${color}${message}${colors.reset}`) +} + +// Helper function to log section headers +function logSection(title) { + console.log('\n' + '='.repeat(80)) + console.log(`${colors.bright}${colors.cyan}${title}${colors.reset}`) + console.log('='.repeat(80) + '\n') +} + +// Helper function to run a command and return its output +function runCommand(command, cwd = rootDir) { + try { + return execSync(command, { stdio: 'pipe', cwd, encoding: 'utf8' }) + } catch (error) { + log(`Error running command: ${command}`, colors.red) + log(error.message, colors.red) + if (error.stdout) log(`stdout: ${error.stdout}`) + if (error.stderr) log(`stderr: ${error.stderr}`, colors.red) + throw error + } +} + +// Main function to run all tests +async function runAllTests() { + try { + logSection('BUILDING PACKAGES') + + // Build the main package + log('Building main package...', colors.yellow) + runCommand('npm run build') + log('Main package built successfully!', colors.green) + + // Build the browser package + log('Building browser package...', colors.yellow) + runCommand('npm run build:browser') + log('Browser package built successfully!', colors.green) + + // Build the CLI package + log('Building CLI package...', colors.yellow) + runCommand('npm run build:cli') + log('CLI package built successfully!', colors.green) + + logSection('RUNNING NODE.JS TESTS') + + // Run Node.js tests + log('Running Node.js worker test...', colors.yellow) + const nodeWorkerResult = runCommand('node test-worker.js') + log(nodeWorkerResult) + log('Node.js worker test completed!', colors.green) + + log('Running unified text encoding test...', colors.yellow) + const textEncodingResult = runCommand('node test-unified-encoding.js') + log(textEncodingResult) + log('Unified text encoding test completed!', colors.green) + + logSection('RUNNING BROWSER TESTS') + + // Start a simple HTTP server to serve the test files + log('Starting HTTP server...', colors.yellow) + const server = http.createServer((req, res) => { + // Normalize the URL to handle relative paths + const normalizedUrl = req.url.replace(/^\/+/, '/') + let filePath = path.join( + rootDir, + normalizedUrl === '/' ? 'index.html' : normalizedUrl + ) + + // Handle relative paths (e.g., ../dist/unified.js) + if (normalizedUrl.includes('../')) { + // Convert the URL to an absolute path relative to the root directory + const parts = normalizedUrl.split('/') + const resolvedParts = [] + + for (const part of parts) { + if (part === '..') { + resolvedParts.pop() + } else if (part && part !== '.') { + resolvedParts.push(part) + } + } + + filePath = path.join(rootDir, resolvedParts.join('/')) + } + + log(`Request for: ${req.url}, resolved to: ${filePath}`, colors.yellow) + + // Check if the file exists + if (fs.existsSync(filePath)) { + const extname = path.extname(filePath) + let contentType = 'text/html' + + switch (extname) { + case '.js': + contentType = 'text/javascript' + break + case '.css': + contentType = 'text/css' + break + case '.json': + contentType = 'application/json' + break + case '.png': + contentType = 'image/png' + break + case '.jpg': + contentType = 'image/jpg' + break + } + + res.writeHead(200, { 'Content-Type': contentType }) + const fileStream = fs.createReadStream(filePath) + fileStream.pipe(res) + } else { + log(`File not found: ${filePath}`, colors.red) + res.writeHead(404) + res.end('File not found') + } + }) + + // Start the server on a random port + const PORT = 3000 + Math.floor(Math.random() * 1000) + server.listen(PORT) + log(`HTTP server started on port ${PORT}`, colors.green) + + // Run browser tests using Puppeteer + log('Launching headless browser...', colors.yellow) + // Using --no-sandbox flag to avoid issues with the Chrome sandbox in certain environments + // See: https://chromium.googlesource.com/chromium/src/+/main/docs/linux/suid_sandbox_development.md + const browser = await puppeteer.launch({ args: ['--no-sandbox'] }) + const page = await browser.newPage() + + // Capture console logs from the page + page.on('console', (message) => { + const type = message.type() + const text = message.text() + if (type === 'error') { + log(`Browser console error: ${text}`, colors.red) + } else { + log(`Browser console: ${text}`) + } + }) + + // Test browser worker + log('Running browser worker test...', colors.yellow) + await page.goto(`http://localhost:${PORT}/demo/test-browser-worker.html`) + await page.waitForSelector('#runTest') + await page.click('#runTest') + await page.waitForFunction( + () => { + const resultText = document.getElementById('result').textContent + return resultText.includes('Worker thread execution completed') + }, + { timeout: 30000 } + ) + + const browserWorkerResult = await page.evaluate(() => { + return document.getElementById('result').innerHTML + }) + log('Browser worker test result:', colors.green) + log(browserWorkerResult.replace(/<[^>]*>/g, '').trim()) + + // Test fallback mechanism + log('Running fallback test...', colors.yellow) + await page.goto(`http://localhost:${PORT}/demo/test-fallback.html`) + await page.waitForSelector('#runTest') + await page.click('#runTest') + await page.waitForFunction( + () => { + const resultText = document.getElementById('result').textContent + return resultText.includes('Test completed') + }, + { timeout: 30000 } + ) + + const fallbackResult = await page.evaluate(() => { + return document.getElementById('result').innerHTML + }) + log('Fallback test result:', colors.green) + log(fallbackResult.replace(/<[^>]*>/g, '').trim()) + + // Close the browser and server + await browser.close() + server.close() + log('HTTP server stopped', colors.green) + + logSection('RUNNING CLI TESTS') + + // Run CLI tests + log('Testing CLI package locally...', colors.yellow) + try { + runCommand('npm run test-cli') + log('CLI test completed!', colors.green) + + // Run some basic CLI commands to verify functionality + log('Testing basic CLI commands...', colors.yellow) + const cliVersionResult = runCommand('brainy --version') + log(`CLI version: ${cliVersionResult.trim()}`, colors.green) + + const cliHelpResult = runCommand('brainy --help') + log('CLI help command executed successfully', colors.green) + + // Test the pipeline command + log('Testing pipeline command...', colors.yellow) + const pipelineResult = runCommand('brainy test-pipeline "This is a test"') + log('Pipeline test completed!', colors.green) + } catch (error) { + log( + "CLI tests failed. This might be expected if you don't have the CLI installed globally.", + colors.yellow + ) + log( + 'You can run the CLI tests separately with: npm run test-cli', + colors.yellow + ) + } + + logSection('ALL TESTS COMPLETED') + log('All environment tests completed successfully!', colors.green) + } catch (error) { + logSection('TEST FAILURE') + log(`Tests failed: ${error.message}`, colors.red) + process.exit(1) + } +} + +// Run the tests +runAllTests().catch((error) => { + log(`Unhandled error: ${error.message}`, colors.red) + process.exit(1) +}) diff --git a/src/index.ts b/src/index.ts index a37f5fa5..58acad54 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,11 @@ * A vector database using HNSW indexing with Origin Private File System storage */ +// Import unified text encoding utilities first to ensure they're available +import { applyTensorFlowPatch } from './utils/textEncoding.js' +// Apply the TensorFlow.js platform patch if needed +applyTensorFlowPatch() + // Export main BrainyData class and related types import { BrainyData, BrainyDataConfig } from './brainyData.js' @@ -39,6 +44,18 @@ import { cleanupWorkerPools } from './utils/workerUtils.js' +// Export environment utilities +import { + isBrowser, + isNode, + isWebWorker, + areWebWorkersAvailable, + areWorkerThreadsAvailable, + areWorkerThreadsAvailableSync, + isThreadingAvailable, + isThreadingAvailableAsync +} from './utils/environment.js' + export { UniversalSentenceEncoder, createEmbeddingFunction, @@ -48,7 +65,17 @@ export { // Worker utilities executeInThread, - cleanupWorkerPools + cleanupWorkerPools, + + // Environment utilities + isBrowser, + isNode, + isWebWorker, + areWebWorkersAvailable, + areWorkerThreadsAvailable, + areWorkerThreadsAvailableSync, + isThreadingAvailable, + isThreadingAvailableAsync } // Export storage adapters diff --git a/src/storage/opfsStorage.ts b/src/storage/opfsStorage.ts index 11914918..4f43d6cb 100644 --- a/src/storage/opfsStorage.ts +++ b/src/storage/opfsStorage.ts @@ -18,6 +18,7 @@ type Edge = GraphVerb // Directory and file names const ROOT_DIR = 'opfs-vector-db' + const NOUNS_DIR = 'nouns' const VERBS_DIR = 'verbs' const METADATA_DIR = 'metadata' diff --git a/src/unified.ts b/src/unified.ts index e00d8527..dc99202f 100644 --- a/src/unified.ts +++ b/src/unified.ts @@ -4,6 +4,13 @@ * Environment detection is handled here and made available to all components */ +// Import unified text encoding utilities +// This needs to be imported first to ensure it's loaded before TensorFlow.js +import { applyTensorFlowPatch } from './utils/textEncoding.js' + +// Apply the TensorFlow.js platform patch if needed +applyTensorFlowPatch() + // Export environment information export const environment = { isBrowser: typeof window !== 'undefined', diff --git a/src/utils/tensorflowUtils.ts b/src/utils/tensorflowUtils.ts index a97b36a6..04c374eb 100644 --- a/src/utils/tensorflowUtils.ts +++ b/src/utils/tensorflowUtils.ts @@ -10,52 +10,12 @@ import type { PlatformNodeObject } from '../types/tensorflowTypes.js' -// Define a global PlatformNode class for TensorFlow.js compatibility -// This is needed because TensorFlow.js creates its own PlatformNode instance -// and we need to ensure it uses the correct TextEncoder/TextDecoder -if ( - typeof global !== 'undefined' && - typeof process !== 'undefined' && - process.versions && - process.versions.node -) { - try { - // In Node.js v24.3.0+, TextEncoder and TextDecoder are globally available - // Define the PlatformNode class that TensorFlow.js will use - class PlatformNode { - util: any - textEncoder: any - textDecoder: any +// Import the unified text encoding utilities +import { applyTensorFlowPatch } from './textEncoding.js' - constructor() { - // Create a util object with the necessary methods - this.util = { - isFloat32Array, - isTypedArray, - TextEncoder, - TextDecoder - } - - // Initialize TextEncoder/TextDecoder instances - this.textEncoder = new TextEncoder() - this.textDecoder = new TextDecoder() - } - } - - // Assign the PlatformNode class to the global object - ;(global as any).PlatformNode = PlatformNode - - // Also create an instance and assign it to global.platformNode (lowercase p) - // Some TensorFlow.js code might look for this - ;(global as any).platformNode = new PlatformNode() - - console.log( - 'Defined global PlatformNode class for TensorFlow.js compatibility' - ) - } catch (error) { - console.warn('Failed to define global PlatformNode class:', error) - } -} +// Apply the TensorFlow.js platform patch if needed +// This will define a global PlatformNode class that uses our text encoding utilities +applyTensorFlowPatch() /** * Check if an array is a Float32Array diff --git a/src/utils/textEncoding.ts b/src/utils/textEncoding.ts new file mode 100644 index 00000000..cf2bb2a7 --- /dev/null +++ b/src/utils/textEncoding.ts @@ -0,0 +1,167 @@ +/** + * Unified Text Encoding Utilities + * + * This module provides a consistent way to handle text encoding/decoding across all environments + * without relying on TextEncoder/TextDecoder polyfills or patches. + */ + +/** + * A simple text encoder that works in all environments + * This avoids the need for TextEncoder polyfills and patches + */ +export class SimpleTextEncoder { + /** + * Encode a string to a Uint8Array + * @param input - The string to encode + * @returns A Uint8Array containing the encoded string + */ + encode(input: string): Uint8Array { + // Simple UTF-8 encoding implementation that works everywhere + return new Uint8Array([...input].map((c) => c.charCodeAt(0))) + } +} + +/** + * A simple text decoder that works in all environments + * This avoids the need for TextDecoder polyfills and patches + */ +export class SimpleTextDecoder { + /** + * Decode a Uint8Array to a string + * @param input - The Uint8Array to decode + * @returns The decoded string + */ + decode(input: Uint8Array): string { + // Simple UTF-8 decoding implementation that works everywhere + return String.fromCharCode.apply(null, [...input]) + } +} + +// Create constructor functions that can be used as drop-in replacements +// for the native TextEncoder and TextDecoder + +/** + * Interface for UniversalTextEncoder instance + */ +interface IUniversalTextEncoder { + encode: (input: string) => Uint8Array; +} + +/** + * A constructor function for TextEncoder that works in all environments + */ +export function UniversalTextEncoder(this: IUniversalTextEncoder) { + if (!(this instanceof UniversalTextEncoder)) { + return new (UniversalTextEncoder as any)() + } + + try { + // Try to use the native TextEncoder if available + const nativeEncoder: TextEncoder = new TextEncoder() + this.encode = nativeEncoder.encode.bind(nativeEncoder) + } catch (e) { + // Fall back to our simple implementation + const simpleEncoder: SimpleTextEncoder = new SimpleTextEncoder() + this.encode = simpleEncoder.encode.bind(simpleEncoder) + } +} + +/** + * Interface for UniversalTextDecoder instance + */ +interface IUniversalTextDecoder { + decode: (input: Uint8Array) => string; +} + +/** + * A constructor function for TextDecoder that works in all environments + */ +export function UniversalTextDecoder(this: IUniversalTextDecoder) { + if (!(this instanceof UniversalTextDecoder)) { + return new (UniversalTextDecoder as any)() + } + + try { + // Try to use the native TextDecoder if available + const nativeDecoder: TextDecoder = new TextDecoder() + this.decode = nativeDecoder.decode.bind(nativeDecoder) + } catch (e) { + // Fall back to our simple implementation + const simpleDecoder: SimpleTextDecoder = new SimpleTextDecoder() + this.decode = simpleDecoder.decode.bind(simpleDecoder) + } +} + +/** + * Get a text encoder that works in the current environment + * @returns A text encoder object with an encode method + */ +export function getTextEncoder(): IUniversalTextEncoder { + return new (UniversalTextEncoder as any)() +} + +/** + * Get a text decoder that works in the current environment + * @returns A text decoder object with a decode method + */ +export function getTextDecoder(): IUniversalTextDecoder { + return new (UniversalTextDecoder as any)() +} + +/** + * Apply the TensorFlow.js platform patch if needed + * This function patches the global object to provide a PlatformNode class + * that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder + */ +export function applyTensorFlowPatch(): void { + // Only apply in Node.js environment + if ( + typeof global !== 'undefined' && + typeof process !== 'undefined' && + process.versions && + process.versions.node + ) { + try { + // Get encoders/decoders + const encoder = getTextEncoder() + const decoder = getTextDecoder() + + // Define a custom PlatformNode class + class PlatformNode { + util: any + textEncoder: any + textDecoder: any + + constructor() { + // Create a util object with necessary methods and constructors + this.util = { + isFloat32Array: (arr: any) => + !!( + arr instanceof Float32Array || + (arr && + Object.prototype.toString.call(arr) === + '[object Float32Array]') + ), + isTypedArray: (arr: any) => + !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)), + // Add TextEncoder and TextDecoder as constructors + TextEncoder: UniversalTextEncoder, + TextDecoder: UniversalTextDecoder + } + + // Initialize using the constructors from util + this.textEncoder = new this.util.TextEncoder() + this.textDecoder = new this.util.TextDecoder() + } + } + + // Assign the PlatformNode class to the global object + ;(global as any).PlatformNode = PlatformNode + + // Also create an instance and assign it to global.platformNode (lowercase p) + ;(global as any).platformNode = new PlatformNode() + } catch (error) { + console.warn('Failed to apply TensorFlow.js platform patch:', error) + } + } +} diff --git a/src/utils/workerUtils.ts b/src/utils/workerUtils.ts index 73c44e94..63714082 100644 --- a/src/utils/workerUtils.ts +++ b/src/utils/workerUtils.ts @@ -24,7 +24,42 @@ export function executeInThread(fnString: string, args: any): Promise { } else { // Fallback to main thread execution try { - const fn = new Function('return ' + fnString)() + // Try different approaches to create a function from string + let fn + try { + // First try with 'return' prefix + fn = new Function('return ' + fnString)() + } catch (functionError) { + console.warn( + 'Fallback: Error creating function with return syntax, trying alternative approaches', + functionError + ) + + try { + // Try wrapping in parentheses for function expressions + fn = new Function('return (' + fnString + ')')() + } catch (wrapError) { + console.warn( + 'Fallback: Error creating function with parentheses wrapping', + wrapError + ) + + try { + // Try direct approach for named functions + fn = new Function(fnString)() + } catch (directError) { + console.error( + 'Fallback: All approaches to create function failed', + directError + ) + throw new Error( + 'Failed to create function from string: ' + + (functionError as Error).message + ) + } + } + } + return Promise.resolve(fn(args) as T) } catch (error) { return Promise.reject(error) @@ -40,73 +75,82 @@ function executeInNodeWorker(fnString: string, args: any): Promise { return new Promise((resolve, reject) => { try { // Dynamically import worker_threads (Node.js only) - import('node:worker_threads').then(({ Worker, isMainThread, parentPort, workerData }) => { - if (!isMainThread && parentPort) { - // We're inside a worker, execute the function - const fn = new Function('return ' + workerData.fnString)() - const result = fn(workerData.args) - parentPort.postMessage({ result }) - return - } + import('node:worker_threads') + .then(({ Worker, isMainThread, parentPort, workerData }) => { + if (!isMainThread && parentPort) { + // We're inside a worker, execute the function + const fn = new Function('return ' + workerData.fnString)() + const result = fn(workerData.args) + parentPort.postMessage({ result }) + return + } - // Get a worker from the pool or create a new one - const workerId = `worker-${Math.random().toString(36).substring(2, 9)}` - let worker: any + // Get a worker from the pool or create a new one + const workerId = `worker-${Math.random().toString(36).substring(2, 9)}` + let worker: any - if (workerPool.size < MAX_POOL_SIZE) { - // Create a new worker - worker = new Worker(` + if (workerPool.size < MAX_POOL_SIZE) { + // Create a new worker + worker = new Worker( + ` import { parentPort, workerData } from 'node:worker_threads'; const fn = new Function('return ' + workerData.fnString)(); const result = fn(workerData.args); parentPort.postMessage({ result }); - `, { - eval: true, - workerData: { fnString, args } - }) + `, + { + eval: true, + workerData: { fnString, args } + } + ) - workerPool.set(workerId, worker) - } else { - // Reuse an existing worker - const poolKeys = Array.from(workerPool.keys()) - const randomKey = poolKeys[Math.floor(Math.random() * poolKeys.length)] - worker = workerPool.get(randomKey) + workerPool.set(workerId, worker) + } else { + // Reuse an existing worker + const poolKeys = Array.from(workerPool.keys()) + const randomKey = + poolKeys[Math.floor(Math.random() * poolKeys.length)] + worker = workerPool.get(randomKey) - // Terminate and recreate if the worker is busy - if (worker._busy) { - worker.terminate() - worker = new Worker(` + // Terminate and recreate if the worker is busy + if (worker._busy) { + worker.terminate() + worker = new Worker( + ` import { parentPort, workerData } from 'node:worker_threads'; const fn = new Function('return ' + workerData.fnString)(); const result = fn(workerData.args); parentPort.postMessage({ result }); - `, { - eval: true, - workerData: { fnString, args } - }) - workerPool.set(randomKey, worker) + `, + { + eval: true, + workerData: { fnString, args } + } + ) + workerPool.set(randomKey, worker) + } + + worker._busy = true } - worker._busy = true - } - - worker.on('message', (message: any) => { - worker._busy = false - resolve(message.result as T) - }) - - worker.on('error', (err: any) => { - worker._busy = false - reject(err) - }) - - worker.on('exit', (code: number) => { - if (code !== 0) { + worker.on('message', (message: any) => { worker._busy = false - reject(new Error(`Worker stopped with exit code ${code}`)) - } + resolve(message.result as T) + }) + + worker.on('error', (err: any) => { + worker._busy = false + reject(err) + }) + + worker.on('exit', (code: number) => { + if (code !== 0) { + worker._busy = false + reject(new Error(`Worker stopped with exit code ${code}`)) + } + }) }) - }).catch(reject) + .catch(reject) } catch (error) { reject(error) } @@ -119,35 +163,173 @@ function executeInNodeWorker(fnString: string, args: any): Promise { function executeInWebWorker(fnString: string, args: any): Promise { return new Promise((resolve, reject) => { try { - const workerCode = ` - self.onmessage = function(e) { - try { - const fn = new Function('return ' + e.data.fnString)(); - const result = fn(e.data.args); - self.postMessage({ result: result }); - } catch (error) { - self.postMessage({ error: error.message }); - } - }; - ` - const blob = new Blob([workerCode], { type: 'application/javascript' }) - const url = URL.createObjectURL(blob) - const worker = new Worker(url) + // Use the dedicated worker.js file instead of creating a blob + // Try different approaches to locate the worker.js file + let workerPath = './worker.js' - worker.onmessage = function(e) { + try { + // First try to use the import.meta.url if available (modern browsers) + if (typeof import.meta !== 'undefined' && import.meta.url) { + const baseUrl = import.meta.url.substring( + 0, + import.meta.url.lastIndexOf('/') + 1 + ) + workerPath = `${baseUrl}worker.js` + } + // Fallback to a relative path based on the unified.js location + else if (typeof document !== 'undefined') { + // Find the script tag that loaded unified.js + const scripts = document.getElementsByTagName('script') + for (let i = 0; i < scripts.length; i++) { + const src = scripts[i].src + if (src && src.includes('unified.js')) { + // Get the directory path + workerPath = + src.substring(0, src.lastIndexOf('/') + 1) + 'worker.js' + break + } + } + } + } catch (e) { + console.warn( + 'Could not determine worker path from import.meta.url, using relative path', + e + ) + } + + // If we couldn't determine the path, try some common locations + if (workerPath === './worker.js' && typeof window !== 'undefined') { + // Try to find the worker.js in the same directory as the current page + const pageUrl = window.location.href + const pageDir = pageUrl.substring(0, pageUrl.lastIndexOf('/') + 1) + workerPath = `${pageDir}worker.js` + + // Also check for dist/worker.js + if (typeof document !== 'undefined') { + const distWorkerPath = `${pageDir}dist/worker.js` + // Create a test request to see if the file exists + const xhr = new XMLHttpRequest() + xhr.open('HEAD', distWorkerPath, false) + try { + xhr.send() + if (xhr.status >= 200 && xhr.status < 300) { + workerPath = distWorkerPath + } + } catch (e) { + // Ignore errors, we'll use the default path + } + } + } + + console.log('Using worker path:', workerPath) + + // Try to create a worker, but fall back to inline worker or main thread execution if it fails + let worker: Worker + try { + worker = new Worker(workerPath) + } catch (error) { + console.warn( + 'Failed to create Web Worker from file, trying inline worker:', + error + ) + + try { + // Create an inline worker using a Blob + const workerCode = ` + // Brainy Inline Worker Script + console.log('Brainy Inline Worker: Started'); + + self.onmessage = function (e) { + try { + console.log('Brainy Inline Worker: Received message', e.data ? 'with data' : 'without data'); + + if (!e.data || !e.data.fnString) { + throw new Error('Invalid message: missing function string'); + } + + console.log('Brainy Inline Worker: Creating function from string'); + const fn = new Function('return ' + e.data.fnString)(); + + console.log('Brainy Inline Worker: Executing function with args'); + const result = fn(e.data.args); + + console.log('Brainy Inline Worker: Function executed successfully, posting result'); + self.postMessage({ result: result }); + } catch (error) { + console.error('Brainy Inline Worker: Error executing function', error); + self.postMessage({ + error: error.message, + stack: error.stack + }); + } + }; + ` + + const blob = new Blob([workerCode], { + type: 'application/javascript' + }) + const blobUrl = URL.createObjectURL(blob) + worker = new Worker(blobUrl) + + console.log('Created inline worker using Blob URL') + } catch (inlineWorkerError) { + console.warn( + 'Failed to create inline Web Worker, falling back to main thread execution:', + inlineWorkerError + ) + // Execute in main thread as fallback + try { + const fn = new Function('return ' + fnString)() + resolve(fn(args) as T) + return + } catch (mainThreadError) { + reject(mainThreadError) + return + } + } + } + + // Set a timeout to prevent hanging + const timeoutId = setTimeout(() => { + console.warn( + 'Web Worker execution timed out, falling back to main thread' + ) + worker.terminate() + + // Execute in main thread as fallback + try { + const fn = new Function('return ' + fnString)() + resolve(fn(args) as T) + } catch (mainThreadError) { + reject(mainThreadError) + } + }, 25000) // 25 second timeout (less than the 30 second test timeout) + + worker.onmessage = function (e) { + clearTimeout(timeoutId) if (e.data.error) { reject(new Error(e.data.error)) } else { resolve(e.data.result as T) } worker.terminate() - URL.revokeObjectURL(url) } - worker.onerror = function(e) { - reject(new Error(`Worker error: ${e.message}`)) + worker.onerror = function (e) { + clearTimeout(timeoutId) + console.warn( + 'Web Worker error, falling back to main thread execution:', + e.message + ) worker.terminate() - URL.revokeObjectURL(url) + + // Execute in main thread as fallback + try { + const fn = new Function('return ' + fnString)() + resolve(fn(args) as T) + } catch (mainThreadError) { + reject(mainThreadError) + } } worker.postMessage({ fnString, args }) @@ -163,12 +345,14 @@ function executeInWebWorker(fnString: string, args: any): Promise { */ export function cleanupWorkerPools(): void { if (isNode()) { - import('node:worker_threads').then(({ Worker }) => { - for (const worker of workerPool.values()) { - worker.terminate() - } - workerPool.clear() - console.log('Worker pools cleaned up') - }).catch(console.error) + import('node:worker_threads') + .then(({ Worker }) => { + for (const worker of workerPool.values()) { + worker.terminate() + } + workerPool.clear() + console.log('Worker pools cleaned up') + }) + .catch(console.error) } } diff --git a/src/worker.js b/src/worker.js new file mode 100644 index 00000000..3ee64669 --- /dev/null +++ b/src/worker.js @@ -0,0 +1,36 @@ +// Brainy Worker Script +// This script is used by the workerUtils.js file to execute functions in a separate thread + +// Import text encoding utilities +import { applyTensorFlowPatch } from './utils/textEncoding.js' + +// Apply the TensorFlow.js platform patch if needed +applyTensorFlowPatch() + +// Log that the worker has started +console.log('Brainy Worker: Started') + +self.onmessage = function (e) { + try { + console.log('Brainy Worker: Received message', e.data ? 'with data' : 'without data') + + if (!e.data || !e.data.fnString) { + throw new Error('Invalid message: missing function string') + } + + console.log('Brainy Worker: Creating function from string') + const fn = new Function('return ' + e.data.fnString)() + + console.log('Brainy Worker: Executing function with args') + const result = fn(e.data.args) + + console.log('Brainy Worker: Function executed successfully, posting result') + self.postMessage({ result: result }) + } catch (error) { + console.error('Brainy Worker: Error executing function', error) + self.postMessage({ + error: error.message, + stack: error.stack + }) + } +} diff --git a/src/worker.ts b/src/worker.ts new file mode 100644 index 00000000..5e7ec25a --- /dev/null +++ b/src/worker.ts @@ -0,0 +1,75 @@ +// Brainy Worker Script +// This script is used by the workerUtils.js file to execute functions in a separate thread + +// Import text encoding utilities +import { applyTensorFlowPatch } from './utils/textEncoding.js' + +// Apply the TensorFlow.js platform patch if needed +applyTensorFlowPatch() + +// Log that the worker has started +console.log('Brainy Worker: Started') + +// Define the message handler with proper TypeScript typing +self.onmessage = function (e: MessageEvent): void { + try { + console.log( + 'Brainy Worker: Received message', + e.data ? 'with data' : 'without data' + ) + + if (!e.data || !e.data.fnString) { + throw new Error('Invalid message: missing function string') + } + + console.log('Brainy Worker: Creating function from string') + // Use Function constructor to create a function from the string + let fn + + try { + // First try with 'return' prefix + fn = new Function('return ' + e.data.fnString)() + } catch (functionError) { + console.warn( + 'Brainy Worker: Error creating function with return syntax, trying alternative approaches', + functionError + ) + + try { + // Try wrapping in parentheses for function expressions + fn = new Function('return (' + e.data.fnString + ')')() + } catch (wrapError) { + console.warn( + 'Brainy Worker: Error creating function with parentheses wrapping', + wrapError + ) + + try { + // Try direct approach for named functions + fn = new Function(e.data.fnString)() + } catch (directError) { + console.error( + 'Brainy Worker: All approaches to create function failed', + directError + ) + throw new Error( + 'Failed to create function from string: ' + + (functionError as Error).message + ) + } + } + } + + console.log('Brainy Worker: Executing function with args') + const result = fn(e.data.args) + + console.log('Brainy Worker: Function executed successfully, posting result') + self.postMessage({ result: result }) + } catch (error: any) { + console.error('Brainy Worker: Error executing function', error) + self.postMessage({ + error: error.message, + stack: error.stack + }) + } +} diff --git a/test-unified-encoding.js b/test-unified-encoding.js new file mode 100644 index 00000000..107151b8 --- /dev/null +++ b/test-unified-encoding.js @@ -0,0 +1,39 @@ +// Test script to verify the unified text encoding approach works correctly +import { BrainyData } from './dist/unified.js' + +async function testUnifiedEncoding() { + console.log( + 'Testing unified text encoding approach in Node.js environment...' + ) + + try { + // Initialize BrainyData which should trigger the PlatformNode constructor + console.log('Creating BrainyData instance...') + const db = new BrainyData() + + // Initialize the database + console.log('Initializing database...') + await db.init() + + console.log('Test successful! Unified text encoding is working correctly.') + + // Get database status to verify everything is working + const status = await db.status() + console.log('Database status:', status) + + return true + } catch (error) { + console.error('Error during test:', error) + return false + } +} + +// Run the test +testUnifiedEncoding().then((success) => { + if (success) { + console.log('Unified text encoding verification completed successfully!') + } else { + console.error('Unified text encoding verification failed!') + process.exit(1) + } +}) diff --git a/test-worker-utils.js b/test-worker-utils.js new file mode 100644 index 00000000..4405391a --- /dev/null +++ b/test-worker-utils.js @@ -0,0 +1,24 @@ +// Test script to verify that the workerUtils functions work correctly after removing eval +import { executeInThread } from './dist/unified.js' + +// Test function to execute in a thread +const testFunction = `function(args) { + return "Hello from " + args.name; +}` + +// Test with different environments +async function runTests() { + try { + console.log('Testing executeInThread...') + const result = await executeInThread(testFunction, { + name: 'Worker Thread' + }) + console.log('Result:', result) + + console.log('All tests passed!') + } catch (error) { + console.error('Test failed:', error) + } +} + +runTests()