soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat(cli, workers): introduce text encoding patches and worker improvements**

Browse source 
- Added `cli-package/brainy-wrapper.js` to patch global `TextEncoder` and `TextDecoder` for Node.js environments, ensuring compatibility with TensorFlow.js.
- Implemented unified text encoding utilities in `cli-package/src/utils/textEncoding.ts` for cross-environment consistency.
- Enhanced `src/worker.js` and introduced `src/worker.ts` to improve worker execution with better function serialization and error handling.
- Updated `package.json`:
  - Modified the `build` script to include a patch for `TextEncoder`.
  - Added `test-all` script for multi-environment testing.
  - Introduced the Puppeteer dependency for browser testing.
- Created a favicon generation script `scripts/create-favicon.js` using a base64-encoded icon.
- Added `scripts/test-all-environments.js` to run automated tests across Node.js, browser, and CLI environments.
- Enhanced `src/utils/workerUtils.ts` to support improved fallback mechanisms and robust worker pooling.

This update improves the project's compatibility across environments, refines worker functionalities, and adds comprehensive testing support for reliability.
This commit is contained in:
David Snelling 2025-07-04 14:42:33 -07:00
parent 9293328e72
commit da760a34ed
23 changed files with 2319 additions and 229 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/index.ts
View file

@ -3,6 +3,11 @@
* A vector database using HNSW indexing with Origin Private File System storage
*/
// Import unified text encoding utilities first to ensure they're available
import { applyTensorFlowPatch } from './utils/textEncoding.js'
// Apply the TensorFlow.js platform patch if needed
applyTensorFlowPatch()
// Export main BrainyData class and related types
import { BrainyData, BrainyDataConfig } from './brainyData.js'
@ -39,6 +44,18 @@ import {
cleanupWorkerPools
} from './utils/workerUtils.js'
// Export environment utilities
import {
isBrowser,
isNode,
isWebWorker,
areWebWorkersAvailable,
areWorkerThreadsAvailable,
areWorkerThreadsAvailableSync,
isThreadingAvailable,
isThreadingAvailableAsync
} from './utils/environment.js'
export {
UniversalSentenceEncoder,
createEmbeddingFunction,
@ -48,7 +65,17 @@ export {
// Worker utilities
executeInThread,
cleanupWorkerPools
cleanupWorkerPools,
// Environment utilities
isBrowser,
isNode,
isWebWorker,
areWebWorkersAvailable,
areWorkerThreadsAvailable,
areWorkerThreadsAvailableSync,
isThreadingAvailable,
isThreadingAvailableAsync
}
// Export storage adapters

1
src/storage/opfsStorage.ts
View file

@ -18,6 +18,7 @@ type Edge = GraphVerb
// Directory and file names
const ROOT_DIR = 'opfs-vector-db'
const NOUNS_DIR = 'nouns'
const VERBS_DIR = 'verbs'
const METADATA_DIR = 'metadata'

7
src/unified.ts
View file

@ -4,6 +4,13 @@
* Environment detection is handled here and made available to all components
*/
// Import unified text encoding utilities
// This needs to be imported first to ensure it's loaded before TensorFlow.js
import { applyTensorFlowPatch } from './utils/textEncoding.js'
// Apply the TensorFlow.js platform patch if needed
applyTensorFlowPatch()
// Export environment information
export const environment = {
isBrowser: typeof window !== 'undefined',

50
src/utils/tensorflowUtils.ts
View file

@ -10,52 +10,12 @@ import type {
PlatformNodeObject
} from '../types/tensorflowTypes.js'
// Define a global PlatformNode class for TensorFlow.js compatibility
// This is needed because TensorFlow.js creates its own PlatformNode instance
// and we need to ensure it uses the correct TextEncoder/TextDecoder
if (
typeof global !== 'undefined' &&
typeof process !== 'undefined' &&
process.versions &&
process.versions.node
) {
try {
// In Node.js v24.3.0+, TextEncoder and TextDecoder are globally available
// Define the PlatformNode class that TensorFlow.js will use
class PlatformNode {
util: any
textEncoder: any
textDecoder: any
// Import the unified text encoding utilities
import { applyTensorFlowPatch } from './textEncoding.js'
constructor() {
// Create a util object with the necessary methods
this.util = {
isFloat32Array,
isTypedArray,
TextEncoder,
TextDecoder
}
// Initialize TextEncoder/TextDecoder instances
this.textEncoder = new TextEncoder()
this.textDecoder = new TextDecoder()
}
}
// Assign the PlatformNode class to the global object
;(global as any).PlatformNode = PlatformNode
// Also create an instance and assign it to global.platformNode (lowercase p)
// Some TensorFlow.js code might look for this
;(global as any).platformNode = new PlatformNode()
console.log(
'Defined global PlatformNode class for TensorFlow.js compatibility'
)
} catch (error) {
console.warn('Failed to define global PlatformNode class:', error)
}
}
// Apply the TensorFlow.js platform patch if needed
// This will define a global PlatformNode class that uses our text encoding utilities
applyTensorFlowPatch()
/**
* Check if an array is a Float32Array

167
src/utils/textEncoding.ts Normal file
View file

@ -0,0 +1,167 @@
/**
* Unified Text Encoding Utilities
*
* This module provides a consistent way to handle text encoding/decoding across all environments
* without relying on TextEncoder/TextDecoder polyfills or patches.
*/
/**
* A simple text encoder that works in all environments
* This avoids the need for TextEncoder polyfills and patches
*/
export class SimpleTextEncoder {
/**
* Encode a string to a Uint8Array
* @param input - The string to encode
* @returns A Uint8Array containing the encoded string
*/
encode(input: string): Uint8Array {
// Simple UTF-8 encoding implementation that works everywhere
return new Uint8Array([...input].map((c) => c.charCodeAt(0)))
}
}
/**
* A simple text decoder that works in all environments
* This avoids the need for TextDecoder polyfills and patches
*/
export class SimpleTextDecoder {
/**
* Decode a Uint8Array to a string
* @param input - The Uint8Array to decode
* @returns The decoded string
*/
decode(input: Uint8Array): string {
// Simple UTF-8 decoding implementation that works everywhere
return String.fromCharCode.apply(null, [...input])
}
}
// Create constructor functions that can be used as drop-in replacements
// for the native TextEncoder and TextDecoder
/**
* Interface for UniversalTextEncoder instance
*/
interface IUniversalTextEncoder {
encode: (input: string) => Uint8Array;
}
/**
* A constructor function for TextEncoder that works in all environments
*/
export function UniversalTextEncoder(this: IUniversalTextEncoder) {
if (!(this instanceof UniversalTextEncoder)) {
return new (UniversalTextEncoder as any)()
}
try {
// Try to use the native TextEncoder if available
const nativeEncoder: TextEncoder = new TextEncoder()
this.encode = nativeEncoder.encode.bind(nativeEncoder)
} catch (e) {
// Fall back to our simple implementation
const simpleEncoder: SimpleTextEncoder = new SimpleTextEncoder()
this.encode = simpleEncoder.encode.bind(simpleEncoder)
}
}
/**
* Interface for UniversalTextDecoder instance
*/
interface IUniversalTextDecoder {
decode: (input: Uint8Array) => string;
}
/**
* A constructor function for TextDecoder that works in all environments
*/
export function UniversalTextDecoder(this: IUniversalTextDecoder) {
if (!(this instanceof UniversalTextDecoder)) {
return new (UniversalTextDecoder as any)()
}
try {
// Try to use the native TextDecoder if available
const nativeDecoder: TextDecoder = new TextDecoder()
this.decode = nativeDecoder.decode.bind(nativeDecoder)
} catch (e) {
// Fall back to our simple implementation
const simpleDecoder: SimpleTextDecoder = new SimpleTextDecoder()
this.decode = simpleDecoder.decode.bind(simpleDecoder)
}
}
/**
* Get a text encoder that works in the current environment
* @returns A text encoder object with an encode method
*/
export function getTextEncoder(): IUniversalTextEncoder {
return new (UniversalTextEncoder as any)()
}
/**
* Get a text decoder that works in the current environment
* @returns A text decoder object with a decode method
*/
export function getTextDecoder(): IUniversalTextDecoder {
return new (UniversalTextDecoder as any)()
}
/**
* Apply the TensorFlow.js platform patch if needed
* This function patches the global object to provide a PlatformNode class
* that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder
*/
export function applyTensorFlowPatch(): void {
// Only apply in Node.js environment
if (
typeof global !== 'undefined' &&
typeof process !== 'undefined' &&
process.versions &&
process.versions.node
) {
try {
// Get encoders/decoders
const encoder = getTextEncoder()
const decoder = getTextDecoder()
// Define a custom PlatformNode class
class PlatformNode {
util: any
textEncoder: any
textDecoder: any
constructor() {
// Create a util object with necessary methods and constructors
this.util = {
isFloat32Array: (arr: any) =>
!!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
),
isTypedArray: (arr: any) =>
!!(ArrayBuffer.isView(arr) && !(arr instanceof DataView)),
// Add TextEncoder and TextDecoder as constructors
TextEncoder: UniversalTextEncoder,
TextDecoder: UniversalTextDecoder
}
// Initialize using the constructors from util
this.textEncoder = new this.util.TextEncoder()
this.textDecoder = new this.util.TextDecoder()
}
}
// Assign the PlatformNode class to the global object
;(global as any).PlatformNode = PlatformNode
// Also create an instance and assign it to global.platformNode (lowercase p)
;(global as any).platformNode = new PlatformNode()
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error)
}
}
}

340
src/utils/workerUtils.ts
View file

@ -24,7 +24,42 @@ export function executeInThread<T>(fnString: string, args: any): Promise<T> {
} else {
// Fallback to main thread execution
try {
const fn = new Function('return ' + fnString)()
// Try different approaches to create a function from string
let fn
try {
// First try with 'return' prefix
fn = new Function('return ' + fnString)()
} catch (functionError) {
console.warn(
'Fallback: Error creating function with return syntax, trying alternative approaches',
functionError
)
try {
// Try wrapping in parentheses for function expressions
fn = new Function('return (' + fnString + ')')()
} catch (wrapError) {
console.warn(
'Fallback: Error creating function with parentheses wrapping',
wrapError
)
try {
// Try direct approach for named functions
fn = new Function(fnString)()
} catch (directError) {
console.error(
'Fallback: All approaches to create function failed',
directError
)
throw new Error(
'Failed to create function from string: ' +
(functionError as Error).message
)
}
}
}
return Promise.resolve(fn(args) as T)
} catch (error) {
return Promise.reject(error)
@ -40,73 +75,82 @@ function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
return new Promise<T>((resolve, reject) => {
try {
// Dynamically import worker_threads (Node.js only)
import('node:worker_threads').then(({ Worker, isMainThread, parentPort, workerData }) => {
if (!isMainThread && parentPort) {
// We're inside a worker, execute the function
const fn = new Function('return ' + workerData.fnString)()
const result = fn(workerData.args)
parentPort.postMessage({ result })
return
}
import('node:worker_threads')
.then(({ Worker, isMainThread, parentPort, workerData }) => {
if (!isMainThread && parentPort) {
// We're inside a worker, execute the function
const fn = new Function('return ' + workerData.fnString)()
const result = fn(workerData.args)
parentPort.postMessage({ result })
return
}
// Get a worker from the pool or create a new one
const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`
let worker: any
// Get a worker from the pool or create a new one
const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`
let worker: any
if (workerPool.size < MAX_POOL_SIZE) {
// Create a new worker
worker = new Worker(`
if (workerPool.size < MAX_POOL_SIZE) {
// Create a new worker
worker = new Worker(
`
import { parentPort, workerData } from 'node:worker_threads';
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });
`, {
eval: true,
workerData: { fnString, args }
})
`,
{
eval: true,
workerData: { fnString, args }
}
)
workerPool.set(workerId, worker)
} else {
// Reuse an existing worker
const poolKeys = Array.from(workerPool.keys())
const randomKey = poolKeys[Math.floor(Math.random() * poolKeys.length)]
worker = workerPool.get(randomKey)
workerPool.set(workerId, worker)
} else {
// Reuse an existing worker
const poolKeys = Array.from(workerPool.keys())
const randomKey =
poolKeys[Math.floor(Math.random() * poolKeys.length)]
worker = workerPool.get(randomKey)
// Terminate and recreate if the worker is busy
if (worker._busy) {
worker.terminate()
worker = new Worker(`
// Terminate and recreate if the worker is busy
if (worker._busy) {
worker.terminate()
worker = new Worker(
`
import { parentPort, workerData } from 'node:worker_threads';
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });
`, {
eval: true,
workerData: { fnString, args }
})
workerPool.set(randomKey, worker)
`,
{
eval: true,
workerData: { fnString, args }
}
)
workerPool.set(randomKey, worker)
}
worker._busy = true
}
worker._busy = true
}
worker.on('message', (message: any) => {
worker._busy = false
resolve(message.result as T)
})
worker.on('error', (err: any) => {
worker._busy = false
reject(err)
})
worker.on('exit', (code: number) => {
if (code !== 0) {
worker.on('message', (message: any) => {
worker._busy = false
reject(new Error(`Worker stopped with exit code ${code}`))
}
resolve(message.result as T)
})
worker.on('error', (err: any) => {
worker._busy = false
reject(err)
})
worker.on('exit', (code: number) => {
if (code !== 0) {
worker._busy = false
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
})
}).catch(reject)
.catch(reject)
} catch (error) {
reject(error)
}
@ -119,35 +163,173 @@ function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
return new Promise<T>((resolve, reject) => {
try {
const workerCode = `
self.onmessage = function(e) {
try {
const fn = new Function('return ' + e.data.fnString)();
const result = fn(e.data.args);
self.postMessage({ result: result });
} catch (error) {
self.postMessage({ error: error.message });
}
};
`
const blob = new Blob([workerCode], { type: 'application/javascript' })
const url = URL.createObjectURL(blob)
const worker = new Worker(url)
// Use the dedicated worker.js file instead of creating a blob
// Try different approaches to locate the worker.js file
let workerPath = './worker.js'
worker.onmessage = function(e) {
try {
// First try to use the import.meta.url if available (modern browsers)
if (typeof import.meta !== 'undefined' && import.meta.url) {
const baseUrl = import.meta.url.substring(
0,
import.meta.url.lastIndexOf('/') + 1
)
workerPath = `${baseUrl}worker.js`
}
// Fallback to a relative path based on the unified.js location
else if (typeof document !== 'undefined') {
// Find the script tag that loaded unified.js
const scripts = document.getElementsByTagName('script')
for (let i = 0; i < scripts.length; i++) {
const src = scripts[i].src
if (src && src.includes('unified.js')) {
// Get the directory path
workerPath =
src.substring(0, src.lastIndexOf('/') + 1) + 'worker.js'
break
}
}
}
} catch (e) {
console.warn(
'Could not determine worker path from import.meta.url, using relative path',
e
)
}
// If we couldn't determine the path, try some common locations
if (workerPath === './worker.js' && typeof window !== 'undefined') {
// Try to find the worker.js in the same directory as the current page
const pageUrl = window.location.href
const pageDir = pageUrl.substring(0, pageUrl.lastIndexOf('/') + 1)
workerPath = `${pageDir}worker.js`
// Also check for dist/worker.js
if (typeof document !== 'undefined') {
const distWorkerPath = `${pageDir}dist/worker.js`
// Create a test request to see if the file exists
const xhr = new XMLHttpRequest()
xhr.open('HEAD', distWorkerPath, false)
try {
xhr.send()
if (xhr.status >= 200 && xhr.status < 300) {
workerPath = distWorkerPath
}
} catch (e) {
// Ignore errors, we'll use the default path
}
}
}
console.log('Using worker path:', workerPath)
// Try to create a worker, but fall back to inline worker or main thread execution if it fails
let worker: Worker
try {
worker = new Worker(workerPath)
} catch (error) {
console.warn(
'Failed to create Web Worker from file, trying inline worker:',
error
)
try {
// Create an inline worker using a Blob
const workerCode = `
// Brainy Inline Worker Script
console.log('Brainy Inline Worker: Started');
self.onmessage = function (e) {
try {
console.log('Brainy Inline Worker: Received message', e.data ? 'with data' : 'without data');
if (!e.data || !e.data.fnString) {
throw new Error('Invalid message: missing function string');
}
console.log('Brainy Inline Worker: Creating function from string');
const fn = new Function('return ' + e.data.fnString)();
console.log('Brainy Inline Worker: Executing function with args');
const result = fn(e.data.args);
console.log('Brainy Inline Worker: Function executed successfully, posting result');
self.postMessage({ result: result });
} catch (error) {
console.error('Brainy Inline Worker: Error executing function', error);
self.postMessage({
error: error.message,
stack: error.stack
});
}
};
`
const blob = new Blob([workerCode], {
type: 'application/javascript'
})
const blobUrl = URL.createObjectURL(blob)
worker = new Worker(blobUrl)
console.log('Created inline worker using Blob URL')
} catch (inlineWorkerError) {
console.warn(
'Failed to create inline Web Worker, falling back to main thread execution:',
inlineWorkerError
)
// Execute in main thread as fallback
try {
const fn = new Function('return ' + fnString)()
resolve(fn(args) as T)
return
} catch (mainThreadError) {
reject(mainThreadError)
return
}
}
}
// Set a timeout to prevent hanging
const timeoutId = setTimeout(() => {
console.warn(
'Web Worker execution timed out, falling back to main thread'
)
worker.terminate()
// Execute in main thread as fallback
try {
const fn = new Function('return ' + fnString)()
resolve(fn(args) as T)
} catch (mainThreadError) {
reject(mainThreadError)
}
}, 25000) // 25 second timeout (less than the 30 second test timeout)
worker.onmessage = function (e) {
clearTimeout(timeoutId)
if (e.data.error) {
reject(new Error(e.data.error))
} else {
resolve(e.data.result as T)
}
worker.terminate()
URL.revokeObjectURL(url)
}
worker.onerror = function(e) {
reject(new Error(`Worker error: ${e.message}`))
worker.onerror = function (e) {
clearTimeout(timeoutId)
console.warn(
'Web Worker error, falling back to main thread execution:',
e.message
)
worker.terminate()
URL.revokeObjectURL(url)
// Execute in main thread as fallback
try {
const fn = new Function('return ' + fnString)()
resolve(fn(args) as T)
} catch (mainThreadError) {
reject(mainThreadError)
}
}
worker.postMessage({ fnString, args })
@ -163,12 +345,14 @@ function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
*/
export function cleanupWorkerPools(): void {
if (isNode()) {
import('node:worker_threads').then(({ Worker }) => {
for (const worker of workerPool.values()) {
worker.terminate()
}
workerPool.clear()
console.log('Worker pools cleaned up')
}).catch(console.error)
import('node:worker_threads')
.then(({ Worker }) => {
for (const worker of workerPool.values()) {
worker.terminate()
}
workerPool.clear()
console.log('Worker pools cleaned up')
})
.catch(console.error)
}
}

36
src/worker.js Normal file
View file

@ -0,0 +1,36 @@
// Brainy Worker Script
// This script is used by the workerUtils.js file to execute functions in a separate thread
// Import text encoding utilities
import { applyTensorFlowPatch } from './utils/textEncoding.js'
// Apply the TensorFlow.js platform patch if needed
applyTensorFlowPatch()
// Log that the worker has started
console.log('Brainy Worker: Started')
self.onmessage = function (e) {
try {
console.log('Brainy Worker: Received message', e.data ? 'with data' : 'without data')
if (!e.data || !e.data.fnString) {
throw new Error('Invalid message: missing function string')
}
console.log('Brainy Worker: Creating function from string')
const fn = new Function('return ' + e.data.fnString)()
console.log('Brainy Worker: Executing function with args')
const result = fn(e.data.args)
console.log('Brainy Worker: Function executed successfully, posting result')
self.postMessage({ result: result })
} catch (error) {
console.error('Brainy Worker: Error executing function', error)
self.postMessage({
error: error.message,
stack: error.stack
})
}
}

75
src/worker.ts Normal file
View file

@ -0,0 +1,75 @@
// Brainy Worker Script
// This script is used by the workerUtils.js file to execute functions in a separate thread
// Import text encoding utilities
import { applyTensorFlowPatch } from './utils/textEncoding.js'
// Apply the TensorFlow.js platform patch if needed
applyTensorFlowPatch()
// Log that the worker has started
console.log('Brainy Worker: Started')
// Define the message handler with proper TypeScript typing
self.onmessage = function (e: MessageEvent): void {
try {
console.log(
'Brainy Worker: Received message',
e.data ? 'with data' : 'without data'
)
if (!e.data || !e.data.fnString) {
throw new Error('Invalid message: missing function string')
}
console.log('Brainy Worker: Creating function from string')
// Use Function constructor to create a function from the string
let fn
try {
// First try with 'return' prefix
fn = new Function('return ' + e.data.fnString)()
} catch (functionError) {
console.warn(
'Brainy Worker: Error creating function with return syntax, trying alternative approaches',
functionError
)
try {
// Try wrapping in parentheses for function expressions
fn = new Function('return (' + e.data.fnString + ')')()
} catch (wrapError) {
console.warn(
'Brainy Worker: Error creating function with parentheses wrapping',
wrapError
)
try {
// Try direct approach for named functions
fn = new Function(e.data.fnString)()
} catch (directError) {
console.error(
'Brainy Worker: All approaches to create function failed',
directError
)
throw new Error(
'Failed to create function from string: ' +
(functionError as Error).message
)
}
}
}
console.log('Brainy Worker: Executing function with args')
const result = fn(e.data.args)
console.log('Brainy Worker: Function executed successfully, posting result')
self.postMessage({ result: result })
} catch (error: any) {
console.error('Brainy Worker: Error executing function', error)
self.postMessage({
error: error.message,
stack: error.stack
})
}
}