soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat(tests): add tests for TextEncoder, TensorFlow.js, and fallback mechanisms**

Browse source 
- Introduced `test-fallback-function.js` and `test-fallback-simple.js` to validate `executeInThread` fallback functionality with both named and anonymous compute-intensive functions.
- Added `test-tensorflow-textencoder.js` for TensorFlow.js and TextEncoder tests in a Node.js environment.
- Created `test-tensorflow-textencoder.html` for browser-based TensorFlow.js and TextEncoder tests.
- Implemented cross-environment test support in `cli-package/src/test-tensorflow-textencoder.ts` for CLI functionality.
- Enhanced `src/utils/embedding.ts`, `textEncoding.ts`, and `brainy-wrapper.js` to include updated global `TextEncoder` and `TextDecoder` utilities for compatibility and worker improvements.
- Standardized and expanded utility methods in `PlatformNode` for broader support, including `isFloat32Array` and `isTypedArray` checks.
- Updated Node.js requirement to `>= 24.4.0` across documentation and configuration files for compatibility improvements.

This update introduces comprehensive testing for fallback mechanisms, TensorFlow.js, and TextEncoder across multiple environments, ensuring robustness and compatibility.
This commit is contained in:
David Snelling 2025-07-11 11:11:56 -07:00
parent 00039f836f
commit f0db5b471f
30 changed files with 1799 additions and 1583 deletions
Download patch file Download diff file Expand all files Collapse all files

83
src/utils/embedding.ts
View file

@ -34,7 +34,88 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
return
}
// No compatibility patches needed - TensorFlow.js now works correctly with Node.js 24+
// Add polyfill for isFloat32Array in Node.js 24.4.0
// This fixes the "Cannot read properties of undefined (reading 'isFloat32Array')" error
if (typeof global !== 'undefined') {
try {
// Define a custom PlatformNode class
class PlatformNode {
util: any
textEncoder: TextEncoder
textDecoder: TextDecoder
constructor() {
// Create a util object with necessary methods
this.util = {
// Add isFloat32Array and isTypedArray directly to util
isFloat32Array: (arr: any) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
)
},
isTypedArray: (arr: any) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
},
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
}
// Initialize encoders using native constructors
this.textEncoder = new TextEncoder()
this.textDecoder = new TextDecoder()
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr: any) {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
// Define isTypedArray directly on the instance
isTypedArray(arr: any) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
}
}
// Assign the PlatformNode class to the global object
;(global as any).PlatformNode = PlatformNode
// Also create an instance and assign it to global.platformNode
;(global as any).platformNode = new PlatformNode()
} catch (error) {
console.warn('Failed to define global PlatformNode class:', error)
}
// Ensure the util object exists
if (!global.util) {
global.util = {}
}
// Add isFloat32Array method if it doesn't exist
if (!global.util.isFloat32Array) {
global.util.isFloat32Array = (obj: any) => {
return !!(
obj instanceof Float32Array ||
(obj &&
Object.prototype.toString.call(obj) === '[object Float32Array]')
)
}
}
// Add isTypedArray method if it doesn't exist
if (!global.util.isTypedArray) {
global.util.isTypedArray = (obj: any) => {
return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView))
}
}
}
}
/**

2
src/utils/environment.ts
View file

@ -60,7 +60,7 @@ export async function areWorkerThreadsAvailable(): Promise<boolean> {
export function areWorkerThreadsAvailableSync(): boolean {
if (!isNode()) return false
// In Node.js 24.3.0+, worker_threads is always available
// In Node.js 24.4.0+, worker_threads is always available
return parseInt(process.versions.node.split('.')[0]) >= 24
}

8
src/utils/tensorflowUtils.ts
View file

@ -10,12 +10,8 @@ import type {
PlatformNodeObject
} from '../types/tensorflowTypes.js'
// Import the unified text encoding utilities
import { applyTensorFlowPatch } from './textEncoding.js'
// Apply the TensorFlow.js platform patch if needed
// This will define a global PlatformNode class that uses our text encoding utilities
applyTensorFlowPatch()
// Note: TensorFlow.js platform patch is applied in setup.ts
// This ensures the global PlatformNode class uses our text encoding utilities
/**
* Check if an array is a Float32Array

146
src/utils/textEncoding.ts
View file

@ -2,140 +2,49 @@
* Unified Text Encoding Utilities
*
* This module provides a consistent way to handle text encoding/decoding across all environments
* without relying on TextEncoder/TextDecoder polyfills or patches.
* using the native TextEncoder/TextDecoder APIs.
*/
/**
* A simple text encoder that works in all environments
* This avoids the need for TextEncoder polyfills and patches
*/
export class SimpleTextEncoder {
/**
* Encode a string to a Uint8Array
* @param input - The string to encode
* @returns A Uint8Array containing the encoded string
*/
encode(input: string): Uint8Array {
// Simple UTF-8 encoding implementation that works everywhere
return new Uint8Array([...input].map((c) => c.charCodeAt(0)))
}
}
/**
* A simple text decoder that works in all environments
* This avoids the need for TextDecoder polyfills and patches
*/
export class SimpleTextDecoder {
/**
* Decode a Uint8Array to a string
* @param input - The Uint8Array to decode
* @returns The decoded string
*/
decode(input: Uint8Array): string {
// Simple UTF-8 decoding implementation that works everywhere
return String.fromCharCode.apply(null, [...input])
}
}
// Create constructor functions that can be used as drop-in replacements
// for the native TextEncoder and TextDecoder
/**
* Interface for UniversalTextEncoder instance
*/
interface IUniversalTextEncoder {
encode: (input: string) => Uint8Array;
}
/**
* A constructor function for TextEncoder that works in all environments
*/
export function UniversalTextEncoder(this: IUniversalTextEncoder) {
if (!(this instanceof UniversalTextEncoder)) {
return new (UniversalTextEncoder as any)()
}
try {
// Try to use the native TextEncoder if available
const nativeEncoder: TextEncoder = new TextEncoder()
this.encode = nativeEncoder.encode.bind(nativeEncoder)
} catch (e) {
// Fall back to our simple implementation
const simpleEncoder: SimpleTextEncoder = new SimpleTextEncoder()
this.encode = simpleEncoder.encode.bind(simpleEncoder)
}
}
/**
* Interface for UniversalTextDecoder instance
*/
interface IUniversalTextDecoder {
decode: (input: Uint8Array) => string;
}
/**
* A constructor function for TextDecoder that works in all environments
*/
export function UniversalTextDecoder(this: IUniversalTextDecoder) {
if (!(this instanceof UniversalTextDecoder)) {
return new (UniversalTextDecoder as any)()
}
try {
// Try to use the native TextDecoder if available
const nativeDecoder: TextDecoder = new TextDecoder()
this.decode = nativeDecoder.decode.bind(nativeDecoder)
} catch (e) {
// Fall back to our simple implementation
const simpleDecoder: SimpleTextDecoder = new SimpleTextDecoder()
this.decode = simpleDecoder.decode.bind(simpleDecoder)
}
}
/**
* Get a text encoder that works in the current environment
* @returns A text encoder object with an encode method
* @returns A TextEncoder instance
*/
export function getTextEncoder(): IUniversalTextEncoder {
return new (UniversalTextEncoder as any)()
export function getTextEncoder(): TextEncoder {
return new TextEncoder()
}
/**
* Get a text decoder that works in the current environment
* @returns A text decoder object with a decode method
* @returns A TextDecoder instance
*/
export function getTextDecoder(): IUniversalTextDecoder {
return new (UniversalTextDecoder as any)()
export function getTextDecoder(): TextDecoder {
return new TextDecoder()
}
/**
* Apply the TensorFlow.js platform patch if needed
* This function patches the global object to provide a PlatformNode class
* that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder
* that uses native TextEncoder/TextDecoder
*/
export function applyTensorFlowPatch(): void {
try {
// Get encoders/decoders
const encoder = getTextEncoder()
const decoder = getTextDecoder()
// Define a custom Platform class that works in both Node.js and browser environments
class Platform {
util: any
textEncoder: any
textDecoder: any
textEncoder: TextEncoder
textDecoder: TextDecoder
constructor() {
// Create a util object with necessary methods and constructors
this.util = {
// Add TextEncoder and TextDecoder as constructors
TextEncoder: UniversalTextEncoder,
TextDecoder: UniversalTextDecoder
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
}
// Initialize using the constructors from util
this.textEncoder = new this.util.TextEncoder()
this.textDecoder = new this.util.TextDecoder()
// Initialize using native constructors
this.textEncoder = new TextEncoder()
this.textDecoder = new TextDecoder()
}
// Define isFloat32Array directly on the instance
@ -143,8 +52,7 @@ export function applyTensorFlowPatch(): void {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
@ -155,10 +63,14 @@ export function applyTensorFlowPatch(): void {
}
// Get the global object in a way that works in both Node.js and browser
const globalObj = typeof global !== 'undefined' ? global :
typeof window !== 'undefined' ? window :
typeof self !== 'undefined' ? self :
{};
const globalObj =
typeof global !== 'undefined'
? global
: typeof window !== 'undefined'
? window
: typeof self !== 'undefined'
? self
: {}
// Only apply in Node.js environment
if (
@ -167,14 +79,14 @@ export function applyTensorFlowPatch(): void {
process.versions.node
) {
// Assign the Platform class to the global object as PlatformNode for Node.js
(globalObj as any).PlatformNode = Platform;
;(globalObj as any).PlatformNode = Platform
// Also create an instance and assign it to global.platformNode (lowercase p)
(globalObj as any).platformNode = new Platform();
;(globalObj as any).platformNode = new Platform()
} else if (typeof window !== 'undefined' || typeof self !== 'undefined') {
// In browser environments, we might need to provide similar functionality
// but we'll use a different name to avoid conflicts
(globalObj as any).PlatformBrowser = Platform;
(globalObj as any).platformBrowser = new Platform();
;(globalObj as any).PlatformBrowser = Platform
;(globalObj as any).platformBrowser = new Platform()
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error)

165
src/utils/workerUtils.ts
View file

@ -48,14 +48,26 @@ export function executeInThread<T>(fnString: string, args: any): Promise<T> {
// Try direct approach for named functions
fn = new Function(fnString)()
} catch (directError) {
console.error(
'Fallback: All approaches to create function failed',
console.warn(
'Fallback: Direct approach failed, trying with function wrapper',
directError
)
throw new Error(
'Failed to create function from string: ' +
(functionError as Error).message
)
try {
// Try wrapping in a function that returns the function expression
fn = new Function(
'return function(args) { return (' + fnString + ')(args); }'
)()
} catch (wrapperError) {
console.error(
'Fallback: All approaches to create function failed',
wrapperError
)
throw new Error(
'Failed to create function from string: ' +
(functionError as Error).message
)
}
}
}
}
@ -94,6 +106,82 @@ function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
worker = new Worker(
`
import { parentPort, workerData } from 'node:worker_threads';
// Add TensorFlow.js platform patch for Node.js
if (typeof global !== 'undefined') {
try {
// Define a custom PlatformNode class
class PlatformNode {
constructor() {
// Create a util object with necessary methods
this.util = {
// Add isFloat32Array and isTypedArray directly to util
isFloat32Array: (arr) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
);
},
isTypedArray: (arr) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
},
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
};
// Initialize encoders using native constructors
this.textEncoder = new TextEncoder();
this.textDecoder = new TextDecoder();
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr) {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
}
// Define isTypedArray directly on the instance
isTypedArray(arr) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
}
}
// Assign the PlatformNode class to the global object
global.PlatformNode = PlatformNode;
// Also create an instance and assign it to global.platformNode
global.platformNode = new PlatformNode();
// Ensure global.util exists and has the necessary methods
if (!global.util) {
global.util = {};
}
// Add isFloat32Array method if it doesn't exist
if (!global.util.isFloat32Array) {
global.util.isFloat32Array = (arr) => {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
};
}
// Add isTypedArray method if it doesn't exist
if (!global.util.isTypedArray) {
global.util.isTypedArray = (arr) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
};
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error);
}
}
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });
@ -118,6 +206,71 @@ function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
worker = new Worker(
`
import { parentPort, workerData } from 'node:worker_threads';
// Add TensorFlow.js platform patch for Node.js
if (typeof global !== 'undefined') {
try {
// Define a custom PlatformNode class
class PlatformNode {
constructor() {
// Create a util object with necessary methods
this.util = {
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
};
// Initialize encoders using native constructors
this.textEncoder = new TextEncoder();
this.textDecoder = new TextDecoder();
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr) {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
}
// Define isTypedArray directly on the instance
isTypedArray(arr) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
}
}
// Assign the PlatformNode class to the global object
global.PlatformNode = PlatformNode;
// Also create an instance and assign it to global.platformNode
global.platformNode = new PlatformNode();
// Ensure global.util exists and has the necessary methods
if (!global.util) {
global.util = {};
}
// Add isFloat32Array method if it doesn't exist
if (!global.util.isFloat32Array) {
global.util.isFloat32Array = (arr) => {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
};
}
// Add isTypedArray method if it doesn't exist
if (!global.util.isTypedArray) {
global.util.isTypedArray = (arr) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
};
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error);
}
}
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });