**feat(tests): add tests for TextEncoder, TensorFlow.js, and fallback mechanisms**

- Introduced `test-fallback-function.js` and `test-fallback-simple.js` to validate `executeInThread` fallback functionality with both named and anonymous compute-intensive functions.
- Added `test-tensorflow-textencoder.js` for TensorFlow.js and TextEncoder tests in a Node.js environment.
- Created `test-tensorflow-textencoder.html` for browser-based TensorFlow.js and TextEncoder tests.
- Implemented cross-environment test support in `cli-package/src/test-tensorflow-textencoder.ts` for CLI functionality.
- Enhanced `src/utils/embedding.ts`, `textEncoding.ts`, and `brainy-wrapper.js` to include updated global `TextEncoder` and `TextDecoder` utilities for compatibility and worker improvements.
- Standardized and expanded utility methods in `PlatformNode` for broader support, including `isFloat32Array` and `isTypedArray` checks.
- Updated Node.js requirement to `>= 24.4.0` across documentation and configuration files for compatibility improvements.

This update introduces comprehensive testing for fallback mechanisms, TensorFlow.js, and TextEncoder across multiple environments, ensuring robustness and compatibility.
This commit is contained in:
David Snelling 2025-07-11 11:11:56 -07:00
parent 04a33b9ae8
commit 5f267b14ed
30 changed files with 1799 additions and 1583 deletions

View file

@ -2,13 +2,13 @@
* Unified Text Encoding Utilities for CLI
*
* This module provides a consistent way to handle text encoding/decoding across all environments
* without relying on TextEncoder/TextDecoder polyfills or patches.
* using the native TextEncoder/TextDecoder APIs.
*/
/**
* Apply the TensorFlow.js platform patch if needed
* This function patches the global object to provide a PlatformNode class
* that uses our text encoding utilities instead of relying on TextEncoder/TextDecoder
* that uses native TextEncoder/TextDecoder
*/
export function applyTensorFlowPatch(): void {
// Only apply in Node.js environment
@ -22,20 +22,32 @@ export function applyTensorFlowPatch(): void {
// Define a custom PlatformNode class
class PlatformNode {
util: any
textEncoder: any
textDecoder: any
textEncoder: TextEncoder
textDecoder: TextDecoder
constructor() {
// Create a util object with necessary methods and constructors
this.util = {
// Add isFloat32Array and isTypedArray directly to util
isFloat32Array: (arr: any) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
)
},
isTypedArray: (arr: any) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
},
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
}
// Initialize using the constructors from util
this.textEncoder = new this.util.TextEncoder()
this.textDecoder = new this.util.TextDecoder()
// Initialize using native constructors
this.textEncoder = new TextEncoder()
this.textDecoder = new TextDecoder()
}
// Define isFloat32Array directly on the instance
@ -43,8 +55,7 @@ export function applyTensorFlowPatch(): void {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
@ -59,6 +70,30 @@ export function applyTensorFlowPatch(): void {
// Also create an instance and assign it to global.platformNode (lowercase p)
;(global as any).platformNode = new PlatformNode()
// Ensure global.util exists and has the necessary methods
// This is needed because TensorFlow.js might look for these methods in global.util
if (!(global as any).util) {
;(global as any).util = {}
}
// Add isFloat32Array method if it doesn't exist
if (!(global as any).util.isFloat32Array) {
;(global as any).util.isFloat32Array = (arr: any) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
}
// Add isTypedArray method if it doesn't exist
if (!(global as any).util.isTypedArray) {
;(global as any).util.isTypedArray = (arr: any) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
}
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error)
}