**feat(tests): add tests for TextEncoder, TensorFlow.js, and fallback mechanisms**

- Introduced `test-fallback-function.js` and `test-fallback-simple.js` to validate `executeInThread` fallback functionality with both named and anonymous compute-intensive functions.
- Added `test-tensorflow-textencoder.js` for TensorFlow.js and TextEncoder tests in a Node.js environment.
- Created `test-tensorflow-textencoder.html` for browser-based TensorFlow.js and TextEncoder tests.
- Implemented cross-environment test support in `cli-package/src/test-tensorflow-textencoder.ts` for CLI functionality.
- Enhanced `src/utils/embedding.ts`, `textEncoding.ts`, and `brainy-wrapper.js` to include updated global `TextEncoder` and `TextDecoder` utilities for compatibility and worker improvements.
- Standardized and expanded utility methods in `PlatformNode` for broader support, including `isFloat32Array` and `isTypedArray` checks.
- Updated Node.js requirement to `>= 24.4.0` across documentation and configuration files for compatibility improvements.

This update introduces comprehensive testing for fallback mechanisms, TensorFlow.js, and TextEncoder across multiple environments, ensuring robustness and compatibility.
This commit is contained in:
David Snelling 2025-07-11 11:11:56 -07:00
parent 00039f836f
commit f0db5b471f
30 changed files with 1799 additions and 1583 deletions

View file

@ -34,7 +34,88 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
return
}
// No compatibility patches needed - TensorFlow.js now works correctly with Node.js 24+
// Add polyfill for isFloat32Array in Node.js 24.4.0
// This fixes the "Cannot read properties of undefined (reading 'isFloat32Array')" error
if (typeof global !== 'undefined') {
try {
// Define a custom PlatformNode class
class PlatformNode {
util: any
textEncoder: TextEncoder
textDecoder: TextDecoder
constructor() {
// Create a util object with necessary methods
this.util = {
// Add isFloat32Array and isTypedArray directly to util
isFloat32Array: (arr: any) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
)
},
isTypedArray: (arr: any) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
},
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
}
// Initialize encoders using native constructors
this.textEncoder = new TextEncoder()
this.textDecoder = new TextDecoder()
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr: any) {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
// Define isTypedArray directly on the instance
isTypedArray(arr: any) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
}
}
// Assign the PlatformNode class to the global object
;(global as any).PlatformNode = PlatformNode
// Also create an instance and assign it to global.platformNode
;(global as any).platformNode = new PlatformNode()
} catch (error) {
console.warn('Failed to define global PlatformNode class:', error)
}
// Ensure the util object exists
if (!global.util) {
global.util = {}
}
// Add isFloat32Array method if it doesn't exist
if (!global.util.isFloat32Array) {
global.util.isFloat32Array = (obj: any) => {
return !!(
obj instanceof Float32Array ||
(obj &&
Object.prototype.toString.call(obj) === '[object Float32Array]')
)
}
}
// Add isTypedArray method if it doesn't exist
if (!global.util.isTypedArray) {
global.util.isTypedArray = (obj: any) => {
return !!(ArrayBuffer.isView(obj) && !(obj instanceof DataView))
}
}
}
}
/**