**feat(patch): enhance TextEncoder/TextDecoder compatibility for TensorFlow.js**

- Refactored `applyTensorFlowPatch` in `textEncoding.ts` to introduce a unified `Platform` class supporting both Node.js and browser environments with native `TextEncoder`/`TextDecoder`.
- Modified global object handling to target `PlatformNode` in Node.js and `PlatformBrowser` in browser environments.
- Added `getTextEncoder` and `getTextDecoder` utility functions to simplify text encoding/decoding across platforms.
- Introduced `setup.ts` to apply environment patches before other modules load.
- Updated CLI entry points (`cli.ts`, `rollup.config.js`) to ensure patching precedes TensorFlow.js usage.
- Enhanced test coverage for TextEncoder compatibility with `test-fix.js`.
- Adjusted imports in `test-tensorflow-textencoder.ts` to align with the updated `./utils/textEncoding.js`.
- Streamlined constructor definitions in `Platform` for improved maintainability.

This update ensures robust compatibility for TensorFlow.js by patching and standardizing text encoding/decoding functionality across environments.
This commit is contained in:
David Snelling 2025-07-11 12:15:11 -07:00
parent 5f267b14ed
commit e3d7398022
7 changed files with 142 additions and 84 deletions

View file

@ -1,101 +1,94 @@
/**
* Unified Text Encoding Utilities for CLI
* Unified Text Encoding Utilities
*
* This module provides a consistent way to handle text encoding/decoding across all environments
* using the native TextEncoder/TextDecoder APIs.
*/
/**
* Get a text encoder that works in the current environment
* @returns A TextEncoder instance
*/
export function getTextEncoder(): TextEncoder {
return new TextEncoder()
}
/**
* Get a text decoder that works in the current environment
* @returns A TextDecoder instance
*/
export function getTextDecoder(): TextDecoder {
return new TextDecoder()
}
/**
* Apply the TensorFlow.js platform patch if needed
* This function patches the global object to provide a PlatformNode class
* that uses native TextEncoder/TextDecoder
*/
export function applyTensorFlowPatch(): void {
// Only apply in Node.js environment
if (
typeof global !== 'undefined' &&
typeof process !== 'undefined' &&
process.versions &&
process.versions.node
) {
try {
// Define a custom PlatformNode class
class PlatformNode {
util: any
textEncoder: TextEncoder
textDecoder: TextDecoder
try {
// Define a custom Platform class that works in both Node.js and browser environments
class Platform {
util: any
textEncoder: TextEncoder
textDecoder: TextDecoder
constructor() {
// Create a util object with necessary methods and constructors
this.util = {
// Add isFloat32Array and isTypedArray directly to util
isFloat32Array: (arr: any) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) ===
'[object Float32Array]')
)
},
isTypedArray: (arr: any) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
},
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
}
// Initialize using native constructors
this.textEncoder = new TextEncoder()
this.textDecoder = new TextDecoder()
constructor() {
// Create a util object with necessary methods and constructors
this.util = {
// Use native TextEncoder and TextDecoder
TextEncoder: globalThis.TextEncoder || TextEncoder,
TextDecoder: globalThis.TextDecoder || TextDecoder
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr: any) {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
// Define isTypedArray directly on the instance
isTypedArray(arr: any) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
}
// Initialize using native constructors directly
this.textEncoder = new (globalThis.TextEncoder || TextEncoder)()
this.textDecoder = new (globalThis.TextDecoder || TextDecoder)()
}
// Assign the PlatformNode class to the global object
;(global as any).PlatformNode = PlatformNode
// Also create an instance and assign it to global.platformNode (lowercase p)
;(global as any).platformNode = new PlatformNode()
// Ensure global.util exists and has the necessary methods
// This is needed because TensorFlow.js might look for these methods in global.util
if (!(global as any).util) {
;(global as any).util = {}
// Define isFloat32Array directly on the instance
isFloat32Array(arr: any) {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
// Add isFloat32Array method if it doesn't exist
if (!(global as any).util.isFloat32Array) {
;(global as any).util.isFloat32Array = (arr: any) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
)
}
// Define isTypedArray directly on the instance
isTypedArray(arr: any) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
}
// Add isTypedArray method if it doesn't exist
if (!(global as any).util.isTypedArray) {
;(global as any).util.isTypedArray = (arr: any) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
}
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error)
}
// Get the global object in a way that works in both Node.js and browser
const globalObj =
typeof global !== 'undefined'
? global
: typeof window !== 'undefined'
? window
: typeof self !== 'undefined'
? self
: {}
// Only apply in Node.js environment
if (
typeof process !== 'undefined' &&
process.versions &&
process.versions.node
) {
// Assign the Platform class to the global object as PlatformNode for Node.js
;(globalObj as any).PlatformNode = Platform
// Also create an instance and assign it to global.platformNode (lowercase p)
;(globalObj as any).platformNode = new Platform()
} else if (typeof window !== 'undefined' || typeof self !== 'undefined') {
// In browser environments, we might need to provide similar functionality
// but we'll use a different name to avoid conflicts
;(globalObj as any).PlatformBrowser = Platform
;(globalObj as any).platformBrowser = new Platform()
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error)
}
}