feat: replace transformers.js with direct ONNX WASM for Bun compatibility
- Remove @huggingface/transformers dependency (539MB native binaries) - Add direct ONNX Runtime Web embedding engine - Bundle all-MiniLM-L6-v2-q8 model (24MB, no runtime downloads) - Works with Node.js, Bun, and bun build --compile - Air-gap compatible: fully self-contained, no internet required New WASM embedding components: - WASMEmbeddingEngine: Main integration class - WordPieceTokenizer: Pure TypeScript tokenizer - EmbeddingPostProcessor: Mean pooling + L2 normalization - ONNXInferenceEngine: Direct ONNX Runtime Web wrapper - AssetLoader: Model file loading Tests added: - 11 WASM embedding integration tests - 8 Bun compatibility tests New npm scripts: - test:wasm - Run WASM embedding tests - test:bun - Run tests with Bun - test:bun:compile - Build and run compiled binary
This commit is contained in:
parent
c1deb7a623
commit
1f59aa2013
21 changed files with 34431 additions and 3459 deletions
51
src/setup.ts
51
src/setup.ts
|
|
@ -1,46 +1,31 @@
|
|||
/**
|
||||
* CRITICAL: This file is imported for its side effects to patch the environment
|
||||
* for Node.js compatibility before any other library code runs.
|
||||
* Brainy Setup - Minimal Polyfills
|
||||
*
|
||||
* It ensures that by the time Transformers.js/ONNX Runtime is imported by any other
|
||||
* module, the necessary compatibility fixes for the current Node.js
|
||||
* environment are already in place.
|
||||
* ARCHITECTURE (v7.0.0):
|
||||
* Brainy uses direct ONNX WASM for embeddings.
|
||||
* No transformers.js dependency, no hacks required.
|
||||
*
|
||||
* This file MUST be imported as the first import in unified.ts to prevent
|
||||
* race conditions with library initialization. Failure to do so may
|
||||
* result in errors like "TextEncoder is not a constructor" when the package
|
||||
* is used in Node.js environments.
|
||||
* This file provides minimal polyfills for cross-environment compatibility:
|
||||
* - TextEncoder/TextDecoder for older environments
|
||||
*
|
||||
* The package.json file marks this file as having side effects to prevent
|
||||
* tree-shaking by bundlers, ensuring the patch is always applied.
|
||||
* BENEFITS:
|
||||
* - Clean codebase with no workarounds
|
||||
* - Works everywhere: Node.js, Bun, Bun --compile, browsers, Deno
|
||||
* - No platform-specific binaries
|
||||
* - Model bundled in package (no runtime downloads)
|
||||
*/
|
||||
|
||||
// Get the appropriate global object for the current environment
|
||||
const globalObj = (() => {
|
||||
if (typeof globalThis !== 'undefined') return globalThis
|
||||
if (typeof global !== 'undefined') return global
|
||||
if (typeof self !== 'undefined') return self
|
||||
return null // No global object available
|
||||
})()
|
||||
// ============================================================================
|
||||
// TextEncoder/TextDecoder Polyfills
|
||||
// ============================================================================
|
||||
const globalObj = globalThis ?? global ?? self
|
||||
|
||||
// Define TextEncoder and TextDecoder globally to make sure they're available
|
||||
// Now works across all environments: Node.js, serverless, and other server environments
|
||||
if (globalObj) {
|
||||
if (!globalObj.TextEncoder) {
|
||||
globalObj.TextEncoder = TextEncoder
|
||||
}
|
||||
if (!globalObj.TextDecoder) {
|
||||
globalObj.TextDecoder = TextDecoder
|
||||
}
|
||||
|
||||
// Create special global constructors for library compatibility
|
||||
(globalObj as any).__TextEncoder__ = TextEncoder
|
||||
if (!globalObj.TextEncoder) globalObj.TextEncoder = TextEncoder
|
||||
if (!globalObj.TextDecoder) globalObj.TextDecoder = TextDecoder
|
||||
;(globalObj as any).__TextEncoder__ = TextEncoder
|
||||
;(globalObj as any).__TextDecoder__ = TextDecoder
|
||||
}
|
||||
|
||||
// Also import normally for ES modules environments
|
||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
||||
|
||||
// Apply the TextEncoder/TextDecoder compatibility patch
|
||||
applyTensorFlowPatch()
|
||||
console.log('Applied TextEncoder/TextDecoder patch via ES modules in setup.ts')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue