feat: add optional Q8 quantized model support

- Add Q8 quantized models (75% smaller than FP32)
- Enhance download scripts with model variant selection
- Add smart model loading with availability detection
- Implement runtime warnings for Q8 compatibility
- Update documentation with Q8 usage examples
- Maintain 100% backward compatibility (FP32 default)

BREAKING CHANGE: None - FP32 remains default

🧠 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-08-29 11:09:40 -07:00
parent 223311f4e7
commit 32df3ee6ae
8 changed files with 259 additions and 40 deletions

View file

@ -128,12 +128,25 @@ export class TransformerEmbedding implements EmbeddingModel {
verbose: this.verbose,
cacheDir: options.cacheDir || './models',
localFilesOnly: localFilesOnly,
dtype: options.dtype || 'fp32', // Use fp32 by default as quantized models aren't available on CDN
dtype: options.dtype || 'fp32', // CRITICAL: fp32 default for backward compatibility
device: options.device || 'auto'
}
// ULTRA-CAREFUL: Runtime warnings for q8 usage
if (this.options.dtype === 'q8') {
const confirmed = process.env.BRAINY_Q8_CONFIRMED === 'true'
if (!confirmed && this.verbose) {
console.warn('🚨 Q8 MODEL WARNING:')
console.warn(' • Q8 creates different embeddings than fp32')
console.warn(' • Q8 is incompatible with existing fp32 data')
console.warn(' • Only use q8 for new projects or when explicitly migrating')
console.warn(' • Set BRAINY_Q8_CONFIRMED=true to silence this warning')
console.warn(' • Q8 model is 75% smaller but may have slightly reduced accuracy')
}
}
if (this.verbose) {
this.logger('log', `Embedding config: localFilesOnly=${localFilesOnly}, model=${this.options.model}, cacheDir=${this.options.cacheDir}`)
this.logger('log', `Embedding config: dtype=${this.options.dtype}, localFilesOnly=${localFilesOnly}, model=${this.options.model}`)
}
// Configure transformers.js environment
@ -258,11 +271,23 @@ export class TransformerEmbedding implements EmbeddingModel {
const startTime = Date.now()
// Check model availability and select appropriate variant
const available = modelManager.getAvailableModels(this.options.model)
const actualType = modelManager.getBestAvailableModel(this.options.dtype as 'fp32' | 'q8', this.options.model)
if (!actualType) {
throw new Error(`No model variants available for ${this.options.model}. Run 'npm run download-models' to download models.`)
}
if (actualType !== this.options.dtype) {
this.logger('log', `Using ${actualType} model (${this.options.dtype} not available)`)
}
// Load the feature extraction pipeline with memory optimizations
const pipelineOptions: any = {
cache_dir: cacheDir,
local_files_only: isBrowser() ? false : this.options.localFilesOnly,
dtype: this.options.dtype || 'fp32', // Use fp32 model as quantized models aren't available on CDN
dtype: actualType, // Use the actual available model type
// CRITICAL: ONNX memory optimizations
session_options: {
enableCpuMemArena: false, // Disable pre-allocated memory arena