fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -26,7 +26,7 @@ use js_sys::{Array, Float32Array};
|
|||
use tokenizers::Tokenizer;
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
// v7.2.0: Model weights are NO LONGER embedded in WASM
|
||||
// Model weights are NO LONGER embedded in WASM
|
||||
//
|
||||
// Previous design: 90MB WASM with model weights embedded via include_bytes!()
|
||||
// Problem: WASM parsing/compilation took 139+ seconds on throttled CPU (Cloud Run)
|
||||
|
|
@ -78,7 +78,7 @@ impl EmbeddingEngine {
|
|||
|
||||
/// Load the model and tokenizer from bytes
|
||||
///
|
||||
/// v7.2.0: This is now the ONLY way to initialize the engine.
|
||||
/// This is now the ONLY way to initialize the engine.
|
||||
/// Model weights are no longer embedded in WASM for faster initialization.
|
||||
///
|
||||
/// # Arguments
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* Pure Rust/WASM implementation for sentence embeddings.
|
||||
* Works with Bun, Node.js, Bun --compile, and browsers.
|
||||
*
|
||||
* v7.2.0 Architecture (20x faster initialization):
|
||||
* Architecture (20x faster initialization):
|
||||
* - WASM file: ~2.4MB (inference code only)
|
||||
* - Model files: ~88MB (loaded separately as raw bytes)
|
||||
* - Init time: ~5-7 seconds (vs 139 seconds with embedded model)
|
||||
|
|
@ -34,7 +34,7 @@ interface CandleWasmModule {
|
|||
}
|
||||
|
||||
interface CandleEngineInstance {
|
||||
// v7.2.0: load() is the only initialization method (no more embedded model)
|
||||
// load() is the only initialization method (no more embedded model)
|
||||
load(modelBytes: Uint8Array, tokenizerBytes: Uint8Array, configBytes: Uint8Array): void
|
||||
is_ready(): boolean
|
||||
embed(text: string): Float32Array
|
||||
|
|
@ -54,7 +54,7 @@ let globalInitPromise: Promise<void> | null = null
|
|||
* Uses the Candle ML framework (Rust/WASM) for inference.
|
||||
* Supports all-MiniLM-L6-v2 with 384-dimensional embeddings.
|
||||
*
|
||||
* v7.2.0: Model weights are loaded separately from WASM for 20x faster init.
|
||||
* Model weights are loaded separately from WASM for 20x faster init.
|
||||
* For bun --compile deployments, both WASM and model files are automatically
|
||||
* embedded in the binary - single file deployment still works.
|
||||
*/
|
||||
|
|
@ -104,7 +104,7 @@ export class CandleEmbeddingEngine {
|
|||
/**
|
||||
* Perform actual initialization
|
||||
*
|
||||
* v7.2.0: WASM and model files are loaded separately for 20x faster init.
|
||||
* WASM and model files are loaded separately for 20x faster init.
|
||||
* - WASM (~2.4MB): Compiles in ~3-5 seconds
|
||||
* - Model (~88MB): Loads as raw bytes in ~1-2 seconds
|
||||
* - Total: ~5-7 seconds (vs 139 seconds with embedded model)
|
||||
|
|
@ -153,7 +153,7 @@ export class CandleEmbeddingEngine {
|
|||
/**
|
||||
* Load the WASM module
|
||||
*
|
||||
* v7.2.0: WASM is now only ~2.4MB (inference code only, no model weights).
|
||||
* WASM is now only ~2.4MB (inference code only, no model weights).
|
||||
* Uses wasmLoader.ts for cross-environment compatibility.
|
||||
*/
|
||||
private async loadWasmModule(): Promise<CandleWasmModule> {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Universal Model Loader for Candle Embeddings
|
||||
*
|
||||
* v7.2.0: Model weights are now loaded separately from WASM for faster initialization.
|
||||
* Model weights are now loaded separately from WASM for faster initialization.
|
||||
* This reduces WASM compilation time from 139 seconds to ~3-5 seconds.
|
||||
*
|
||||
* Loads model files from appropriate source based on environment:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue