feat: add structured content extraction and batch embedding optimization to highlight()
Fix highlight() hanging on structured text input by addressing 3 root causes:
1. embedBatch() now uses native WASM batch API (single forward pass instead
of N individual embed() calls via Promise.all)
2. highlight() auto-detects content type (plain text, rich-text JSON, HTML,
Markdown) and extracts meaningful text segments. Supports TipTap, Slate.js,
Lexical, Draft.js, and Quill Delta formats. New contentType hint and
contentExtractor callback for custom parsers.
3. Semantic matching phase has 10s timeout - falls back to text-only matches
instead of hanging indefinitely.
Also fixes extractTextContent() array check: uses type-based detection
(typeof data[0] === 'number') instead of length-based (data.length > 10)
so arrays of objects are properly indexed for text search.
New types: ContentType, ContentCategory, ExtractedSegment
New fields: HighlightParams.contentType, HighlightParams.contentExtractor,
Highlight.contentCategory
This commit is contained in:
parent
2b901974ed
commit
cca1cd8ce2
12 changed files with 1341 additions and 37 deletions
|
|
@ -206,6 +206,35 @@ export class EmbeddingManager {
|
|||
return vector
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch embed multiple texts using native WASM batch API
|
||||
*
|
||||
* Uses the engine's embedBatch() for a single WASM forward pass
|
||||
* instead of N individual embed() calls.
|
||||
*
|
||||
* @param texts Array of strings to embed
|
||||
* @returns Array of embedding vectors (384 dimensions each)
|
||||
*/
|
||||
async embedBatch(texts: string[]): Promise<number[][]> {
|
||||
if (texts.length === 0) return []
|
||||
|
||||
const isTestMode =
|
||||
process.env.BRAINY_UNIT_TEST === 'true' ||
|
||||
(globalThis as any).__BRAINY_UNIT_TEST__
|
||||
|
||||
if (isTestMode) {
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
throw new Error('CRITICAL: Mock embeddings in production!')
|
||||
}
|
||||
return texts.map(t => this.getMockEmbedding(t))
|
||||
}
|
||||
|
||||
await this.init()
|
||||
const results = await this.engine.embedBatch(texts)
|
||||
this.embedCount += texts.length
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embedding function for compatibility
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue