feat: add structured content extraction and batch embedding optimization to highlight()
Fix highlight() hanging on structured text input by addressing 3 root causes:
1. embedBatch() now uses native WASM batch API (single forward pass instead
of N individual embed() calls via Promise.all)
2. highlight() auto-detects content type (plain text, rich-text JSON, HTML,
Markdown) and extracts meaningful text segments. Supports TipTap, Slate.js,
Lexical, Draft.js, and Quill Delta formats. New contentType hint and
contentExtractor callback for custom parsers.
3. Semantic matching phase has 10s timeout - falls back to text-only matches
instead of hanging indefinitely.
Also fixes extractTextContent() array check: uses type-based detection
(typeof data[0] === 'number') instead of length-based (data.length > 10)
so arrays of objects are properly indexed for text search.
New types: ContentType, ContentCategory, ExtractedSegment
New fields: HighlightParams.contentType, HighlightParams.contentExtractor,
Highlight.contentCategory
This commit is contained in:
parent
2b901974ed
commit
cca1cd8ce2
12 changed files with 1341 additions and 37 deletions
|
|
@ -738,4 +738,242 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
expect(programmingMatch?.score).toBe(1.0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Structured Content Extraction (v7.9.0)', () => {
|
||||
it('should extract text from TipTap/ProseMirror JSON', async () => {
|
||||
const tiptapDoc = JSON.stringify({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'heading',
|
||||
attrs: { level: 1 },
|
||||
content: [{ type: 'text', text: 'David the Warrior' }]
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [{ type: 'text', text: 'He is a brave fighter who battles dragons' }]
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior fighter',
|
||||
text: tiptapDoc,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
// Should find "Warrior" as text match from heading
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
// Should annotate heading content
|
||||
expect(warriorMatch?.contentCategory).toBe('heading')
|
||||
})
|
||||
|
||||
it('should extract text from Slate.js JSON', async () => {
|
||||
const slateDoc = JSON.stringify([
|
||||
{
|
||||
type: 'heading',
|
||||
children: [{ text: 'Introduction' }]
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
children: [
|
||||
{ text: 'This is ' },
|
||||
{ text: 'bold text', bold: true },
|
||||
{ text: ' in a paragraph' }
|
||||
]
|
||||
}
|
||||
])
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'Introduction bold',
|
||||
text: slateDoc,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
const introMatch = highlights.find(h => h.text === 'Introduction')
|
||||
expect(introMatch).toBeDefined()
|
||||
expect(introMatch?.matchType).toBe('text')
|
||||
expect(introMatch?.contentCategory).toBe('heading')
|
||||
})
|
||||
|
||||
it('should extract text from Quill Delta JSON', async () => {
|
||||
const quilDoc = JSON.stringify({
|
||||
ops: [
|
||||
{ insert: 'Hello World\n', attributes: { header: 1 } },
|
||||
{ insert: 'This is a paragraph with warrior content\n' }
|
||||
]
|
||||
})
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior',
|
||||
text: quilDoc,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
})
|
||||
|
||||
it('should fall back to text extraction for generic JSON', async () => {
|
||||
const apiResponse = JSON.stringify({
|
||||
name: 'warrior',
|
||||
description: 'A brave fighter',
|
||||
stats: { strength: 10, agility: 8 }
|
||||
})
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior',
|
||||
text: apiResponse,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
})
|
||||
|
||||
it('should extract text from HTML with headings and code', async () => {
|
||||
const html = '<h1>Warrior Guide</h1><p>A brave fighter battles enemies.</p><code>const warrior = new Fighter()</code>'
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior fighter',
|
||||
text: html,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
// Should find "Warrior" from heading
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.contentCategory).toBe('heading')
|
||||
|
||||
// Should find "fighter" from paragraph
|
||||
const fighterMatch = highlights.find(h => h.text.toLowerCase() === 'fighter')
|
||||
expect(fighterMatch).toBeDefined()
|
||||
expect(fighterMatch?.contentCategory).toBe('prose')
|
||||
})
|
||||
|
||||
it('should extract text from Markdown with headings and code', async () => {
|
||||
const markdown = '# Warrior Guide\n\nA brave fighter battles enemies.\n\n```\nconst warrior = new Fighter()\n```'
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior fighter',
|
||||
text: markdown,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
// Should find text from heading
|
||||
const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'heading')
|
||||
|| highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'heading')
|
||||
expect(headingMatch).toBeDefined()
|
||||
})
|
||||
|
||||
it('should preserve plain text behavior (regression)', async () => {
|
||||
const plainText = 'David Smith is a brave warrior who battles dragons'
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior',
|
||||
text: plainText,
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
expect(warriorMatch?.score).toBe(1.0)
|
||||
// Plain text gets 'prose' category
|
||||
expect(warriorMatch?.contentCategory).toBe('prose')
|
||||
})
|
||||
|
||||
it('should use contentType hint to skip auto-detection', async () => {
|
||||
// Pass HTML as plain text via content type hint
|
||||
const htmlAsPlain = '<h1>test</h1>'
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'test',
|
||||
text: htmlAsPlain,
|
||||
granularity: 'word',
|
||||
threshold: 0.3,
|
||||
contentType: 'plaintext' // Force plain text treatment
|
||||
})
|
||||
|
||||
// When treated as plain text, the raw HTML tags become part of chunks
|
||||
// The content should not be parsed as HTML
|
||||
expect(highlights).toBeDefined()
|
||||
})
|
||||
|
||||
it('should use custom contentExtractor when provided', async () => {
|
||||
const customExtractor = (text: string) => {
|
||||
return [
|
||||
{ text: 'custom extracted warrior content', contentCategory: 'prose' as const },
|
||||
{ text: 'Code Section', contentCategory: 'code' as const }
|
||||
]
|
||||
}
|
||||
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior',
|
||||
text: 'ignored because custom extractor is used',
|
||||
granularity: 'word',
|
||||
threshold: 0.3,
|
||||
contentExtractor: customExtractor
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
expect(warriorMatch?.contentCategory).toBe('prose')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Timeout Protection (v7.9.0)', () => {
|
||||
it('should return text-only results if semantic phase times out', async () => {
|
||||
// We test the timeout path by ensuring the highlight method doesn't hang.
|
||||
// The mock embeddings are fast so timeout won't trigger in normal test,
|
||||
// but we verify the method completes successfully.
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior',
|
||||
text: 'David is a brave warrior who battles dragons',
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
// Should complete without hanging
|
||||
expect(highlights).toBeDefined()
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('embedBatch uses native batch API (v7.9.0)', () => {
|
||||
it('should embed multiple texts in a single call', async () => {
|
||||
const texts = ['hello world', 'foo bar', 'test content']
|
||||
const embeddings = await brain.embedBatch(texts)
|
||||
|
||||
expect(embeddings.length).toBe(3)
|
||||
// Each embedding should be 384 dimensions
|
||||
for (const emb of embeddings) {
|
||||
expect(emb.length).toBe(384)
|
||||
}
|
||||
})
|
||||
|
||||
it('should return empty array for empty input', async () => {
|
||||
const embeddings = await brain.embedBatch([])
|
||||
expect(embeddings).toEqual([])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
321
tests/unit/utils/contentExtractor.test.ts
Normal file
321
tests/unit/utils/contentExtractor.test.ts
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
/**
|
||||
* Content Extractor Tests (v7.9.0)
|
||||
*
|
||||
* Tests for content type detection and text extraction from
|
||||
* structured formats (rich-text JSON, HTML, Markdown).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
detectContentType,
|
||||
extractForHighlighting
|
||||
} from '../../../src/utils/contentExtractor'
|
||||
|
||||
describe('Content Extractor (v7.9.0)', () => {
|
||||
describe('detectContentType()', () => {
|
||||
it('should detect plaintext', () => {
|
||||
expect(detectContentType('Hello world')).toBe('plaintext')
|
||||
expect(detectContentType('Just some regular text.')).toBe('plaintext')
|
||||
})
|
||||
|
||||
it('should detect rich-text JSON (object)', () => {
|
||||
const tiptap = JSON.stringify({ type: 'doc', content: [] })
|
||||
expect(detectContentType(tiptap)).toBe('richtext-json')
|
||||
})
|
||||
|
||||
it('should detect rich-text JSON (array)', () => {
|
||||
const slate = JSON.stringify([{ type: 'paragraph', children: [{ text: 'hi' }] }])
|
||||
expect(detectContentType(slate)).toBe('richtext-json')
|
||||
})
|
||||
|
||||
it('should detect HTML', () => {
|
||||
expect(detectContentType('<h1>Title</h1>')).toBe('html')
|
||||
expect(detectContentType('<div>content</div>')).toBe('html')
|
||||
expect(detectContentType('<!DOCTYPE html><html>')).toBe('html')
|
||||
})
|
||||
|
||||
it('should detect Markdown', () => {
|
||||
expect(detectContentType('# Heading\n\nSome text')).toBe('markdown')
|
||||
expect(detectContentType('```\ncode\n```')).toBe('markdown')
|
||||
})
|
||||
|
||||
it('should handle invalid JSON as plaintext', () => {
|
||||
expect(detectContentType('{ broken json')).toBe('plaintext')
|
||||
expect(detectContentType('[not valid')).toBe('plaintext')
|
||||
})
|
||||
|
||||
it('should handle whitespace-prefixed JSON', () => {
|
||||
const json = ' { "type": "doc" }'
|
||||
expect(detectContentType(json)).toBe('richtext-json')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — TipTap/ProseMirror', () => {
|
||||
it('should extract text nodes with content categories', () => {
|
||||
const doc = JSON.stringify({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'heading',
|
||||
attrs: { level: 1 },
|
||||
content: [{ type: 'text', text: 'My Heading' }]
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [{ type: 'text', text: 'Regular paragraph text' }]
|
||||
},
|
||||
{
|
||||
type: 'codeBlock',
|
||||
content: [{ type: 'text', text: 'const x = 1' }]
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(3)
|
||||
|
||||
expect(segments[0].text).toBe('My Heading')
|
||||
expect(segments[0].contentCategory).toBe('heading')
|
||||
|
||||
expect(segments[1].text).toBe('Regular paragraph text')
|
||||
expect(segments[1].contentCategory).toBe('prose')
|
||||
|
||||
expect(segments[2].text).toBe('const x = 1')
|
||||
expect(segments[2].contentCategory).toBe('code')
|
||||
})
|
||||
|
||||
it('should handle nested content with marks', () => {
|
||||
const doc = JSON.stringify({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{ type: 'text', text: 'Hello ' },
|
||||
{ type: 'text', text: 'bold world', marks: [{ type: 'bold' }] }
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(2)
|
||||
expect(segments[0].text).toBe('Hello ')
|
||||
expect(segments[1].text).toBe('bold world')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Slate.js', () => {
|
||||
it('should extract text from Slate children nodes', () => {
|
||||
const doc = JSON.stringify([
|
||||
{
|
||||
type: 'heading',
|
||||
children: [{ text: 'Slate Heading' }]
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
children: [
|
||||
{ text: 'First part ' },
|
||||
{ text: 'second part', bold: true }
|
||||
]
|
||||
}
|
||||
])
|
||||
|
||||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(3)
|
||||
expect(segments[0].text).toBe('Slate Heading')
|
||||
expect(segments[0].contentCategory).toBe('heading')
|
||||
expect(segments[1].text).toBe('First part ')
|
||||
expect(segments[1].contentCategory).toBe('prose')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Lexical', () => {
|
||||
it('should extract text from Lexical root structure', () => {
|
||||
const doc = JSON.stringify({
|
||||
root: {
|
||||
children: [
|
||||
{
|
||||
type: 'heading',
|
||||
children: [{ type: 'text', text: 'Lexical Heading' }]
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
children: [{ type: 'text', text: 'Lexical body text' }]
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(2)
|
||||
expect(segments[0].text).toBe('Lexical Heading')
|
||||
expect(segments[0].contentCategory).toBe('heading')
|
||||
expect(segments[1].text).toBe('Lexical body text')
|
||||
expect(segments[1].contentCategory).toBe('prose')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Draft.js', () => {
|
||||
it('should extract text from Draft.js blocks', () => {
|
||||
const doc = JSON.stringify({
|
||||
blocks: [
|
||||
{ type: 'header-one', text: 'Draft Heading' },
|
||||
{ type: 'unstyled', text: 'Draft body text' }
|
||||
]
|
||||
})
|
||||
|
||||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBeGreaterThanOrEqual(2)
|
||||
// Draft.js blocks have text property directly with type
|
||||
const headingSegment = segments.find(s => s.text === 'Draft Heading')
|
||||
const bodySegment = segments.find(s => s.text === 'Draft body text')
|
||||
expect(headingSegment).toBeDefined()
|
||||
expect(bodySegment).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Quill Delta', () => {
|
||||
it('should extract text from Quill Delta ops', () => {
|
||||
const doc = JSON.stringify({
|
||||
ops: [
|
||||
{ insert: 'Hello World\n' },
|
||||
{ insert: 'Second line\n' }
|
||||
]
|
||||
})
|
||||
|
||||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(2)
|
||||
expect(segments[0].text).toContain('Hello World')
|
||||
expect(segments[1].text).toContain('Second line')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Generic JSON fallback', () => {
|
||||
it('should extract all string values from unrecognized JSON', () => {
|
||||
const generic = JSON.stringify({
|
||||
name: 'David',
|
||||
description: 'A brave warrior',
|
||||
stats: { strength: 10 }
|
||||
})
|
||||
|
||||
const segments = extractForHighlighting(generic)
|
||||
expect(segments.length).toBeGreaterThan(0)
|
||||
const allText = segments.map(s => s.text).join(' ')
|
||||
expect(allText).toContain('David')
|
||||
expect(allText).toContain('brave warrior')
|
||||
})
|
||||
|
||||
it('should return empty for JSON with no text', () => {
|
||||
const noText = JSON.stringify({ count: 42, active: true })
|
||||
const segments = extractForHighlighting(noText)
|
||||
// No string values to extract
|
||||
expect(segments).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — HTML', () => {
|
||||
it('should extract headings, paragraphs, and code', () => {
|
||||
const html = '<h1>Title</h1><p>Body text here.</p><pre><code>let x = 1</code></pre>'
|
||||
|
||||
const segments = extractForHighlighting(html)
|
||||
expect(segments.length).toBeGreaterThanOrEqual(3)
|
||||
|
||||
const headingSegment = segments.find(s => s.text === 'Title')
|
||||
expect(headingSegment).toBeDefined()
|
||||
expect(headingSegment?.contentCategory).toBe('heading')
|
||||
|
||||
const bodySegment = segments.find(s => s.text === 'Body text here.')
|
||||
expect(bodySegment).toBeDefined()
|
||||
expect(bodySegment?.contentCategory).toBe('prose')
|
||||
|
||||
const codeSegment = segments.find(s => s.text === 'let x = 1')
|
||||
expect(codeSegment).toBeDefined()
|
||||
expect(codeSegment?.contentCategory).toBe('code')
|
||||
})
|
||||
|
||||
it('should skip script and style content', () => {
|
||||
const html = '<p>Visible</p><script>alert("hidden")</script><style>.foo{}</style><p>Also visible</p>'
|
||||
|
||||
const segments = extractForHighlighting(html)
|
||||
const allText = segments.map(s => s.text).join(' ')
|
||||
expect(allText).toContain('Visible')
|
||||
expect(allText).toContain('Also visible')
|
||||
expect(allText).not.toContain('alert')
|
||||
expect(allText).not.toContain('.foo')
|
||||
})
|
||||
|
||||
it('should decode common HTML entities', () => {
|
||||
const html = '<p>Tom & Jerry <3</p>'
|
||||
const segments = extractForHighlighting(html)
|
||||
expect(segments.length).toBeGreaterThan(0)
|
||||
expect(segments[0].text).toContain('Tom & Jerry <3')
|
||||
})
|
||||
|
||||
it('should handle nested heading tags', () => {
|
||||
const html = '<h2><span>Nested Heading</span></h2>'
|
||||
const segments = extractForHighlighting(html)
|
||||
const heading = segments.find(s => s.text === 'Nested Heading')
|
||||
expect(heading).toBeDefined()
|
||||
expect(heading?.contentCategory).toBe('heading')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Markdown', () => {
|
||||
it('should extract headings', () => {
|
||||
const md = '# Main Title\n\nSome body text\n\n## Subtitle'
|
||||
|
||||
const segments = extractForHighlighting(md)
|
||||
const heading1 = segments.find(s => s.text === 'Main Title')
|
||||
const heading2 = segments.find(s => s.text === 'Subtitle')
|
||||
const body = segments.find(s => s.text === 'Some body text')
|
||||
|
||||
expect(heading1).toBeDefined()
|
||||
expect(heading1?.contentCategory).toBe('heading')
|
||||
expect(heading2).toBeDefined()
|
||||
expect(heading2?.contentCategory).toBe('heading')
|
||||
expect(body).toBeDefined()
|
||||
expect(body?.contentCategory).toBe('prose')
|
||||
})
|
||||
|
||||
it('should extract fenced code blocks', () => {
|
||||
const md = '# Title\n\n```typescript\nconst x = 1\nconst y = 2\n```\n\nMore text'
|
||||
|
||||
const segments = extractForHighlighting(md)
|
||||
const code = segments.find(s => s.contentCategory === 'code')
|
||||
expect(code).toBeDefined()
|
||||
expect(code?.text).toContain('const x = 1')
|
||||
})
|
||||
|
||||
it('should extract indented code blocks', () => {
|
||||
// Indented code alone doesn't trigger markdown detection, so pass explicit type
|
||||
const md = 'Normal text\n\n code line 1\n code line 2\n\nMore text'
|
||||
|
||||
const segments = extractForHighlighting(md, 'markdown')
|
||||
const code = segments.find(s => s.contentCategory === 'code')
|
||||
expect(code).toBeDefined()
|
||||
expect(code?.text).toContain('code line 1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Plaintext', () => {
|
||||
it('should return single prose segment for plain text', () => {
|
||||
const text = 'Just some plain text content'
|
||||
const segments = extractForHighlighting(text)
|
||||
|
||||
expect(segments.length).toBe(1)
|
||||
expect(segments[0].text).toBe(text)
|
||||
expect(segments[0].contentCategory).toBe('prose')
|
||||
})
|
||||
|
||||
it('should respect contentType override', () => {
|
||||
// Even if content looks like HTML, plaintext hint should skip detection
|
||||
const html = '<h1>Title</h1>'
|
||||
const segments = extractForHighlighting(html, 'plaintext')
|
||||
|
||||
expect(segments.length).toBe(1)
|
||||
expect(segments[0].text).toBe(html) // Raw HTML, not extracted
|
||||
expect(segments[0].contentCategory).toBe('prose')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -215,4 +215,70 @@ describe('MetadataIndexManager Text Indexing (v7.7.0)', () => {
|
|||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractTextContent() array handling (v7.9.0 fix)', () => {
|
||||
let manager: MetadataIndexManager
|
||||
|
||||
beforeEach(() => {
|
||||
const mockStorage = {
|
||||
getMetadata: async () => null,
|
||||
saveMetadata: async () => {},
|
||||
getNoun: async () => null,
|
||||
getNouns: async () => ({ items: [], total: 0, hasMore: false }),
|
||||
getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
|
||||
} as any
|
||||
|
||||
manager = new MetadataIndexManager(mockStorage, {})
|
||||
})
|
||||
|
||||
it('should skip numeric arrays (vectors/embeddings)', () => {
|
||||
const result = manager.extractTextContent([0.1, 0.2, 0.3, 0.4, 0.5])
|
||||
expect(result).toBe('')
|
||||
})
|
||||
|
||||
it('should extract text from string arrays', () => {
|
||||
const result = manager.extractTextContent(['hello', 'world', 'test'])
|
||||
expect(result).toContain('hello')
|
||||
expect(result).toContain('world')
|
||||
expect(result).toContain('test')
|
||||
})
|
||||
|
||||
it('should extract text from arrays of objects', () => {
|
||||
const result = manager.extractTextContent([
|
||||
{ name: 'Alice', role: 'engineer' },
|
||||
{ name: 'Bob', role: 'designer' },
|
||||
{ name: 'Carol', role: 'manager' },
|
||||
{ name: 'Dave', role: 'analyst' },
|
||||
{ name: 'Eve', role: 'scientist' },
|
||||
{ name: 'Frank', role: 'developer' },
|
||||
{ name: 'Grace', role: 'architect' },
|
||||
{ name: 'Heidi', role: 'lead' },
|
||||
{ name: 'Ivan', role: 'intern' },
|
||||
{ name: 'Judy', role: 'director' },
|
||||
{ name: 'Karl', role: 'founder' },
|
||||
])
|
||||
// Should NOT skip arrays of objects even with >10 elements
|
||||
expect(result).toContain('Alice')
|
||||
expect(result).toContain('Karl')
|
||||
expect(result).toContain('engineer')
|
||||
expect(result).toContain('founder')
|
||||
})
|
||||
|
||||
it('should handle large numeric arrays (vectors)', () => {
|
||||
const vector = Array.from({ length: 384 }, (_, i) => Math.sin(i))
|
||||
const result = manager.extractTextContent(vector)
|
||||
expect(result).toBe('')
|
||||
})
|
||||
|
||||
it('should handle empty arrays', () => {
|
||||
const result = manager.extractTextContent([])
|
||||
expect(result).toBe('')
|
||||
})
|
||||
|
||||
it('should handle mixed arrays starting with string', () => {
|
||||
const result = manager.extractTextContent(['text', 42, true])
|
||||
expect(result).toContain('text')
|
||||
expect(result).toContain('42')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue