feat: expand ContentCategory to universal 6-category set for highlight()
Replace document-centric categories (prose/heading/code/label) with a universal set that works across documents, code, and UI: - title: headings, identifiers, labels, JSON keys - annotation: comments, docstrings, captions, alt text - content: paragraphs, list items, flowing text - value: string literals, numbers, form values - code: unparsed code blocks - structural: keywords, operators, punctuation Built-in extractors now produce title/content/code. All 6 categories are available for custom parsers (e.g. tree-sitter). Also adds inline code detection in Markdown: backtick spans within prose lines are split into separate code/content segments.
This commit is contained in:
parent
6c27f9f7fd
commit
ff80b87a0a
6 changed files with 699 additions and 651 deletions
|
|
@ -769,7 +769,7 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
// Should annotate heading content
|
||||
expect(warriorMatch?.contentCategory).toBe('heading')
|
||||
expect(warriorMatch?.contentCategory).toBe('title')
|
||||
})
|
||||
|
||||
it('should extract text from Slate.js JSON', async () => {
|
||||
|
|
@ -799,7 +799,7 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
const introMatch = highlights.find(h => h.text === 'Introduction')
|
||||
expect(introMatch).toBeDefined()
|
||||
expect(introMatch?.matchType).toBe('text')
|
||||
expect(introMatch?.contentCategory).toBe('heading')
|
||||
expect(introMatch?.contentCategory).toBe('title')
|
||||
})
|
||||
|
||||
it('should extract text from Quill Delta JSON', async () => {
|
||||
|
|
@ -856,12 +856,12 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
// Should find "Warrior" from heading
|
||||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.contentCategory).toBe('heading')
|
||||
expect(warriorMatch?.contentCategory).toBe('title')
|
||||
|
||||
// Should find "fighter" from paragraph
|
||||
const fighterMatch = highlights.find(h => h.text.toLowerCase() === 'fighter')
|
||||
expect(fighterMatch).toBeDefined()
|
||||
expect(fighterMatch?.contentCategory).toBe('prose')
|
||||
expect(fighterMatch?.contentCategory).toBe('content')
|
||||
})
|
||||
|
||||
it('should extract text from Markdown with headings and code', async () => {
|
||||
|
|
@ -876,8 +876,8 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
// Should find text from heading
|
||||
const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'heading')
|
||||
|| highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'heading')
|
||||
const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'title')
|
||||
|| highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'title')
|
||||
expect(headingMatch).toBeDefined()
|
||||
})
|
||||
|
||||
|
|
@ -896,8 +896,8 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
expect(warriorMatch?.score).toBe(1.0)
|
||||
// Plain text gets 'prose' category
|
||||
expect(warriorMatch?.contentCategory).toBe('prose')
|
||||
// Plain text gets 'content' category
|
||||
expect(warriorMatch?.contentCategory).toBe('content')
|
||||
})
|
||||
|
||||
it('should use contentType hint to skip auto-detection', async () => {
|
||||
|
|
@ -920,7 +920,7 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
it('should use custom contentExtractor when provided', async () => {
|
||||
const customExtractor = (text: string) => {
|
||||
return [
|
||||
{ text: 'custom extracted warrior content', contentCategory: 'prose' as const },
|
||||
{ text: 'custom extracted warrior content', contentCategory: 'content' as const },
|
||||
{ text: 'Code Section', contentCategory: 'code' as const }
|
||||
]
|
||||
}
|
||||
|
|
@ -937,8 +937,9 @@ describe('Hybrid Search (v7.7.0)', () => {
|
|||
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
|
||||
expect(warriorMatch).toBeDefined()
|
||||
expect(warriorMatch?.matchType).toBe('text')
|
||||
expect(warriorMatch?.contentCategory).toBe('prose')
|
||||
expect(warriorMatch?.contentCategory).toBe('content')
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
describe('Timeout Protection (v7.9.0)', () => {
|
||||
|
|
|
|||
|
|
@ -75,10 +75,10 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
expect(segments.length).toBe(3)
|
||||
|
||||
expect(segments[0].text).toBe('My Heading')
|
||||
expect(segments[0].contentCategory).toBe('heading')
|
||||
expect(segments[0].contentCategory).toBe('title')
|
||||
|
||||
expect(segments[1].text).toBe('Regular paragraph text')
|
||||
expect(segments[1].contentCategory).toBe('prose')
|
||||
expect(segments[1].contentCategory).toBe('content')
|
||||
|
||||
expect(segments[2].text).toBe('const x = 1')
|
||||
expect(segments[2].contentCategory).toBe('code')
|
||||
|
|
@ -124,9 +124,9 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(3)
|
||||
expect(segments[0].text).toBe('Slate Heading')
|
||||
expect(segments[0].contentCategory).toBe('heading')
|
||||
expect(segments[0].contentCategory).toBe('title')
|
||||
expect(segments[1].text).toBe('First part ')
|
||||
expect(segments[1].contentCategory).toBe('prose')
|
||||
expect(segments[1].contentCategory).toBe('content')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -150,9 +150,9 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
const segments = extractForHighlighting(doc)
|
||||
expect(segments.length).toBe(2)
|
||||
expect(segments[0].text).toBe('Lexical Heading')
|
||||
expect(segments[0].contentCategory).toBe('heading')
|
||||
expect(segments[0].contentCategory).toBe('title')
|
||||
expect(segments[1].text).toBe('Lexical body text')
|
||||
expect(segments[1].contentCategory).toBe('prose')
|
||||
expect(segments[1].contentCategory).toBe('content')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -223,11 +223,11 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
|
||||
const headingSegment = segments.find(s => s.text === 'Title')
|
||||
expect(headingSegment).toBeDefined()
|
||||
expect(headingSegment?.contentCategory).toBe('heading')
|
||||
expect(headingSegment?.contentCategory).toBe('title')
|
||||
|
||||
const bodySegment = segments.find(s => s.text === 'Body text here.')
|
||||
expect(bodySegment).toBeDefined()
|
||||
expect(bodySegment?.contentCategory).toBe('prose')
|
||||
expect(bodySegment?.contentCategory).toBe('content')
|
||||
|
||||
const codeSegment = segments.find(s => s.text === 'let x = 1')
|
||||
expect(codeSegment).toBeDefined()
|
||||
|
|
@ -257,7 +257,7 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
const segments = extractForHighlighting(html)
|
||||
const heading = segments.find(s => s.text === 'Nested Heading')
|
||||
expect(heading).toBeDefined()
|
||||
expect(heading?.contentCategory).toBe('heading')
|
||||
expect(heading?.contentCategory).toBe('title')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -271,11 +271,11 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
const body = segments.find(s => s.text === 'Some body text')
|
||||
|
||||
expect(heading1).toBeDefined()
|
||||
expect(heading1?.contentCategory).toBe('heading')
|
||||
expect(heading1?.contentCategory).toBe('title')
|
||||
expect(heading2).toBeDefined()
|
||||
expect(heading2?.contentCategory).toBe('heading')
|
||||
expect(heading2?.contentCategory).toBe('title')
|
||||
expect(body).toBeDefined()
|
||||
expect(body?.contentCategory).toBe('prose')
|
||||
expect(body?.contentCategory).toBe('content')
|
||||
})
|
||||
|
||||
it('should extract fenced code blocks', () => {
|
||||
|
|
@ -296,16 +296,51 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
expect(code).toBeDefined()
|
||||
expect(code?.text).toContain('code line 1')
|
||||
})
|
||||
|
||||
it('should split inline code spans into separate segments', () => {
|
||||
const md = '# Title\n\nUse the `authenticate()` function to verify users'
|
||||
|
||||
const segments = extractForHighlighting(md)
|
||||
// Should have: title segment, content "Use the", code "authenticate()", content "function to verify users"
|
||||
const codeSegment = segments.find(s => s.text === 'authenticate()' && s.contentCategory === 'code')
|
||||
expect(codeSegment).toBeDefined()
|
||||
|
||||
const beforeCode = segments.find(s => s.text === 'Use the' && s.contentCategory === 'content')
|
||||
expect(beforeCode).toBeDefined()
|
||||
|
||||
const afterCode = segments.find(s => s.text === 'function to verify users' && s.contentCategory === 'content')
|
||||
expect(afterCode).toBeDefined()
|
||||
})
|
||||
|
||||
it('should handle multiple inline code spans in a single line', () => {
|
||||
const md = '# API\n\nCall `foo()` then `bar()` for results'
|
||||
|
||||
const segments = extractForHighlighting(md, 'markdown')
|
||||
const fooSegment = segments.find(s => s.text === 'foo()' && s.contentCategory === 'code')
|
||||
const barSegment = segments.find(s => s.text === 'bar()' && s.contentCategory === 'code')
|
||||
expect(fooSegment).toBeDefined()
|
||||
expect(barSegment).toBeDefined()
|
||||
})
|
||||
|
||||
it('should not split backticks inside fenced code blocks', () => {
|
||||
const md = '# Title\n\n```\nconst x = `template`\n```'
|
||||
|
||||
const segments = extractForHighlighting(md)
|
||||
const code = segments.find(s => s.contentCategory === 'code')
|
||||
expect(code).toBeDefined()
|
||||
// The fenced block content should be intact, not split by backticks
|
||||
expect(code?.text).toContain('const x = `template`')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractForHighlighting() — Plaintext', () => {
|
||||
it('should return single prose segment for plain text', () => {
|
||||
it('should return single content segment for plain text', () => {
|
||||
const text = 'Just some plain text content'
|
||||
const segments = extractForHighlighting(text)
|
||||
|
||||
expect(segments.length).toBe(1)
|
||||
expect(segments[0].text).toBe(text)
|
||||
expect(segments[0].contentCategory).toBe('prose')
|
||||
expect(segments[0].contentCategory).toBe('content')
|
||||
})
|
||||
|
||||
it('should respect contentType override', () => {
|
||||
|
|
@ -315,7 +350,7 @@ describe('Content Extractor (v7.9.0)', () => {
|
|||
|
||||
expect(segments.length).toBe(1)
|
||||
expect(segments[0].text).toBe(html) // Raw HTML, not extracted
|
||||
expect(segments[0].contentCategory).toBe('prose')
|
||||
expect(segments[0].contentCategory).toBe('content')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue