feat: expand ContentCategory to universal 6-category set for highlight()

Replace document-centric categories (prose/heading/code/label) with a
universal set that works across documents, code, and UI:

- title: headings, identifiers, labels, JSON keys
- annotation: comments, docstrings, captions, alt text
- content: paragraphs, list items, flowing text
- value: string literals, numbers, form values
- code: unparsed code blocks
- structural: keywords, operators, punctuation

Built-in extractors now produce title/content/code. All 6 categories
are available for custom parsers (e.g. tree-sitter).

Also adds inline code detection in Markdown: backtick spans within
prose lines are split into separate code/content segments.
This commit is contained in:
David Snelling 2026-01-27 13:44:58 -08:00
parent 6c27f9f7fd
commit ff80b87a0a
6 changed files with 699 additions and 651 deletions

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -83,11 +83,12 @@ export interface Result<T = any> {
// Score transparency // Score transparency
explanation?: ScoreExplanation explanation?: ScoreExplanation
// Match visibility (v7.8.0) - shows what matched in hybrid search // Match visibility - shows what matched in hybrid search
textMatches?: string[] // Query words found in entity (e.g., ["david", "warrior"]) textMatches?: string[] // Query words found in entity (e.g., ["david", "warrior"])
textScore?: number // Normalized text match score (0-1) textScore?: number // Normalized text match score (0-1)
semanticScore?: number // Semantic similarity score (0-1) semanticScore?: number // Semantic similarity score (0-1)
matchSource?: 'text' | 'semantic' | 'both' // Where this result came from matchSource?: 'text' | 'semantic' | 'both' // Where this result came from
rrfScore?: number // Raw RRF fusion score (for advanced ranking analysis)
} }
/** /**
@ -188,7 +189,7 @@ export interface FindParams<T = any> {
offset?: number // Skip N results offset?: number // Skip N results
cursor?: string // Cursor-based pagination cursor?: string // Cursor-based pagination
// Sorting (v4.5.4) // Sorting
orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority') orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority')
order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc'
@ -196,10 +197,10 @@ export interface FindParams<T = any> {
mode?: SearchMode // Search strategy mode?: SearchMode // Search strategy
explain?: boolean // Return scoring explanation explain?: boolean // Return scoring explanation
includeRelations?: boolean // Include entity relationships includeRelations?: boolean // Include entity relationships
excludeVFS?: boolean // v4.7.0: Exclude VFS entities from results (default: false - VFS included) excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included)
service?: string // Multi-tenancy filter service?: string // Multi-tenancy filter
// Hybrid search options (v7.7.0) // Hybrid search options
searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid
hybridAlpha?: number // Weight between text (0.0) and semantic (1.0) search, default: auto-detected by query length hybridAlpha?: number // Weight between text (0.0) and semantic (1.0) search, default: auto-detected by query length
@ -252,7 +253,7 @@ export interface SimilarParams<T = any> {
type?: NounType | NounType[] // Restrict to types type?: NounType | NounType[] // Restrict to types
where?: Partial<T> // Additional filters where?: Partial<T> // Additional filters
service?: string // Multi-tenancy service?: string // Multi-tenancy
excludeVFS?: boolean // v4.7.0: Exclude VFS entities (default: false - VFS included) excludeVFS?: boolean // Exclude VFS entities (default: false - VFS included)
} }
/** /**
@ -289,8 +290,8 @@ export interface SimilarParams<T = any> {
* }) * })
* ``` * ```
* *
* @since v4.1.3 - Fixed bug where calling without parameters returned empty array * Fixed bug where calling without parameters returned empty array
* @since v4.1.3 - Added string ID shorthand syntax * Added string ID shorthand syntax
*/ */
export interface GetRelationsParams { export interface GetRelationsParams {
/** /**
@ -376,7 +377,7 @@ export interface DeleteManyParams {
where?: any // Delete by metadata where?: any // Delete by metadata
limit?: number // Max to delete (safety) limit?: number // Max to delete (safety)
onProgress?: (done: number, total: number) => void onProgress?: (done: number, total: number) => void
continueOnError?: boolean // v6.2.0: Continue processing if a delete fails continueOnError?: boolean // Continue processing if a delete fails
} }
/** /**
@ -403,7 +404,7 @@ export interface BatchResult<T = any> {
duration: number // Time taken in ms duration: number // Time taken in ms
} }
// ============= Import Progress (v4.5.0) ============= // ============= Import Progress =============
/** /**
* Import stage enumeration * Import stage enumeration
@ -435,7 +436,6 @@ export type ImportStatus =
* - Time estimates * - Time estimates
* - Performance metrics * - Performance metrics
* *
* @since v4.5.0
*/ */
export interface ImportProgress { export interface ImportProgress {
// Overall Progress // Overall Progress
@ -540,7 +540,7 @@ export interface ImportResult {
/** /**
* Options for brain.get() entity retrieval * Options for brain.get() entity retrieval
* *
* **Performance Optimization (v5.11.1)**: * **Performance Optimization**:
* By default, brain.get() loads ONLY metadata (not vectors), resulting in: * By default, brain.get() loads ONLY metadata (not vectors), resulting in:
* - **76-81% faster** reads (10ms vs 43ms for metadata-only) * - **76-81% faster** reads (10ms vs 43ms for metadata-only)
* - **95% less bandwidth** (300 bytes vs 6KB per entity) * - **95% less bandwidth** (300 bytes vs 6KB per entity)
@ -573,7 +573,6 @@ export interface ImportResult {
* await vfs.readFile('/file.txt') // 53ms → 10ms (81% faster) * await vfs.readFile('/file.txt') // 53ms → 10ms (81% faster)
* ``` * ```
* *
* @since v5.11.1
*/ */
export interface GetOptions { export interface GetOptions {
/** /**
@ -635,7 +634,7 @@ export type AggregateMetric =
// ============= Configuration ============= // ============= Configuration =============
/** /**
* Integration Hub configuration (v7.4.0) * Integration Hub configuration
* *
* Enables external tool integrations: Excel, Power BI, Google Sheets, etc. * Enables external tool integrations: Excel, Power BI, Google Sheets, etc.
* Works in all environments with zero external dependencies. * Works in all environments with zero external dependencies.
@ -722,18 +721,18 @@ export interface BrainyConfig {
batchWrites?: boolean // Enable write batching for better performance batchWrites?: boolean // Enable write batching for better performance
maxConcurrentOperations?: number // Limit concurrent file operations maxConcurrentOperations?: number // Limit concurrent file operations
// HNSW persistence mode (v6.2.8) // HNSW persistence mode
// Controls when HNSW graph connections are persisted to storage // Controls when HNSW graph connections are persisted to storage
// - 'immediate': Persist on every add (slow but durable, default for filesystem) // - 'immediate': Persist on every add (slow but durable, default for filesystem)
// - 'deferred': Persist only on flush/close (fast, default for cloud storage) // - 'deferred': Persist only on flush/close (fast, default for cloud storage)
// Cloud storage (GCS/S3/R2/Azure) should use 'deferred' for 30-50× faster adds // Cloud storage (GCS/S3/R2/Azure) should use 'deferred' for 30-50× faster adds
hnswPersistMode?: 'immediate' | 'deferred' hnswPersistMode?: 'immediate' | 'deferred'
// Memory management options (v5.11.0) // Memory management options
maxQueryLimit?: number // Override auto-detected query result limit (max: 100000) maxQueryLimit?: number // Override auto-detected query result limit (max: 100000)
reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB) reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB)
// Embedding initialization (v7.1.2) // Embedding initialization
// Controls when the WASM embedding engine is initialized // Controls when the WASM embedding engine is initialized
// - false (default): Lazy initialization on first embed() call // - false (default): Lazy initialization on first embed() call
// - true: Eager initialization during brain.init() // - true: Eager initialization during brain.init()
@ -745,7 +744,7 @@ export interface BrainyConfig {
verbose?: boolean // Enable verbose logging verbose?: boolean // Enable verbose logging
silent?: boolean // Suppress all logging output silent?: boolean // Suppress all logging output
// Integration Hub (v7.4.0) // Integration Hub
// Enable external tool integrations: Excel, Power BI, Google Sheets, etc. // Enable external tool integrations: Excel, Power BI, Google Sheets, etc.
// - true: Enable all integrations with default paths // - true: Enable all integrations with default paths
// - false/undefined: Disable integrations (default) // - false/undefined: Disable integrations (default)
@ -789,54 +788,55 @@ export interface NeuralAnomalyParams {
returnScores?: boolean // Return anomaly scores returnScores?: boolean // Return anomaly scores
} }
// ============= Content Extraction Types (v7.9.0) ============= // ============= Content Extraction Types =============
/** /**
* Detected content type for smart text extraction * Detected content type for smart text extraction
* *
* @since v7.9.0
*/ */
export type ContentType = 'plaintext' | 'richtext-json' | 'html' | 'markdown' export type ContentType = 'plaintext' | 'richtext-json' | 'html' | 'markdown'
/** /**
* Content category for extracted segments * Content category for extracted segments
* *
* Allows UI to apply different styling based on content role: * Universal categories that work across documents, code, and UI:
* - 'prose': Regular text content (paragraphs, list items) * - 'title': Names the subject headings, identifiers, labels, JSON keys
* - 'heading': Heading/title text (h1-h6) * - 'annotation': Human explanation comments, docstrings, captions, alt text
* - 'code': Code blocks or inline code * - 'content': Body substance paragraphs, list items, flowing text
* - 'label': Labels, captions, or metadata-like text * - 'value': Data literals strings, numbers, form values, error messages
* - 'code': Unparsed code blocks (custom parsers decompose into above)
* - 'structural': Boilerplate keywords, operators, punctuation, formatting
* *
* @since v7.9.0 * Built-in extractors produce: 'title', 'content', 'code'.
* All 6 categories are available for custom parsers.
*/ */
export type ContentCategory = 'prose' | 'heading' | 'code' | 'label' export type ContentCategory = 'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural'
/** /**
* A segment of extracted text with its content category * A segment of extracted text with its content category
* *
* @since v7.9.0
*/ */
export interface ExtractedSegment { export interface ExtractedSegment {
/** The extracted text content */ /** The extracted text content */
text: string text: string
/** What role this text plays in the document */ /** What role this text plays: 'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural' */
contentCategory: ContentCategory contentCategory: ContentCategory
} }
// ============= Semantic Highlighting Types (v7.8.0) ============= // ============= Semantic Highlighting Types =============
/** /**
* Parameters for hybrid highlighting (v7.8.0, enhanced v7.9.0) * Parameters for hybrid highlighting
* *
* Zero-config highlighting that returns both text (exact) and semantic (concept) matches. * Zero-config highlighting that returns both text (exact) and semantic (concept) matches.
* Perfect for UI highlighting at different levels. * Perfect for UI highlighting at different levels.
* *
* v7.9.0: Added contentType hint and contentExtractor callback for structured text * Added contentType hint and contentExtractor callback for structured text
* (rich-text JSON, HTML, Markdown). Auto-detects format when not specified. * (rich-text JSON, HTML, Markdown). Auto-detects format when not specified.
* *
* @example * @example
* ```typescript * ```typescript
* // Plain text (unchanged from v7.8.0) * // Plain text
* const highlights = await brain.highlight({ * const highlights = await brain.highlight({
* query: "david the warrior", * query: "david the warrior",
* text: "David Smith is a brave fighter who battles dragons" * text: "David Smith is a brave fighter who battles dragons"
@ -872,7 +872,6 @@ export interface HighlightParams {
/** /**
* Optional content type hint to skip auto-detection. * Optional content type hint to skip auto-detection.
* When omitted, the content type is detected from the text content. * When omitted, the content type is detected from the text content.
* @since v7.9.0
*/ */
contentType?: ContentType contentType?: ContentType
@ -880,7 +879,6 @@ export interface HighlightParams {
* Optional custom content extractor function. * Optional custom content extractor function.
* When provided, bypasses built-in detection and extraction entirely. * When provided, bypasses built-in detection and extraction entirely.
* Use this to plug in custom parsers (tree-sitter, Monaco, proprietary formats). * Use this to plug in custom parsers (tree-sitter, Monaco, proprietary formats).
* @since v7.9.0
*/ */
contentExtractor?: (text: string) => ExtractedSegment[] contentExtractor?: (text: string) => ExtractedSegment[]
} }
@ -906,9 +904,8 @@ export interface Highlight {
matchType: 'text' | 'semantic' matchType: 'text' | 'semantic'
/** /**
* Content category of the source segment (heading, code, prose, label). * Content category of the source segment: 'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural'.
* Present when the input text was structured (JSON, HTML, Markdown). * Present when the input text was structured (JSON, HTML, Markdown).
* @since v7.9.0
*/ */
contentCategory?: ContentCategory contentCategory?: ContentCategory
} }

View file

@ -1,8 +1,8 @@
/** /**
* Content Extractor (v7.9.0) * Content Extractor
* *
* Detects content type (plaintext, rich-text JSON, HTML, Markdown) and extracts * Detects content type (plaintext, rich-text JSON, HTML, Markdown) and extracts
* text segments with content categories (prose, heading, code, label). * text segments with content categories (title, content, code, etc.).
* *
* Supports common rich-text editor formats: * Supports common rich-text editor formats:
* - TipTap / ProseMirror: { type: 'doc', content: [...] } * - TipTap / ProseMirror: { type: 'doc', content: [...] }
@ -12,7 +12,7 @@
* - Quill Delta: { ops: [{ insert }] } * - Quill Delta: { ops: [{ insert }] }
* *
* Falls back gracefully: structured text that doesn't match known patterns * Falls back gracefully: structured text that doesn't match known patterns
* is extracted as plain prose via recursive text collection. * is extracted as plain content via recursive text collection.
*/ */
import type { ContentType, ContentCategory, ExtractedSegment } from '../types/brainy.types.js' import type { ContentType, ContentCategory, ExtractedSegment } from '../types/brainy.types.js'
@ -65,7 +65,7 @@ export function extractForHighlighting(
return extractFromMarkdown(text) return extractFromMarkdown(text)
case 'plaintext': case 'plaintext':
default: default:
return [{ text, contentCategory: 'prose' }] return [{ text, contentCategory: 'content' }]
} }
} }
@ -81,7 +81,7 @@ function extractFromJson(text: string): ExtractedSegment[] {
parsed = JSON.parse(text.trim()) parsed = JSON.parse(text.trim())
} catch { } catch {
// Invalid JSON — treat as plain text // Invalid JSON — treat as plain text
return [{ text, contentCategory: 'prose' }] return [{ text, contentCategory: 'content' }]
} }
const segments = walkRichTextNodes(parsed) const segments = walkRichTextNodes(parsed)
@ -94,7 +94,7 @@ function extractFromJson(text: string): ExtractedSegment[] {
// Fallback: collect all string values from the JSON // Fallback: collect all string values from the JSON
const fallbackText = extractTextFromJsonValue(parsed) const fallbackText = extractTextFromJsonValue(parsed)
if (fallbackText.trim()) { if (fallbackText.trim()) {
return [{ text: fallbackText, contentCategory: 'prose' }] return [{ text: fallbackText, contentCategory: 'content' }]
} }
return [] return []
@ -121,13 +121,13 @@ function walkRichTextNodes(node: any): ExtractedSegment[] {
// Leaf: text content (TipTap/ProseMirror/Lexical/Slate) // Leaf: text content (TipTap/ProseMirror/Lexical/Slate)
if (typeof node.text === 'string' && node.text.length > 0) { if (typeof node.text === 'string' && node.text.length > 0) {
segments.push({ text: node.text, contentCategory: 'prose' }) segments.push({ text: node.text, contentCategory: 'content' })
return segments return segments
} }
// Leaf: Quill Delta insert // Leaf: Quill Delta insert
if (typeof node.insert === 'string' && node.insert.length > 0) { if (typeof node.insert === 'string' && node.insert.length > 0) {
segments.push({ text: node.insert, contentCategory: 'prose' }) segments.push({ text: node.insert, contentCategory: 'content' })
return segments return segments
} }
@ -146,8 +146,8 @@ function walkRichTextNodes(node: any): ExtractedSegment[] {
if (Array.isArray(children)) { if (Array.isArray(children)) {
for (const child of children) { for (const child of children) {
const childSegments = walkRichTextNodes(child) const childSegments = walkRichTextNodes(child)
// Apply parent's category if it's more specific than 'prose' // Apply parent's category if it's more specific than 'content'
if (category !== 'prose') { if (category !== 'content') {
childSegments.forEach(s => { s.contentCategory = category }) childSegments.forEach(s => { s.contentCategory = category })
} }
segments.push(...childSegments) segments.push(...childSegments)
@ -166,11 +166,11 @@ function walkRichTextNodes(node: any): ExtractedSegment[] {
* Categorize a node type string into a ContentCategory * Categorize a node type string into a ContentCategory
*/ */
function categorizeNodeType(type?: string): ContentCategory { function categorizeNodeType(type?: string): ContentCategory {
if (!type) return 'prose' if (!type) return 'content'
const t = type.toLowerCase() const t = type.toLowerCase()
if (t === 'heading' || /^h[1-6]$/.test(t)) return 'heading' if (t === 'heading' || /^h[1-6]$/.test(t)) return 'title'
if (t === 'code' || t === 'codeblock' || t === 'code_block') return 'code' if (t === 'code' || t === 'codeblock' || t === 'code_block') return 'code'
return 'prose' return 'content'
} }
/** /**
@ -201,7 +201,7 @@ function extractTextFromJsonValue(value: any): string {
function extractFromHtml(html: string): ExtractedSegment[] { function extractFromHtml(html: string): ExtractedSegment[] {
const segments: ExtractedSegment[] = [] const segments: ExtractedSegment[] = []
let current = '' let current = ''
let currentCategory: ContentCategory = 'prose' let currentCategory: ContentCategory = 'content'
let i = 0 let i = 0
let insideTag = false let insideTag = false
let tagName = '' let tagName = ''
@ -304,10 +304,10 @@ function extractFromHtml(html: string): ExtractedSegment[] {
function getCategoryFromTagStack(stack: string[]): ContentCategory { function getCategoryFromTagStack(stack: string[]): ContentCategory {
for (let i = stack.length - 1; i >= 0; i--) { for (let i = stack.length - 1; i >= 0; i--) {
const tag = stack[i] const tag = stack[i]
if (/^h[1-6]$/.test(tag)) return 'heading' if (/^h[1-6]$/.test(tag)) return 'title'
if (tag === 'code' || tag === 'pre') return 'code' if (tag === 'code' || tag === 'pre') return 'code'
} }
return 'prose' return 'content'
} }
/** /**
@ -328,6 +328,41 @@ function decodeHtmlEntity(entity: string): string {
// ============= Markdown Extraction ============= // ============= Markdown Extraction =============
/**
* Split a prose line into alternating content/code segments based on
* backtick-delimited inline code spans. Lines without backticks return
* a single 'content' segment.
*/
function splitInlineCode(line: string): ExtractedSegment[] {
const segments: ExtractedSegment[] = []
const pattern = /`([^`]+)`/g
let lastIndex = 0
let match: RegExpExecArray | null
while ((match = pattern.exec(line)) !== null) {
// Text before the backtick span
if (match.index > lastIndex) {
const before = line.substring(lastIndex, match.index).trim()
if (before.length > 0) {
segments.push({ text: before, contentCategory: 'content' })
}
}
// The code span (without backticks)
segments.push({ text: match[1], contentCategory: 'code' })
lastIndex = match.index + match[0].length
}
// Remaining text after last backtick span (or entire line if no backticks)
if (lastIndex < line.length) {
const remaining = line.substring(lastIndex).trim()
if (remaining.length > 0) {
segments.push({ text: remaining, contentCategory: 'content' })
}
}
return segments
}
/** /**
* Extract text from Markdown with category detection * Extract text from Markdown with category detection
*/ */
@ -381,14 +416,14 @@ function extractFromMarkdown(text: string): ExtractedSegment[] {
// Heading (# style) // Heading (# style)
const headingMatch = line.match(/^#{1,6}\s+(.+)$/) const headingMatch = line.match(/^#{1,6}\s+(.+)$/)
if (headingMatch) { if (headingMatch) {
segments.push({ text: headingMatch[1].trim(), contentCategory: 'heading' }) segments.push({ text: headingMatch[1].trim(), contentCategory: 'title' })
i++ i++
continue continue
} }
// Regular prose line // Regular content line — split out inline code spans
if (line.trim().length > 0) { if (line.trim().length > 0) {
segments.push({ text: line.trim(), contentCategory: 'prose' }) segments.push(...splitInlineCode(line.trim()))
} }
i++ i++

View file

@ -769,7 +769,7 @@ describe('Hybrid Search (v7.7.0)', () => {
expect(warriorMatch).toBeDefined() expect(warriorMatch).toBeDefined()
expect(warriorMatch?.matchType).toBe('text') expect(warriorMatch?.matchType).toBe('text')
// Should annotate heading content // Should annotate heading content
expect(warriorMatch?.contentCategory).toBe('heading') expect(warriorMatch?.contentCategory).toBe('title')
}) })
it('should extract text from Slate.js JSON', async () => { it('should extract text from Slate.js JSON', async () => {
@ -799,7 +799,7 @@ describe('Hybrid Search (v7.7.0)', () => {
const introMatch = highlights.find(h => h.text === 'Introduction') const introMatch = highlights.find(h => h.text === 'Introduction')
expect(introMatch).toBeDefined() expect(introMatch).toBeDefined()
expect(introMatch?.matchType).toBe('text') expect(introMatch?.matchType).toBe('text')
expect(introMatch?.contentCategory).toBe('heading') expect(introMatch?.contentCategory).toBe('title')
}) })
it('should extract text from Quill Delta JSON', async () => { it('should extract text from Quill Delta JSON', async () => {
@ -856,12 +856,12 @@ describe('Hybrid Search (v7.7.0)', () => {
// Should find "Warrior" from heading // Should find "Warrior" from heading
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior') const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
expect(warriorMatch).toBeDefined() expect(warriorMatch).toBeDefined()
expect(warriorMatch?.contentCategory).toBe('heading') expect(warriorMatch?.contentCategory).toBe('title')
// Should find "fighter" from paragraph // Should find "fighter" from paragraph
const fighterMatch = highlights.find(h => h.text.toLowerCase() === 'fighter') const fighterMatch = highlights.find(h => h.text.toLowerCase() === 'fighter')
expect(fighterMatch).toBeDefined() expect(fighterMatch).toBeDefined()
expect(fighterMatch?.contentCategory).toBe('prose') expect(fighterMatch?.contentCategory).toBe('content')
}) })
it('should extract text from Markdown with headings and code', async () => { it('should extract text from Markdown with headings and code', async () => {
@ -876,8 +876,8 @@ describe('Hybrid Search (v7.7.0)', () => {
expect(highlights.length).toBeGreaterThan(0) expect(highlights.length).toBeGreaterThan(0)
// Should find text from heading // Should find text from heading
const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'heading') const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'title')
|| highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'heading') || highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'title')
expect(headingMatch).toBeDefined() expect(headingMatch).toBeDefined()
}) })
@ -896,8 +896,8 @@ describe('Hybrid Search (v7.7.0)', () => {
expect(warriorMatch).toBeDefined() expect(warriorMatch).toBeDefined()
expect(warriorMatch?.matchType).toBe('text') expect(warriorMatch?.matchType).toBe('text')
expect(warriorMatch?.score).toBe(1.0) expect(warriorMatch?.score).toBe(1.0)
// Plain text gets 'prose' category // Plain text gets 'content' category
expect(warriorMatch?.contentCategory).toBe('prose') expect(warriorMatch?.contentCategory).toBe('content')
}) })
it('should use contentType hint to skip auto-detection', async () => { it('should use contentType hint to skip auto-detection', async () => {
@ -920,7 +920,7 @@ describe('Hybrid Search (v7.7.0)', () => {
it('should use custom contentExtractor when provided', async () => { it('should use custom contentExtractor when provided', async () => {
const customExtractor = (text: string) => { const customExtractor = (text: string) => {
return [ return [
{ text: 'custom extracted warrior content', contentCategory: 'prose' as const }, { text: 'custom extracted warrior content', contentCategory: 'content' as const },
{ text: 'Code Section', contentCategory: 'code' as const } { text: 'Code Section', contentCategory: 'code' as const }
] ]
} }
@ -937,8 +937,9 @@ describe('Hybrid Search (v7.7.0)', () => {
const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior') const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior')
expect(warriorMatch).toBeDefined() expect(warriorMatch).toBeDefined()
expect(warriorMatch?.matchType).toBe('text') expect(warriorMatch?.matchType).toBe('text')
expect(warriorMatch?.contentCategory).toBe('prose') expect(warriorMatch?.contentCategory).toBe('content')
}) })
}) })
describe('Timeout Protection (v7.9.0)', () => { describe('Timeout Protection (v7.9.0)', () => {

View file

@ -75,10 +75,10 @@ describe('Content Extractor (v7.9.0)', () => {
expect(segments.length).toBe(3) expect(segments.length).toBe(3)
expect(segments[0].text).toBe('My Heading') expect(segments[0].text).toBe('My Heading')
expect(segments[0].contentCategory).toBe('heading') expect(segments[0].contentCategory).toBe('title')
expect(segments[1].text).toBe('Regular paragraph text') expect(segments[1].text).toBe('Regular paragraph text')
expect(segments[1].contentCategory).toBe('prose') expect(segments[1].contentCategory).toBe('content')
expect(segments[2].text).toBe('const x = 1') expect(segments[2].text).toBe('const x = 1')
expect(segments[2].contentCategory).toBe('code') expect(segments[2].contentCategory).toBe('code')
@ -124,9 +124,9 @@ describe('Content Extractor (v7.9.0)', () => {
const segments = extractForHighlighting(doc) const segments = extractForHighlighting(doc)
expect(segments.length).toBe(3) expect(segments.length).toBe(3)
expect(segments[0].text).toBe('Slate Heading') expect(segments[0].text).toBe('Slate Heading')
expect(segments[0].contentCategory).toBe('heading') expect(segments[0].contentCategory).toBe('title')
expect(segments[1].text).toBe('First part ') expect(segments[1].text).toBe('First part ')
expect(segments[1].contentCategory).toBe('prose') expect(segments[1].contentCategory).toBe('content')
}) })
}) })
@ -150,9 +150,9 @@ describe('Content Extractor (v7.9.0)', () => {
const segments = extractForHighlighting(doc) const segments = extractForHighlighting(doc)
expect(segments.length).toBe(2) expect(segments.length).toBe(2)
expect(segments[0].text).toBe('Lexical Heading') expect(segments[0].text).toBe('Lexical Heading')
expect(segments[0].contentCategory).toBe('heading') expect(segments[0].contentCategory).toBe('title')
expect(segments[1].text).toBe('Lexical body text') expect(segments[1].text).toBe('Lexical body text')
expect(segments[1].contentCategory).toBe('prose') expect(segments[1].contentCategory).toBe('content')
}) })
}) })
@ -223,11 +223,11 @@ describe('Content Extractor (v7.9.0)', () => {
const headingSegment = segments.find(s => s.text === 'Title') const headingSegment = segments.find(s => s.text === 'Title')
expect(headingSegment).toBeDefined() expect(headingSegment).toBeDefined()
expect(headingSegment?.contentCategory).toBe('heading') expect(headingSegment?.contentCategory).toBe('title')
const bodySegment = segments.find(s => s.text === 'Body text here.') const bodySegment = segments.find(s => s.text === 'Body text here.')
expect(bodySegment).toBeDefined() expect(bodySegment).toBeDefined()
expect(bodySegment?.contentCategory).toBe('prose') expect(bodySegment?.contentCategory).toBe('content')
const codeSegment = segments.find(s => s.text === 'let x = 1') const codeSegment = segments.find(s => s.text === 'let x = 1')
expect(codeSegment).toBeDefined() expect(codeSegment).toBeDefined()
@ -257,7 +257,7 @@ describe('Content Extractor (v7.9.0)', () => {
const segments = extractForHighlighting(html) const segments = extractForHighlighting(html)
const heading = segments.find(s => s.text === 'Nested Heading') const heading = segments.find(s => s.text === 'Nested Heading')
expect(heading).toBeDefined() expect(heading).toBeDefined()
expect(heading?.contentCategory).toBe('heading') expect(heading?.contentCategory).toBe('title')
}) })
}) })
@ -271,11 +271,11 @@ describe('Content Extractor (v7.9.0)', () => {
const body = segments.find(s => s.text === 'Some body text') const body = segments.find(s => s.text === 'Some body text')
expect(heading1).toBeDefined() expect(heading1).toBeDefined()
expect(heading1?.contentCategory).toBe('heading') expect(heading1?.contentCategory).toBe('title')
expect(heading2).toBeDefined() expect(heading2).toBeDefined()
expect(heading2?.contentCategory).toBe('heading') expect(heading2?.contentCategory).toBe('title')
expect(body).toBeDefined() expect(body).toBeDefined()
expect(body?.contentCategory).toBe('prose') expect(body?.contentCategory).toBe('content')
}) })
it('should extract fenced code blocks', () => { it('should extract fenced code blocks', () => {
@ -296,16 +296,51 @@ describe('Content Extractor (v7.9.0)', () => {
expect(code).toBeDefined() expect(code).toBeDefined()
expect(code?.text).toContain('code line 1') expect(code?.text).toContain('code line 1')
}) })
it('should split inline code spans into separate segments', () => {
const md = '# Title\n\nUse the `authenticate()` function to verify users'
const segments = extractForHighlighting(md)
// Should have: title segment, content "Use the", code "authenticate()", content "function to verify users"
const codeSegment = segments.find(s => s.text === 'authenticate()' && s.contentCategory === 'code')
expect(codeSegment).toBeDefined()
const beforeCode = segments.find(s => s.text === 'Use the' && s.contentCategory === 'content')
expect(beforeCode).toBeDefined()
const afterCode = segments.find(s => s.text === 'function to verify users' && s.contentCategory === 'content')
expect(afterCode).toBeDefined()
})
it('should handle multiple inline code spans in a single line', () => {
const md = '# API\n\nCall `foo()` then `bar()` for results'
const segments = extractForHighlighting(md, 'markdown')
const fooSegment = segments.find(s => s.text === 'foo()' && s.contentCategory === 'code')
const barSegment = segments.find(s => s.text === 'bar()' && s.contentCategory === 'code')
expect(fooSegment).toBeDefined()
expect(barSegment).toBeDefined()
})
it('should not split backticks inside fenced code blocks', () => {
const md = '# Title\n\n```\nconst x = `template`\n```'
const segments = extractForHighlighting(md)
const code = segments.find(s => s.contentCategory === 'code')
expect(code).toBeDefined()
// The fenced block content should be intact, not split by backticks
expect(code?.text).toContain('const x = `template`')
})
}) })
describe('extractForHighlighting() — Plaintext', () => { describe('extractForHighlighting() — Plaintext', () => {
it('should return single prose segment for plain text', () => { it('should return single content segment for plain text', () => {
const text = 'Just some plain text content' const text = 'Just some plain text content'
const segments = extractForHighlighting(text) const segments = extractForHighlighting(text)
expect(segments.length).toBe(1) expect(segments.length).toBe(1)
expect(segments[0].text).toBe(text) expect(segments[0].text).toBe(text)
expect(segments[0].contentCategory).toBe('prose') expect(segments[0].contentCategory).toBe('content')
}) })
it('should respect contentType override', () => { it('should respect contentType override', () => {
@ -315,7 +350,7 @@ describe('Content Extractor (v7.9.0)', () => {
expect(segments.length).toBe(1) expect(segments.length).toBe(1)
expect(segments[0].text).toBe(html) // Raw HTML, not extracted expect(segments[0].text).toBe(html) // Raw HTML, not extracted
expect(segments[0].contentCategory).toBe('prose') expect(segments[0].contentCategory).toBe('content')
}) })
}) })
}) })