/** * Content Extractor * * Detects content type (plaintext, rich-text JSON, HTML, Markdown) and extracts * text segments with content categories (title, content, code, etc.). * * Supports common rich-text editor formats: * - TipTap / ProseMirror: { type: 'doc', content: [...] } * - Slate.js: [{ type, children: [{ text }] }] * - Lexical: { root: { children: [...] } } * - Draft.js: { blocks: [{ text }] } * - Quill Delta: { ops: [{ insert }] } * * Falls back gracefully: structured text that doesn't match known patterns * is extracted as plain content via recursive text collection. */ import type { ContentType, ContentCategory, ExtractedSegment } from '../types/brainy.types.js' /** * Detect content type from text content (no filename needed) */ export function detectContentType(text: string): ContentType { const trimmed = text.trimStart() // JSON detection (covers TipTap, Slate, Lexical, Draft.js, Quill, etc.) if (trimmed.startsWith('{') || trimmed.startsWith('[')) { try { JSON.parse(trimmed) return 'richtext-json' } catch { // Not valid JSON, fall through } } // HTML detection (tag at start of content) if (/<[a-z!][a-z0-9]*[\s>/]/i.test(trimmed.substring(0, 200))) { return 'html' } // Markdown detection (heading or code fence at start) if (/^#{1,6}\s/m.test(trimmed.substring(0, 500)) || /^```/m.test(trimmed.substring(0, 500))) { return 'markdown' } return 'plaintext' } /** * Extract text segments from content, auto-detecting type if not provided */ export function extractForHighlighting( text: string, contentType?: ContentType ): ExtractedSegment[] { const type = contentType || detectContentType(text) switch (type) { case 'richtext-json': return extractFromJson(text) case 'html': return extractFromHtml(text) case 'markdown': return extractFromMarkdown(text) case 'plaintext': default: return [{ text, contentCategory: 'content' }] } } // ============= Rich-Text JSON Extraction ============= /** * Extract text from any JSON rich-text format. * Walks the node tree looking for text leaves across all common editors. */ function extractFromJson(text: string): ExtractedSegment[] { let parsed: any try { parsed = JSON.parse(text.trim()) } catch { // Invalid JSON — treat as plain text return [{ text, contentCategory: 'content' }] } const segments = walkRichTextNodes(parsed) // If the walker found real text segments, return them if (segments.length > 0 && segments.some(s => s.text.trim().length > 0)) { return segments.filter(s => s.text.trim().length > 0) } // Fallback: collect all string values from the JSON const fallbackText = extractTextFromJsonValue(parsed) if (fallbackText.trim()) { return [{ text: fallbackText, contentCategory: 'content' }] } return [] } /** * Walk rich-text nodes recursively. * Handles TipTap/ProseMirror, Slate, Lexical, Draft.js, and Quill Delta. */ function walkRichTextNodes(node: any): ExtractedSegment[] { if (node === null || node === undefined) return [] const segments: ExtractedSegment[] = [] // Handle arrays (Slate root is an array, Draft.js blocks, Quill ops) if (Array.isArray(node)) { for (const child of node) { segments.push(...walkRichTextNodes(child)) } return segments } if (typeof node !== 'object') return [] // Leaf: text content (TipTap/ProseMirror/Lexical/Slate) if (typeof node.text === 'string' && node.text.length > 0) { segments.push({ text: node.text, contentCategory: 'content' }) return segments } // Leaf: Quill Delta insert if (typeof node.insert === 'string' && node.insert.length > 0) { segments.push({ text: node.insert, contentCategory: 'content' }) return segments } // Draft.js block with text if (typeof node.text === 'string' && node.type !== undefined && node.text.length > 0) { const category = categorizeNodeType(node.type) segments.push({ text: node.text, contentCategory: category }) return segments } // Categorize by node type const category = categorizeNodeType(node.type) // Walk children arrays: content (TipTap), children (Slate/Lexical), blocks (Draft.js), ops (Quill) const children = node.content || node.children || node.blocks || node.ops if (Array.isArray(children)) { for (const child of children) { const childSegments = walkRichTextNodes(child) // Apply parent's category if it's more specific than 'content' if (category !== 'content') { childSegments.forEach(s => { s.contentCategory = category }) } segments.push(...childSegments) } } // Lexical root wrapper if (node.root && typeof node.root === 'object') { segments.push(...walkRichTextNodes(node.root)) } return segments } /** * Categorize a node type string into a ContentCategory */ function categorizeNodeType(type?: string): ContentCategory { if (!type) return 'content' const t = type.toLowerCase() if (t === 'heading' || /^h[1-6]$/.test(t)) return 'title' if (t === 'code' || t === 'codeblock' || t === 'code_block') return 'code' return 'content' } /** * Fallback: recursively extract all string values from JSON */ function extractTextFromJsonValue(value: any): string { if (typeof value === 'string') return value if (Array.isArray(value)) { return value.map(extractTextFromJsonValue).filter(Boolean).join(' ') } if (typeof value === 'object' && value !== null) { const texts: string[] = [] for (const v of Object.values(value)) { const text = extractTextFromJsonValue(v) if (text) texts.push(text) } return texts.join(' ') } return '' } // ============= HTML Extraction ============= /** * Extract text from HTML using a simple state-machine tag parser. * No external dependencies. */ function extractFromHtml(html: string): ExtractedSegment[] { const segments: ExtractedSegment[] = [] let current = '' let currentCategory: ContentCategory = 'content' let i = 0 let insideTag = false let tagName = '' let isClosingTag = false let skipContent = false // Tag stack to track nesting const tagStack: string[] = [] while (i < html.length) { if (html[i] === '<') { // Flush current text if we have any if (current.trim() && !skipContent) { segments.push({ text: current.trim(), contentCategory: currentCategory }) } current = '' insideTag = true tagName = '' isClosingTag = false i++ // Check for closing tag if (i < html.length && html[i] === '/') { isClosingTag = true i++ } // Read tag name while (i < html.length && html[i] !== '>' && html[i] !== ' ' && html[i] !== '/') { tagName += html[i] i++ } // Skip to end of tag while (i < html.length && html[i] !== '>') { i++ } if (i < html.length) i++ // skip '>' const tagLower = tagName.toLowerCase() if (isClosingTag) { // Pop tag stack if (tagStack.length > 0 && tagStack[tagStack.length - 1] === tagLower) { tagStack.pop() } if (tagLower === 'script' || tagLower === 'style') { skipContent = false } // Reset category based on remaining stack currentCategory = getCategoryFromTagStack(tagStack) } else { // Opening tag if (tagLower === 'script' || tagLower === 'style') { skipContent = true } // Self-closing tags don't affect stack const selfClosing = html[i - 2] === '/' || ['br', 'hr', 'img', 'input', 'meta', 'link'].includes(tagLower) if (!selfClosing) { tagStack.push(tagLower) currentCategory = getCategoryFromTagStack(tagStack) } } insideTag = false continue } if (!insideTag) { // Decode common HTML entities if (html[i] === '&') { const entityEnd = html.indexOf(';', i) if (entityEnd !== -1 && entityEnd - i < 10) { const entity = html.substring(i, entityEnd + 1) current += decodeHtmlEntity(entity) i = entityEnd + 1 continue } } current += html[i] } i++ } // Flush remaining text if (current.trim() && !skipContent) { segments.push({ text: current.trim(), contentCategory: currentCategory }) } return segments.filter(s => s.text.length > 0) } /** * Determine content category from the current tag stack */ function getCategoryFromTagStack(stack: string[]): ContentCategory { for (let i = stack.length - 1; i >= 0; i--) { const tag = stack[i] if (/^h[1-6]$/.test(tag)) return 'title' if (tag === 'code' || tag === 'pre') return 'code' } return 'content' } /** * Decode common HTML entities */ function decodeHtmlEntity(entity: string): string { const entities: Record = { '&': '&', '<': '<', '>': '>', '"': '"', ''': "'", ''': "'", ' ': ' ', } return entities[entity] || entity } // ============= Markdown Extraction ============= /** * Split a prose line into alternating content/code segments based on * backtick-delimited inline code spans. Lines without backticks return * a single 'content' segment. */ function splitInlineCode(line: string): ExtractedSegment[] { const segments: ExtractedSegment[] = [] const pattern = /`([^`]+)`/g let lastIndex = 0 let match: RegExpExecArray | null while ((match = pattern.exec(line)) !== null) { // Text before the backtick span if (match.index > lastIndex) { const before = line.substring(lastIndex, match.index).trim() if (before.length > 0) { segments.push({ text: before, contentCategory: 'content' }) } } // The code span (without backticks) segments.push({ text: match[1], contentCategory: 'code' }) lastIndex = match.index + match[0].length } // Remaining text after last backtick span (or entire line if no backticks) if (lastIndex < line.length) { const remaining = line.substring(lastIndex).trim() if (remaining.length > 0) { segments.push({ text: remaining, contentCategory: 'content' }) } } return segments } /** * Extract text from Markdown with category detection */ function extractFromMarkdown(text: string): ExtractedSegment[] { const segments: ExtractedSegment[] = [] const lines = text.split('\n') let inCodeBlock = false let codeContent = '' let i = 0 while (i < lines.length) { const line = lines[i] // Fenced code block toggle if (/^```/.test(line.trimStart())) { if (inCodeBlock) { // End of code block if (codeContent.trim()) { segments.push({ text: codeContent.trim(), contentCategory: 'code' }) } codeContent = '' inCodeBlock = false } else { // Start of code block inCodeBlock = true } i++ continue } if (inCodeBlock) { codeContent += (codeContent ? '\n' : '') + line i++ continue } // Indented code block (4 spaces or 1 tab) if (/^(?: |\t)/.test(line) && line.trim().length > 0) { let codeLines = line.replace(/^(?: |\t)/, '') i++ while (i < lines.length && (/^(?: |\t)/.test(lines[i]) || lines[i].trim() === '')) { codeLines += '\n' + lines[i].replace(/^(?: |\t)/, '') i++ } if (codeLines.trim()) { segments.push({ text: codeLines.trim(), contentCategory: 'code' }) } continue } // Heading (# style) const headingMatch = line.match(/^#{1,6}\s+(.+)$/) if (headingMatch) { segments.push({ text: headingMatch[1].trim(), contentCategory: 'title' }) i++ continue } // Regular content line — split out inline code spans if (line.trim().length > 0) { segments.push(...splitInlineCode(line.trim())) } i++ } // Flush unclosed code block if (inCodeBlock && codeContent.trim()) { segments.push({ text: codeContent.trim(), contentCategory: 'code' }) } return segments.filter(s => s.text.length > 0) }