fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
**How to Use Progress Tracking in Your Applications**
|
||||
|
||||
Brainy v4.5.0+ provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX).
|
||||
Brainy provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX).
|
||||
|
||||
> **⚠️ KEY FEATURE:** The progress API is **100% standardized**. Write your progress handler ONCE and it works for ALL formats with zero format-specific code! See [Standard Import Progress API](./standard-import-progress.md) for the complete interface documentation.
|
||||
|
||||
|
|
@ -20,12 +20,12 @@ const brain = await Brainy.create()
|
|||
|
||||
// Import with progress tracking
|
||||
const result = await brain.import(fs.readFileSync('large-file.xlsx'), {
|
||||
onProgress: (progress) => {
|
||||
console.log(`Progress: ${progress.stage}`)
|
||||
console.log(` Message: ${progress.message}`)
|
||||
console.log(` Entities: ${progress.entities || 0}`)
|
||||
console.log(` Relationships: ${progress.relationships || 0}`)
|
||||
}
|
||||
onProgress: (progress) => {
|
||||
console.log(`Progress: ${progress.stage}`)
|
||||
console.log(` Message: ${progress.message}`)
|
||||
console.log(` Entities: ${progress.entities || 0}`)
|
||||
console.log(` Relationships: ${progress.relationships || 0}`)
|
||||
}
|
||||
})
|
||||
|
||||
console.log(`Import complete: ${result.entities.length} entities created`)
|
||||
|
|
@ -34,30 +34,30 @@ console.log(`Import complete: ${result.entities.length} entities created`)
|
|||
**Expected Output:**
|
||||
```
|
||||
Progress: detecting
|
||||
Message: Detecting format...
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Message: Detecting format...
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Progress: extracting
|
||||
Message: Loading Excel workbook...
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Message: Loading Excel workbook...
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Progress: extracting
|
||||
Message: Reading sheet: Sales (1/3)
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Message: Reading sheet: Sales (1/3)
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Progress: extracting
|
||||
Message: Parsing Excel (33%)
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Message: Parsing Excel (33%)
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Progress: extracting
|
||||
Message: Reading sheet: Products (2/3)
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
Message: Reading sheet: Products (2/3)
|
||||
Entities: 0
|
||||
Relationships: 0
|
||||
... (more progress updates)
|
||||
Progress: complete
|
||||
Message: Import complete
|
||||
Entities: 1523
|
||||
Relationships: 892
|
||||
Message: Import complete
|
||||
Entities: 1523
|
||||
Relationships: 892
|
||||
Import complete: 1523 entities created
|
||||
```
|
||||
|
||||
|
|
@ -70,19 +70,19 @@ The examples below show format-specific messages, but **you don't need format-sp
|
|||
```typescript
|
||||
// ONE HANDLER FOR ALL FORMATS!
|
||||
function universalProgressHandler(progress) {
|
||||
console.log(`[${progress.stage}] ${progress.message}`)
|
||||
console.log(`[${progress.stage}] ${progress.message}`)
|
||||
|
||||
if (progress.processed && progress.total) {
|
||||
console.log(` Progress: ${progress.processed}/${progress.total}`)
|
||||
}
|
||||
if (progress.processed && progress.total) {
|
||||
console.log(` Progress: ${progress.processed}/${progress.total}`)
|
||||
}
|
||||
|
||||
if (progress.entities || progress.relationships) {
|
||||
console.log(` Extracted: ${progress.entities || 0} entities, ${progress.relationships || 0} relationships`)
|
||||
}
|
||||
if (progress.entities || progress.relationships) {
|
||||
console.log(` Extracted: ${progress.entities || 0} entities, ${progress.relationships || 0} relationships`)
|
||||
}
|
||||
|
||||
if (progress.throughput && progress.eta) {
|
||||
console.log(` Rate: ${progress.throughput.toFixed(1)}/sec, ETA: ${Math.round(progress.eta/1000)}s`)
|
||||
}
|
||||
if (progress.throughput && progress.eta) {
|
||||
console.log(` Rate: ${progress.throughput.toFixed(1)}/sec, ETA: ${Math.round(progress.eta/1000)}s`)
|
||||
}
|
||||
}
|
||||
|
||||
// Use it for ANY format!
|
||||
|
|
@ -105,20 +105,20 @@ The examples below show **what messages look like** for different formats using
|
|||
|
||||
```typescript
|
||||
await brain.import(csvBuffer, {
|
||||
format: 'csv',
|
||||
onProgress: (progress) => {
|
||||
if (progress.stage === 'extracting') {
|
||||
// CSV reports: "Parsing CSV (45%)", "Extracted 1000 rows", etc.
|
||||
console.log(progress.message)
|
||||
}
|
||||
}
|
||||
format: 'csv',
|
||||
onProgress: (progress) => {
|
||||
if (progress.stage === 'extracting') {
|
||||
// CSV reports: "Parsing CSV (45%)", "Extracted 1000 rows", etc.
|
||||
console.log(progress.message)
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
**CSV Progress Messages:**
|
||||
- ✅ "Detecting CSV encoding and delimiter..."
|
||||
- ✅ "Parsing CSV rows (delimiter: ",")"
|
||||
- ✅ "Parsed 75%" (via bytes processed)
|
||||
- ✅ "Parsed 75%" (via bytes processed)
|
||||
- ✅ "Extracted 1000 rows"
|
||||
- ✅ "Converting types: 5000/10000 rows..."
|
||||
- ✅ "CSV processing complete: 10000 rows"
|
||||
|
|
@ -129,12 +129,12 @@ await brain.import(csvBuffer, {
|
|||
|
||||
```typescript
|
||||
await brain.import(pdfBuffer, {
|
||||
format: 'pdf',
|
||||
onProgress: (progress) => {
|
||||
// PDF reports exact page numbers
|
||||
console.log(progress.message)
|
||||
// Example: "Processing page 5 of 23"
|
||||
}
|
||||
format: 'pdf',
|
||||
onProgress: (progress) => {
|
||||
// PDF reports exact page numbers
|
||||
console.log(progress.message)
|
||||
// Example: "Processing page 5 of 23"
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -152,12 +152,12 @@ await brain.import(pdfBuffer, {
|
|||
|
||||
```typescript
|
||||
await brain.import(excelBuffer, {
|
||||
format: 'excel',
|
||||
onProgress: (progress) => {
|
||||
// Excel reports sheet names
|
||||
console.log(progress.message)
|
||||
// Example: "Reading sheet: Q2 Sales (2/5)"
|
||||
}
|
||||
format: 'excel',
|
||||
onProgress: (progress) => {
|
||||
// Excel reports sheet names
|
||||
console.log(progress.message)
|
||||
// Example: "Reading sheet: Q2 Sales (2/5)"
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -175,11 +175,11 @@ await brain.import(excelBuffer, {
|
|||
|
||||
```typescript
|
||||
await brain.import(jsonBuffer, {
|
||||
format: 'json',
|
||||
onProgress: (progress) => {
|
||||
// JSON reports every 10 nodes
|
||||
console.log(`Processed ${progress.processed} nodes, found ${progress.entities} entities`)
|
||||
}
|
||||
format: 'json',
|
||||
onProgress: (progress) => {
|
||||
// JSON reports every 10 nodes
|
||||
console.log(`Processed ${progress.processed} nodes, found ${progress.entities} entities`)
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -189,10 +189,10 @@ await brain.import(jsonBuffer, {
|
|||
|
||||
```typescript
|
||||
await brain.import(markdownString, {
|
||||
format: 'markdown',
|
||||
onProgress: (progress) => {
|
||||
console.log(`Section ${progress.processed}/${progress.total}`)
|
||||
}
|
||||
format: 'markdown',
|
||||
onProgress: (progress) => {
|
||||
console.log(`Section ${progress.processed}/${progress.total}`)
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -204,44 +204,44 @@ await brain.import(markdownString, {
|
|||
|
||||
```typescript
|
||||
function ImportProgress({ file }: { file: File }) {
|
||||
const [progress, setProgress] = useState({
|
||||
stage: 'idle',
|
||||
message: '',
|
||||
percent: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
const [progress, setProgress] = useState({
|
||||
stage: 'idle',
|
||||
message: '',
|
||||
percent: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
const handleImport = async () => {
|
||||
const buffer = await file.arrayBuffer()
|
||||
const handleImport = async () => {
|
||||
const buffer = await file.arrayBuffer()
|
||||
|
||||
await brain.import(Buffer.from(buffer), {
|
||||
onProgress: (p) => {
|
||||
setProgress({
|
||||
stage: p.stage,
|
||||
message: p.message,
|
||||
// Estimate percentage from stage
|
||||
percent: {
|
||||
detecting: 10,
|
||||
extracting: 50,
|
||||
'storing-vfs': 80,
|
||||
'storing-graph': 90,
|
||||
complete: 100
|
||||
}[p.stage] || 0,
|
||||
entities: p.entities || 0,
|
||||
relationships: p.relationships || 0
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
await brain.import(Buffer.from(buffer), {
|
||||
onProgress: (p) => {
|
||||
setProgress({
|
||||
stage: p.stage,
|
||||
message: p.message,
|
||||
// Estimate percentage from stage
|
||||
percent: {
|
||||
detecting: 10,
|
||||
extracting: 50,
|
||||
'storing-vfs': 80,
|
||||
'storing-graph': 90,
|
||||
complete: 100
|
||||
}[p.stage] || 0,
|
||||
entities: p.entities || 0,
|
||||
relationships: p.relationships || 0
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<ProgressBar value={progress.percent} />
|
||||
<p>{progress.message}</p>
|
||||
<p>Entities: {progress.entities} | Relationships: {progress.relationships}</p>
|
||||
</div>
|
||||
)
|
||||
return (
|
||||
<div>
|
||||
<ProgressBar value={progress.percent} />
|
||||
<p>{progress.message}</p>
|
||||
<p>Entities: {progress.entities} | Relationships: {progress.relationships}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -255,13 +255,13 @@ import ora from 'ora'
|
|||
const spinner = ora('Starting import...').start()
|
||||
|
||||
await brain.import(buffer, {
|
||||
onProgress: (progress) => {
|
||||
spinner.text = progress.message
|
||||
onProgress: (progress) => {
|
||||
spinner.text = progress.message
|
||||
|
||||
if (progress.stage === 'complete') {
|
||||
spinner.succeed(`Import complete: ${progress.entities} entities`)
|
||||
}
|
||||
}
|
||||
if (progress.stage === 'complete') {
|
||||
spinner.succeed(`Import complete: ${progress.entities} entities`)
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -285,20 +285,20 @@ let startTime = Date.now()
|
|||
let lastUpdate = startTime
|
||||
|
||||
await brain.import(buffer, {
|
||||
onProgress: (progress) => {
|
||||
const elapsed = Date.now() - startTime
|
||||
const rate = progress.entities / (elapsed / 1000) // entities/sec
|
||||
onProgress: (progress) => {
|
||||
const elapsed = Date.now() - startTime
|
||||
const rate = progress.entities / (elapsed / 1000) // entities/sec
|
||||
|
||||
console.clear()
|
||||
console.log('Import Progress Dashboard')
|
||||
console.log('========================')
|
||||
console.log(`Stage: ${progress.stage}`)
|
||||
console.log(`Status: ${progress.message}`)
|
||||
console.log(`Entities: ${progress.entities}`)
|
||||
console.log(`Relationships: ${progress.relationships}`)
|
||||
console.log(`Rate: ${rate.toFixed(1)} entities/sec`)
|
||||
console.log(`Elapsed: ${(elapsed / 1000).toFixed(1)}s`)
|
||||
}
|
||||
console.clear()
|
||||
console.log('Import Progress Dashboard')
|
||||
console.log('========================')
|
||||
console.log(`Stage: ${progress.stage}`)
|
||||
console.log(`Status: ${progress.message}`)
|
||||
console.log(`Entities: ${progress.entities}`)
|
||||
console.log(`Relationships: ${progress.relationships}`)
|
||||
console.log(`Rate: ${rate.toFixed(1)} entities/sec`)
|
||||
console.log(`Elapsed: ${(elapsed / 1000).toFixed(1)}s`)
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -310,21 +310,21 @@ await brain.import(buffer, {
|
|||
|
||||
```typescript
|
||||
const formatMessages = {
|
||||
csv: (p) => `CSV: ${p.message}`,
|
||||
pdf: (p) => `PDF: ${p.message}`,
|
||||
excel: (p) => `Excel: ${p.message}`,
|
||||
json: (p) => `JSON: ${p.processed} nodes, ${p.entities} entities`,
|
||||
markdown: (p) => `Markdown: Section ${p.processed}/${p.total}`,
|
||||
yaml: (p) => `YAML: ${p.processed} nodes`,
|
||||
docx: (p) => `DOCX: ${p.processed} paragraphs`
|
||||
csv: (p) => `CSV: ${p.message}`,
|
||||
pdf: (p) => `PDF: ${p.message}`,
|
||||
excel: (p) => `Excel: ${p.message}`,
|
||||
json: (p) => `JSON: ${p.processed} nodes, ${p.entities} entities`,
|
||||
markdown: (p) => `Markdown: Section ${p.processed}/${p.total}`,
|
||||
yaml: (p) => `YAML: ${p.processed} nodes`,
|
||||
docx: (p) => `DOCX: ${p.processed} paragraphs`
|
||||
}
|
||||
|
||||
await brain.import(buffer, {
|
||||
onProgress: (progress) => {
|
||||
// Format is available in progress.stage metadata
|
||||
const message = formatMessages[detectedFormat]?.(progress) || progress.message
|
||||
console.log(message)
|
||||
}
|
||||
onProgress: (progress) => {
|
||||
// Format is available in progress.stage metadata
|
||||
const message = formatMessages[detectedFormat]?.(progress) || progress.message
|
||||
console.log(message)
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -336,18 +336,18 @@ await brain.import(buffer, {
|
|||
|
||||
```typescript
|
||||
let lastUIUpdate = 0
|
||||
const THROTTLE_MS = 100 // Update UI max once per 100ms
|
||||
const THROTTLE_MS = 100 // Update UI max once per 100ms
|
||||
|
||||
await brain.import(buffer, {
|
||||
onProgress: (progress) => {
|
||||
const now = Date.now()
|
||||
if (now - lastUIUpdate < THROTTLE_MS && progress.stage !== 'complete') {
|
||||
return // Skip this update
|
||||
}
|
||||
onProgress: (progress) => {
|
||||
const now = Date.now()
|
||||
if (now - lastUIUpdate < THROTTLE_MS && progress.stage !== 'complete') {
|
||||
return // Skip this update
|
||||
}
|
||||
|
||||
lastUIUpdate = now
|
||||
updateUI(progress) // Only update every 100ms
|
||||
}
|
||||
lastUIUpdate = now
|
||||
updateUI(progress) // Only update every 100ms
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue