fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -42,17 +42,17 @@ export interface SmartCSVOptions extends FormatHandlerOptions {
|
|||
csvDelimiter?: string
|
||||
csvHeaders?: boolean
|
||||
|
||||
/** Progress callback (v3.39.0: Enhanced with performance metrics) */
|
||||
/** Progress callback (Enhanced with performance metrics) */
|
||||
onProgress?: (stats: {
|
||||
processed: number
|
||||
total: number
|
||||
entities: number
|
||||
relationships: number
|
||||
/** Rows per second (v3.39.0) */
|
||||
/** Rows per second */
|
||||
throughput?: number
|
||||
/** Estimated time remaining in ms (v3.39.0) */
|
||||
/** Estimated time remaining in ms */
|
||||
eta?: number
|
||||
/** Current phase (v3.39.0) */
|
||||
/** Current phase */
|
||||
phase?: string
|
||||
}) => void
|
||||
}
|
||||
|
|
@ -169,7 +169,7 @@ export class SmartCSVImporter {
|
|||
}
|
||||
|
||||
// Parse CSV using existing handler
|
||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
||||
// Pass progress hooks to handler for file parsing progress
|
||||
const processedData = await this.csvHandler.process(buffer, {
|
||||
...options,
|
||||
csvDelimiter: opts.csvDelimiter,
|
||||
|
|
@ -217,7 +217,7 @@ export class SmartCSVImporter {
|
|||
// Detect column names
|
||||
const columns = this.detectColumns(rows[0], opts)
|
||||
|
||||
// Process each row with BATCHED PARALLEL PROCESSING (v3.39.0)
|
||||
// Process each row with BATCHED PARALLEL PROCESSING
|
||||
const extractedRows: ExtractedRow[] = []
|
||||
const entityMap = new Map<string, string>()
|
||||
const stats = {
|
||||
|
|
@ -375,7 +375,7 @@ export class SmartCSVImporter {
|
|||
total: rows.length,
|
||||
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
|
||||
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
|
||||
// Additional performance metrics (v3.39.0)
|
||||
// Additional performance metrics
|
||||
throughput: Math.round(rowsPerSecond * 10) / 10,
|
||||
eta: Math.round(estimatedTimeRemaining),
|
||||
phase: 'extracting'
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
* - NaturalLanguageProcessor for relationship inference
|
||||
* - Hierarchical relationship creation based on heading hierarchy
|
||||
*
|
||||
* v4.2.0: New format handler
|
||||
* New format handler
|
||||
* NO MOCKS - Production-ready implementation
|
||||
*/
|
||||
|
||||
|
|
@ -187,7 +187,7 @@ export class SmartDOCXImporter {
|
|||
await this.init()
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
// Report parsing start
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
|
|
@ -200,7 +200,7 @@ export class SmartDOCXImporter {
|
|||
// Extract HTML for structure analysis (headings, tables)
|
||||
const htmlResult = await mammoth.convertToHtml({ buffer })
|
||||
|
||||
// v4.5.0: Report parsing complete
|
||||
// Report parsing complete
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
|
|
|
|||
|
|
@ -36,17 +36,17 @@ export interface SmartExcelOptions extends FormatHandlerOptions {
|
|||
typeColumn?: string // e.g., "Type", "Category"
|
||||
relatedColumn?: string // e.g., "Related Terms", "See Also"
|
||||
|
||||
/** Progress callback (v3.38.0: Enhanced with performance metrics) */
|
||||
/** Progress callback (Enhanced with performance metrics) */
|
||||
onProgress?: (stats: {
|
||||
processed: number
|
||||
total: number
|
||||
entities: number
|
||||
relationships: number
|
||||
/** Rows per second (v3.38.0) */
|
||||
/** Rows per second */
|
||||
throughput?: number
|
||||
/** Estimated time remaining in ms (v3.38.0) */
|
||||
/** Estimated time remaining in ms */
|
||||
eta?: number
|
||||
/** Current phase (v3.38.0) */
|
||||
/** Current phase */
|
||||
phase?: string
|
||||
}) => void
|
||||
}
|
||||
|
|
@ -111,7 +111,7 @@ export interface SmartExcelResult {
|
|||
}
|
||||
}
|
||||
|
||||
/** Sheet-specific data for VFS extraction (v4.2.0) */
|
||||
/** Sheet-specific data for VFS extraction */
|
||||
sheets?: Array<{
|
||||
name: string
|
||||
rows: ExtractedRow[]
|
||||
|
|
@ -161,7 +161,7 @@ export class SmartExcelImporter {
|
|||
const opts = {
|
||||
enableNeuralExtraction: true,
|
||||
enableRelationshipInference: true,
|
||||
// CONCEPT EXTRACTION PRODUCTION-READY (v3.33.0+):
|
||||
// CONCEPT EXTRACTION PRODUCTION-READY:
|
||||
// Type embeddings are now pre-computed at build time - zero runtime cost!
|
||||
// All 42 noun types + 127 verb types instantly available
|
||||
//
|
||||
|
|
@ -184,7 +184,7 @@ export class SmartExcelImporter {
|
|||
}
|
||||
|
||||
// Parse Excel using existing handler
|
||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
||||
// Pass progress hooks to handler for file parsing progress
|
||||
const processedData = await this.excelHandler.process(buffer, {
|
||||
...options,
|
||||
totalBytes: buffer.length,
|
||||
|
|
@ -227,7 +227,7 @@ export class SmartExcelImporter {
|
|||
return this.emptyResult(startTime)
|
||||
}
|
||||
|
||||
// CRITICAL FIX (v4.8.6): Detect columns per-sheet, not globally
|
||||
// CRITICAL FIX: Detect columns per-sheet, not globally
|
||||
// Different sheets may have different column structures (Term vs Name, etc.)
|
||||
// Group rows by sheet and detect columns for each sheet separately
|
||||
const rowsBySheet = new Map<string, typeof rows>()
|
||||
|
|
@ -247,7 +247,7 @@ export class SmartExcelImporter {
|
|||
}
|
||||
}
|
||||
|
||||
// Process each row with BATCHED PARALLEL PROCESSING (v3.38.0)
|
||||
// Process each row with BATCHED PARALLEL PROCESSING
|
||||
const extractedRows: ExtractedRow[] = []
|
||||
const entityMap = new Map<string, string>()
|
||||
const stats = {
|
||||
|
|
@ -269,7 +269,7 @@ export class SmartExcelImporter {
|
|||
chunk.map(async (row, chunkIndex) => {
|
||||
const i = chunkStart + chunkIndex
|
||||
|
||||
// CRITICAL FIX (v4.8.6): Use sheet-specific column mapping
|
||||
// CRITICAL FIX: Use sheet-specific column mapping
|
||||
const sheet = row._sheet || 'default'
|
||||
const columns = columnsBySheet.get(sheet) || this.detectColumns(row, opts)
|
||||
|
||||
|
|
@ -353,7 +353,7 @@ export class SmartExcelImporter {
|
|||
}
|
||||
|
||||
// ============================================
|
||||
// v4.9.0: Enhanced column-based relationship detection
|
||||
// Enhanced column-based relationship detection
|
||||
// ============================================
|
||||
// Parse explicit "Related Terms" column
|
||||
if (relatedTerms) {
|
||||
|
|
@ -379,7 +379,7 @@ export class SmartExcelImporter {
|
|||
}
|
||||
}
|
||||
|
||||
// v4.9.0: Check for additional relationship-indicating columns
|
||||
// Check for additional relationship-indicating columns
|
||||
// Expanded patterns for various relationship types
|
||||
const relationshipColumnPatterns = [
|
||||
{ pattern: /^(location|home|lives in|resides|dwelling|place)$/i, defaultType: VerbType.LocatedAt },
|
||||
|
|
@ -485,14 +485,14 @@ export class SmartExcelImporter {
|
|||
total: rows.length,
|
||||
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
|
||||
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
|
||||
// Additional performance metrics (v3.38.0)
|
||||
// Additional performance metrics
|
||||
throughput: Math.round(rowsPerSecond * 10) / 10,
|
||||
eta: Math.round(estimatedTimeRemaining),
|
||||
phase: 'extracting'
|
||||
} as any)
|
||||
}
|
||||
|
||||
// Group rows by sheet for VFS extraction (v4.2.0)
|
||||
// Group rows by sheet for VFS extraction
|
||||
const sheetGroups = new Map<string, ExtractedRow[]>()
|
||||
extractedRows.forEach((extractedRow, index) => {
|
||||
const originalRow = rows[index]
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ export class SmartJSONImporter {
|
|||
...options
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
// Report parsing start
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
|
|
@ -186,7 +186,7 @@ export class SmartJSONImporter {
|
|||
jsonData = data
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing complete, starting traversal
|
||||
// Report parsing complete, starting traversal
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
|
|
@ -228,7 +228,7 @@ export class SmartJSONImporter {
|
|||
}
|
||||
)
|
||||
|
||||
// v4.5.0: Report completion
|
||||
// Report completion
|
||||
opts.onProgress({
|
||||
processed: nodesProcessed,
|
||||
entities: entities.length,
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ export class SmartMarkdownImporter {
|
|||
...options
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
// Report parsing start
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
|
|
@ -185,7 +185,7 @@ export class SmartMarkdownImporter {
|
|||
// Parse markdown into sections
|
||||
const parsedSections = this.parseMarkdown(markdown, opts)
|
||||
|
||||
// v4.5.0: Report parsing complete
|
||||
// Report parsing complete
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
total: parsedSections.length,
|
||||
|
|
@ -218,7 +218,7 @@ export class SmartMarkdownImporter {
|
|||
})
|
||||
}
|
||||
|
||||
// v4.5.0: Report completion
|
||||
// Report completion
|
||||
const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0)
|
||||
const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0)
|
||||
opts.onProgress({
|
||||
|
|
|
|||
|
|
@ -40,17 +40,17 @@ export interface SmartPDFOptions extends FormatHandlerOptions {
|
|||
/** Group by page or full document */
|
||||
groupBy?: 'page' | 'document'
|
||||
|
||||
/** Progress callback (v3.39.0: Enhanced with performance metrics) */
|
||||
/** Progress callback (Enhanced with performance metrics) */
|
||||
onProgress?: (stats: {
|
||||
processed: number
|
||||
total: number
|
||||
entities: number
|
||||
relationships: number
|
||||
/** Sections per second (v3.39.0) */
|
||||
/** Sections per second */
|
||||
throughput?: number
|
||||
/** Estimated time remaining in ms (v3.39.0) */
|
||||
/** Estimated time remaining in ms */
|
||||
eta?: number
|
||||
/** Current phase (v3.39.0) */
|
||||
/** Current phase */
|
||||
phase?: string
|
||||
}) => void
|
||||
}
|
||||
|
|
@ -181,7 +181,7 @@ export class SmartPDFImporter {
|
|||
}
|
||||
|
||||
// Parse PDF using existing handler
|
||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
||||
// Pass progress hooks to handler for file parsing progress
|
||||
const processedData = await this.pdfHandler.process(buffer, {
|
||||
...options,
|
||||
totalBytes: buffer.length,
|
||||
|
|
@ -228,7 +228,7 @@ export class SmartPDFImporter {
|
|||
// Group data by page or combine into single document
|
||||
const grouped = this.groupData(data, opts)
|
||||
|
||||
// Process each group with BATCHED PARALLEL PROCESSING (v3.39.0)
|
||||
// Process each group with BATCHED PARALLEL PROCESSING
|
||||
const sections: ExtractedSection[] = []
|
||||
const entityMap = new Map<string, string>()
|
||||
const stats = {
|
||||
|
|
@ -270,7 +270,7 @@ export class SmartPDFImporter {
|
|||
total: totalGroups,
|
||||
entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
|
||||
relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0),
|
||||
// Additional performance metrics (v3.39.0)
|
||||
// Additional performance metrics
|
||||
throughput: Math.round(sectionsPerSecond * 10) / 10,
|
||||
eta: Math.round(estimatedTimeRemaining),
|
||||
phase: 'extracting'
|
||||
|
|
@ -367,7 +367,7 @@ export class SmartPDFImporter {
|
|||
|
||||
const combinedText = texts.join('\n\n')
|
||||
|
||||
// Parallel extraction: entities AND concepts at the same time (v3.39.0)
|
||||
// Parallel extraction: entities AND concepts at the same time
|
||||
const [extractedEntities, concepts] = await Promise.all([
|
||||
// Extract entities if enabled
|
||||
options.enableNeuralExtraction && combinedText.length > 0
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* - NaturalLanguageProcessor for relationship inference
|
||||
* - Hierarchical relationship creation (parent-child, contains, etc.)
|
||||
*
|
||||
* v4.2.0: New format handler
|
||||
* New format handler
|
||||
* NO MOCKS - Production-ready implementation
|
||||
*/
|
||||
|
||||
|
|
@ -159,7 +159,7 @@ export class SmartYAMLImporter {
|
|||
): Promise<SmartYAMLResult> {
|
||||
const startTime = Date.now()
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
// Report parsing start
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
|
|
@ -178,7 +178,7 @@ export class SmartYAMLImporter {
|
|||
throw new Error(`Failed to parse YAML: ${error.message}`)
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing complete
|
||||
// Report parsing complete
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
|
|
|
|||
|
|
@ -40,10 +40,10 @@ export interface VFSStructureOptions {
|
|||
/** Create metadata file */
|
||||
createMetadataFile?: boolean
|
||||
|
||||
/** Import tracking context (v4.10.0) */
|
||||
/** Import tracking context */
|
||||
trackingContext?: TrackingContext
|
||||
|
||||
/** Progress callback (v4.11.1) - Reports VFS creation progress */
|
||||
/** Progress callback - Reports VFS creation progress */
|
||||
onProgress?: (progress: {
|
||||
stage: 'directories' | 'entities' | 'metadata'
|
||||
message: string
|
||||
|
|
@ -98,7 +98,7 @@ export class VFSStructureGenerator {
|
|||
// Get brain's cached VFS instance (creates if doesn't exist)
|
||||
this.vfs = this.brain.vfs
|
||||
|
||||
// CRITICAL FIX (v4.10.2): Always call vfs.init() explicitly
|
||||
// CRITICAL FIX: Always call vfs.init() explicitly
|
||||
// The previous code tried to check if initialized via stat('/') but this was unreliable
|
||||
// vfs.init() is idempotent, so calling it multiple times is safe
|
||||
await this.vfs.init()
|
||||
|
|
@ -123,7 +123,7 @@ export class VFSStructureGenerator {
|
|||
// Ensure VFS is initialized
|
||||
await this.init()
|
||||
|
||||
// v4.11.1: Calculate total operations for progress tracking
|
||||
// Calculate total operations for progress tracking
|
||||
const groups = this.groupEntities(importResult, options)
|
||||
const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0)
|
||||
const totalOperations =
|
||||
|
|
@ -161,7 +161,7 @@ export class VFSStructureGenerator {
|
|||
try {
|
||||
await this.vfs.mkdir(options.rootPath, {
|
||||
recursive: true,
|
||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
||||
metadata: trackingMetadata // Add tracking metadata
|
||||
})
|
||||
result.directories.push(options.rootPath)
|
||||
result.operations++
|
||||
|
|
@ -179,7 +179,7 @@ export class VFSStructureGenerator {
|
|||
if (options.preserveSource && options.sourceBuffer && options.sourceFilename) {
|
||||
const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}`
|
||||
await this.vfs.writeFile(sourcePath, options.sourceBuffer, {
|
||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
||||
metadata: trackingMetadata // Add tracking metadata
|
||||
})
|
||||
result.files.push({
|
||||
path: sourcePath,
|
||||
|
|
@ -200,7 +200,7 @@ export class VFSStructureGenerator {
|
|||
try {
|
||||
await this.vfs.mkdir(groupPath, {
|
||||
recursive: true,
|
||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
||||
metadata: trackingMetadata // Add tracking metadata
|
||||
})
|
||||
result.directories.push(groupPath)
|
||||
result.operations++
|
||||
|
|
@ -242,7 +242,7 @@ export class VFSStructureGenerator {
|
|||
|
||||
await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2), {
|
||||
metadata: {
|
||||
...trackingMetadata, // v4.10.0: Add tracking metadata
|
||||
...trackingMetadata, // Add tracking metadata
|
||||
entityId: extracted.entity.id
|
||||
}
|
||||
})
|
||||
|
|
@ -253,7 +253,7 @@ export class VFSStructureGenerator {
|
|||
})
|
||||
result.operations++
|
||||
|
||||
// v4.11.1: Report progress every 10 entities (or on last entity)
|
||||
// Report progress every 10 entities (or on last entity)
|
||||
if (entityCount % 10 === 0 || entityCount === entities.length) {
|
||||
reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`)
|
||||
}
|
||||
|
|
@ -280,7 +280,7 @@ export class VFSStructureGenerator {
|
|||
}
|
||||
|
||||
await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2), {
|
||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
||||
metadata: trackingMetadata // Add tracking metadata
|
||||
})
|
||||
result.files.push({
|
||||
path: relationshipsPath,
|
||||
|
|
@ -322,7 +322,7 @@ export class VFSStructureGenerator {
|
|||
}
|
||||
|
||||
await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2), {
|
||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
||||
metadata: trackingMetadata // Add tracking metadata
|
||||
})
|
||||
result.files.push({
|
||||
path: metadataPath,
|
||||
|
|
@ -332,7 +332,7 @@ export class VFSStructureGenerator {
|
|||
reportProgress('metadata', 'Created metadata file')
|
||||
}
|
||||
|
||||
// v4.11.1: Final progress update
|
||||
// Final progress update
|
||||
if (options.onProgress) {
|
||||
options.onProgress({
|
||||
stage: 'metadata',
|
||||
|
|
@ -355,7 +355,7 @@ export class VFSStructureGenerator {
|
|||
): Map<string, typeof importResult.rows> {
|
||||
const groups = new Map<string, typeof importResult.rows>()
|
||||
|
||||
// Handle sheet-based grouping (v4.2.0)
|
||||
// Handle sheet-based grouping
|
||||
if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) {
|
||||
for (const sheet of importResult.sheets) {
|
||||
groups.set(sheet.name, sheet.rows)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue