fix: prevent infinite loop in pagination when storage returns phantom items
- Fix FileSystemStorage counting non-existent files in totalCount - Add safety check in BaseStorage to prevent hasMore:true with empty items - Ensure pagination terminates correctly even with corrupted storage state
This commit is contained in:
parent
8f7eed05e0
commit
5f10f8d9ab
10 changed files with 340 additions and 22 deletions
|
|
@ -22,6 +22,7 @@ import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
|
|||
import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js'
|
||||
import { MetadataIndexManager } from './utils/metadataIndex.js'
|
||||
import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js'
|
||||
import { configureLogger, LogLevel } from './utils/logger.js'
|
||||
import {
|
||||
Entity,
|
||||
Relation,
|
||||
|
|
@ -94,15 +95,24 @@ export class Brainy<T = any> {
|
|||
storage: { ...this.config.storage, ...configOverrides.storage },
|
||||
model: { ...this.config.model, ...configOverrides.model },
|
||||
index: { ...this.config.index, ...configOverrides.index },
|
||||
augmentations: { ...this.config.augmentations, ...configOverrides.augmentations }
|
||||
augmentations: { ...this.config.augmentations, ...configOverrides.augmentations },
|
||||
verbose: configOverrides.verbose ?? this.config.verbose,
|
||||
silent: configOverrides.silent ?? this.config.silent
|
||||
}
|
||||
|
||||
|
||||
// Set dimensions if provided
|
||||
if (dimensions) {
|
||||
this.dimensions = dimensions
|
||||
}
|
||||
}
|
||||
|
||||
// Configure logging based on config options
|
||||
if (this.config.silent) {
|
||||
configureLogger({ level: -1 as LogLevel }) // Suppress all logs
|
||||
} else if (this.config.verbose) {
|
||||
configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging
|
||||
}
|
||||
|
||||
try {
|
||||
// Setup and initialize storage
|
||||
this.storage = await this.setupStorage()
|
||||
|
|
@ -1487,7 +1497,9 @@ export class Brainy<T = any> {
|
|||
warmup: config?.warmup ?? false,
|
||||
realtime: config?.realtime ?? false,
|
||||
multiTenancy: config?.multiTenancy ?? false,
|
||||
telemetry: config?.telemetry ?? false
|
||||
telemetry: config?.telemetry ?? false,
|
||||
verbose: config?.verbose ?? false,
|
||||
silent: config?.silent ?? false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4053,4 +4053,6 @@ export const PATTERNS_METADATA = {
|
|||
}
|
||||
}
|
||||
|
||||
console.log(`🧠 Brainy Pattern Library loaded: ${EMBEDDED_PATTERNS.length} patterns, ${(PATTERNS_METADATA.sizeBytes.total / 1024).toFixed(1)}KB total`)
|
||||
// Only log if not suppressed - controlled by logging configuration
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
prodLog.info(`🧠 Brainy Pattern Library loaded: ${EMBEDDED_PATTERNS.length} patterns, ${(PATTERNS_METADATA.sizeBytes.total / 1024).toFixed(1)}KB total`)
|
||||
|
|
|
|||
|
|
@ -623,9 +623,25 @@ export class FileSystemStorage extends BaseStorage {
|
|||
|
||||
// Get page of files
|
||||
const pageFiles = nounFiles.slice(startIndex, startIndex + limit)
|
||||
|
||||
// Load nouns
|
||||
|
||||
// Load nouns - count actual successfully loaded items
|
||||
const items: HNSWNoun[] = []
|
||||
let successfullyLoaded = 0
|
||||
let totalValidFiles = 0
|
||||
|
||||
// First pass: count total valid files (for accurate totalCount)
|
||||
// This is necessary to fix the pagination bug
|
||||
for (const file of nounFiles) {
|
||||
try {
|
||||
// Just check if file exists and is readable
|
||||
await fs.promises.access(path.join(this.nounsDir, file), fs.constants.R_OK)
|
||||
totalValidFiles++
|
||||
} catch {
|
||||
// File not readable, skip
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: load the current page
|
||||
for (const file of pageFiles) {
|
||||
try {
|
||||
const data = await fs.promises.readFile(
|
||||
|
|
@ -633,7 +649,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
'utf-8'
|
||||
)
|
||||
const noun = JSON.parse(data)
|
||||
|
||||
|
||||
// Apply filter if provided
|
||||
if (options.filter) {
|
||||
// Simple filter implementation
|
||||
|
|
@ -646,21 +662,24 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
if (!matches) continue
|
||||
}
|
||||
|
||||
|
||||
items.push(noun)
|
||||
successfullyLoaded++
|
||||
} catch (error) {
|
||||
console.warn(`Failed to read noun file ${file}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
const hasMore = startIndex + limit < nounFiles.length
|
||||
|
||||
// CRITICAL FIX: hasMore should be based on actual valid files, not just file count
|
||||
// Also check if we actually loaded any items from this page
|
||||
const hasMore = (startIndex + limit < totalValidFiles) && (successfullyLoaded > 0 || startIndex === 0)
|
||||
const nextCursor = hasMore && pageFiles.length > 0
|
||||
? pageFiles[pageFiles.length - 1].replace('.json', '')
|
||||
: undefined
|
||||
|
||||
|
||||
return {
|
||||
items,
|
||||
totalCount: nounFiles.length,
|
||||
totalCount: totalValidFiles, // Use actual valid file count, not all files
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
|
|
|
|||
|
|
@ -414,10 +414,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// Apply offset if needed (some adapters might not support offset)
|
||||
const items = result.items.slice(offset)
|
||||
|
||||
// CRITICAL SAFETY CHECK: Prevent infinite loops
|
||||
// If we have no items but hasMore is true, force hasMore to false
|
||||
// This prevents pagination bugs from causing infinite loops
|
||||
const safeHasMore = items.length > 0 ? result.hasMore : false
|
||||
|
||||
return {
|
||||
items,
|
||||
totalCount: result.totalCount || totalCount,
|
||||
hasMore: result.hasMore,
|
||||
hasMore: safeHasMore,
|
||||
nextCursor: result.nextCursor
|
||||
}
|
||||
}
|
||||
|
|
@ -606,10 +611,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// Apply offset if needed (some adapters might not support offset)
|
||||
const items = result.items.slice(offset)
|
||||
|
||||
// CRITICAL SAFETY CHECK: Prevent infinite loops
|
||||
// If we have no items but hasMore is true, force hasMore to false
|
||||
// This prevents pagination bugs from causing infinite loops
|
||||
const safeHasMore = items.length > 0 ? result.hasMore : false
|
||||
|
||||
return {
|
||||
items,
|
||||
totalCount: result.totalCount || totalCount,
|
||||
hasMore: result.hasMore,
|
||||
hasMore: safeHasMore,
|
||||
nextCursor: result.nextCursor
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -343,6 +343,10 @@ export interface BrainyConfig {
|
|||
realtime?: boolean // Enable real-time updates
|
||||
multiTenancy?: boolean // Enable service isolation
|
||||
telemetry?: boolean // Send anonymous usage stats
|
||||
|
||||
// Logging configuration
|
||||
verbose?: boolean // Enable verbose logging
|
||||
silent?: boolean // Suppress all logging output
|
||||
}
|
||||
|
||||
// ============= Neural API Types =============
|
||||
|
|
|
|||
|
|
@ -1507,12 +1507,33 @@ export class MetadataIndexManager {
|
|||
const nounLimit = 25 // Even smaller batches during initialization to prevent socket exhaustion
|
||||
let hasMoreNouns = true
|
||||
let totalNounsProcessed = 0
|
||||
|
||||
while (hasMoreNouns) {
|
||||
let consecutiveEmptyBatches = 0
|
||||
const MAX_ITERATIONS = 10000 // Safety limit to prevent infinite loops
|
||||
let iterations = 0
|
||||
|
||||
while (hasMoreNouns && iterations < MAX_ITERATIONS) {
|
||||
iterations++
|
||||
const result = await this.storage.getNouns({
|
||||
pagination: { offset: nounOffset, limit: nounLimit }
|
||||
})
|
||||
|
||||
|
||||
// CRITICAL SAFETY CHECK: Prevent infinite loop on empty results
|
||||
if (result.items.length === 0) {
|
||||
consecutiveEmptyBatches++
|
||||
if (consecutiveEmptyBatches >= 3) {
|
||||
prodLog.warn('⚠️ Breaking metadata rebuild loop: received 3 consecutive empty batches')
|
||||
break
|
||||
}
|
||||
// If hasMore is true but items are empty, it's likely a bug
|
||||
if (result.hasMore) {
|
||||
prodLog.warn(`⚠️ Storage returned empty items but hasMore=true at offset ${nounOffset}`)
|
||||
hasMoreNouns = false // Force exit
|
||||
break
|
||||
}
|
||||
} else {
|
||||
consecutiveEmptyBatches = 0 // Reset counter on non-empty batch
|
||||
}
|
||||
|
||||
// CRITICAL FIX: Use batch metadata reading to prevent socket exhaustion
|
||||
const nounIds = result.items.map(noun => noun.id)
|
||||
|
||||
|
|
@ -1581,12 +1602,32 @@ export class MetadataIndexManager {
|
|||
const verbLimit = 25 // Even smaller batches during initialization to prevent socket exhaustion
|
||||
let hasMoreVerbs = true
|
||||
let totalVerbsProcessed = 0
|
||||
|
||||
while (hasMoreVerbs) {
|
||||
let consecutiveEmptyVerbBatches = 0
|
||||
let verbIterations = 0
|
||||
|
||||
while (hasMoreVerbs && verbIterations < MAX_ITERATIONS) {
|
||||
verbIterations++
|
||||
const result = await this.storage.getVerbs({
|
||||
pagination: { offset: verbOffset, limit: verbLimit }
|
||||
})
|
||||
|
||||
|
||||
// CRITICAL SAFETY CHECK: Prevent infinite loop on empty results
|
||||
if (result.items.length === 0) {
|
||||
consecutiveEmptyVerbBatches++
|
||||
if (consecutiveEmptyVerbBatches >= 3) {
|
||||
prodLog.warn('⚠️ Breaking verb metadata rebuild loop: received 3 consecutive empty batches')
|
||||
break
|
||||
}
|
||||
// If hasMore is true but items are empty, it's likely a bug
|
||||
if (result.hasMore) {
|
||||
prodLog.warn(`⚠️ Storage returned empty verb items but hasMore=true at offset ${verbOffset}`)
|
||||
hasMoreVerbs = false // Force exit
|
||||
break
|
||||
}
|
||||
} else {
|
||||
consecutiveEmptyVerbBatches = 0 // Reset counter on non-empty batch
|
||||
}
|
||||
|
||||
// CRITICAL FIX: Use batch verb metadata reading to prevent socket exhaustion
|
||||
const verbIds = result.items.map(verb => verb.id)
|
||||
|
||||
|
|
@ -1647,11 +1688,19 @@ export class MetadataIndexManager {
|
|||
await this.yieldToEventLoop()
|
||||
}
|
||||
|
||||
// Check if we hit iteration limits
|
||||
if (iterations >= MAX_ITERATIONS) {
|
||||
prodLog.error(`❌ Metadata noun rebuild hit maximum iteration limit (${MAX_ITERATIONS}). This indicates a bug in storage pagination.`)
|
||||
}
|
||||
if (verbIterations >= MAX_ITERATIONS) {
|
||||
prodLog.error(`❌ Metadata verb rebuild hit maximum iteration limit (${MAX_ITERATIONS}). This indicates a bug in storage pagination.`)
|
||||
}
|
||||
|
||||
// Flush to storage with final yield
|
||||
prodLog.debug('💾 Flushing metadata index to storage...')
|
||||
await this.flush()
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
|
||||
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
|
||||
prodLog.info(`🎯 Initial indexing may show minor socket timeouts - this is expected and doesn't affect data processing`)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue