soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: exclude __words__ keyword index from corruption detection and getStats()

Browse source 
The __words__ keyword index stores 50-5000 entries per entity (one per
word), which inflated avg entries/entity well above the corruption
threshold of 100. This caused:

1. validateConsistency() to falsely detect corruption on every startup,
   triggering unnecessary clearAllIndexData() + rebuild() cycles
2. getStats() to log false "Metadata index may be corrupted" warnings
   and report inflated totalEntries/totalIds stats

Both methods now skip __words__ when counting, so stats and health
checks reflect metadata fields only (noun, type, createdAt, etc.).
Keyword search is unaffected since the __words__ field index itself
is not modified.
This commit is contained in:
David Snelling 2026-01-27 15:38:21 -08:00
parent 32dbdcec61
commit 364360d447
128 changed files with 5637 additions and 5682 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/api/ConfigAPI.ts
View file

@ -60,7 +60,7 @@ export class ConfigAPI {
// Store in cache
this.configCache.set(key, entry)
// v4.0.0: Persist to storage as NounMetadata
// Persist to storage as NounMetadata
const configId = this.CONFIG_NOUN_PREFIX + key
const nounMetadata = {
noun: 'config' as any,
@ -94,7 +94,7 @@ export class ConfigAPI {
return defaultValue
}
// v4.0.0: Extract ConfigEntry from NounMetadata
// Extract ConfigEntry from NounMetadata
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
: ((metadata.createdAt as unknown as number) || Date.now())
@ -140,7 +140,7 @@ export class ConfigAPI {
// Remove from cache
this.configCache.delete(key)
// v4.0.0: Remove from storage
// Remove from storage
const configId = this.CONFIG_NOUN_PREFIX + key
await this.storage.deleteNounMetadata(configId)
}
@ -149,7 +149,7 @@ export class ConfigAPI {
* List all configuration keys
*/
async list(): Promise<string[]> {
// v4.0.0: Get all nouns and filter for config entries
// Get all nouns and filter for config entries
const result = await this.storage.getNouns({ pagination: { limit: 10000 } })
const configKeys: string[] = []
@ -213,7 +213,7 @@ export class ConfigAPI {
for (const [key, entry] of Object.entries(config)) {
this.configCache.set(key, entry)
const configId = this.CONFIG_NOUN_PREFIX + key
// v4.0.0: Convert ConfigEntry to NounMetadata
// Convert ConfigEntry to NounMetadata
const nounMetadata = {
noun: 'config' as any,
...entry,
@ -240,7 +240,7 @@ export class ConfigAPI {
return null
}
// v4.0.0: Extract ConfigEntry from NounMetadata
// Extract ConfigEntry from NounMetadata
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
: ((metadata.createdAt as unknown as number) || Date.now())

2
src/api/DataAPI.ts
View file

@ -210,7 +210,7 @@ export class DataAPI {
this.validateImportItem(mapped)
}
// v4.0.0: Save entity - separate vector and metadata
// Save entity - separate vector and metadata
const id = mapped.id || this.generateId()
const noun: HNSWNoun = {
id,

10
src/augmentations/brainyAugmentation.ts
View file

@ -375,7 +375,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
}
/**
* Get CommitLog for temporal features (v5.0.0+)
* Get CommitLog for temporal features
*
* Provides access to commit history for time-travel queries, audit trails,
* and branch management. Available after initialize() is called.
@ -414,7 +414,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
}
/**
* Get BlobStorage for content-addressable storage (v5.0.0+)
* Get BlobStorage for content-addressable storage
*
* Provides access to the underlying blob storage system for storing
* and retrieving content-addressed data. Available after initialize() is called.
@ -454,7 +454,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
}
/**
* Get RefManager for branch/ref management (v5.0.0+)
* Get RefManager for branch/ref management
*
* Provides access to the reference manager for creating, updating,
* and managing Git-style branches and refs. Available after initialize() is called.
@ -496,7 +496,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
}
/**
* Get current branch name (v5.0.0+)
* Get current branch name
*
* Convenience helper for getting the current branch from the Brainy instance.
* Available after initialize() is called.
@ -525,7 +525,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
if (typeof brain.getCurrentBranch !== 'function') {
throw new Error(
`${this.name}: getCurrentBranch() not available on Brainy instance. ` +
`This method requires Brainy v5.0.0+.`
`This method requires Brainy with VFS support.`
)
}

2
src/augmentations/intelligentImport/FormatHandlerRegistry.ts
View file

@ -1,5 +1,5 @@
/**
* Format Handler Registry (v5.2.0)
* Format Handler Registry
*
* Central registry for format handlers with:
* - MIME type-based routing

2
src/augmentations/intelligentImport/IntelligentImportAugmentation.ts
View file

@ -35,7 +35,7 @@ export class IntelligentImportAugmentation extends BaseAugmentation {
enableCSV: true,
enableExcel: true,
enablePDF: true,
enableImage: true, // v5.2.0: Image handler enabled by default
enableImage: true, // Image handler enabled by default
maxFileSize: 100 * 1024 * 1024, // 100MB default
enableCache: true,
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours

12
src/augmentations/intelligentImport/handlers/csvHandler.ts
View file

@ -40,7 +40,7 @@ export class CSVHandler extends BaseFormatHandler {
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
const totalBytes = buffer.length
// v4.5.0: Report total bytes for progress tracking
// Report total bytes for progress tracking
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(0)
}
@ -55,7 +55,7 @@ export class CSVHandler extends BaseFormatHandler {
// Detect delimiter if not specified
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
// v4.5.0: Report progress - parsing started
// Report progress - parsing started
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
}
@ -75,7 +75,7 @@ export class CSVHandler extends BaseFormatHandler {
cast: false // We'll do type inference ourselves
})
// v4.5.0: Report bytes processed (entire file parsed)
// Report bytes processed (entire file parsed)
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(totalBytes)
}
@ -83,7 +83,7 @@ export class CSVHandler extends BaseFormatHandler {
// Convert to array of objects
const data = Array.isArray(records) ? records : [records]
// v4.5.0: Report data extraction progress
// Report data extraction progress
if (progressHooks?.onDataExtracted) {
progressHooks.onDataExtracted(data.length, data.length)
}
@ -101,7 +101,7 @@ export class CSVHandler extends BaseFormatHandler {
converted[key] = this.convertValue(value, types[key] || 'string')
}
// v4.5.0: Report progress every 1000 rows
// Report progress every 1000 rows
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
}
@ -111,7 +111,7 @@ export class CSVHandler extends BaseFormatHandler {
const processingTime = Date.now() - startTime
// v4.5.0: Final progress update
// Final progress update
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
}

18
src/augmentations/intelligentImport/handlers/excelHandler.ts
View file

@ -27,7 +27,7 @@ export class ExcelHandler extends BaseFormatHandler {
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
const totalBytes = buffer.length
// v4.5.0: Report start
// Report start
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(0)
}
@ -47,7 +47,7 @@ export class ExcelHandler extends BaseFormatHandler {
// Determine which sheets to process
const sheetsToProcess = this.getSheetsToProcess(workbook, options)
// v4.5.0: Report workbook loaded
// Report workbook loaded
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`)
}
@ -59,7 +59,7 @@ export class ExcelHandler extends BaseFormatHandler {
for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) {
const sheetName = sheetsToProcess[sheetIndex]
// v4.5.0: Report current sheet
// Report current sheet
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(
`Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})`
@ -117,19 +117,19 @@ export class ExcelHandler extends BaseFormatHandler {
headers
}
// v4.5.0: Estimate bytes processed (sheets are sequential)
// Estimate bytes processed (sheets are sequential)
const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes)
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(bytesProcessed)
}
// v4.5.0: Report extraction progress
// Report extraction progress
if (progressHooks?.onDataExtracted) {
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
}
}
// v4.5.0: Report data extraction complete
// Report data extraction complete
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`)
}
@ -152,7 +152,7 @@ export class ExcelHandler extends BaseFormatHandler {
}
}
// v4.5.0: Report progress every 1000 rows (avoid spam)
// Report progress every 1000 rows (avoid spam)
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`)
}
@ -160,14 +160,14 @@ export class ExcelHandler extends BaseFormatHandler {
return converted
})
// v4.5.0: Final progress - all bytes processed
// Final progress - all bytes processed
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(totalBytes)
}
const processingTime = Date.now() - startTime
// v4.5.0: Report completion
// Report completion
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(
`Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows`

6
src/augmentations/intelligentImport/handlers/imageHandler.ts
View file

@ -1,5 +1,5 @@
/**
* Image Import Handler (v5.8.0 - Pure JavaScript)
* Image Import Handler
*
* Handles image files with:
* - EXIF metadata extraction (camera, GPS, timestamps) via exifr
@ -7,7 +7,7 @@
* - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG
*
* NO NATIVE DEPENDENCIES - Pure JavaScript implementation
* Replaces Sharp (v5.7.x) with lightweight pure-JS alternatives
* Replaces Sharp with lightweight pure-JS alternatives
*/
import { BaseFormatHandler } from './base.js'
@ -96,7 +96,7 @@ export interface ImageHandlerOptions extends FormatHandlerOptions {
* Enables developers to import images into the knowledge graph with
* full metadata extraction.
*
* v5.8.0: Pure JavaScript implementation (no native dependencies)
* Pure JavaScript implementation (no native dependencies)
*/
export class ImageHandler extends BaseFormatHandler {
readonly format = 'image'

14
src/augmentations/intelligentImport/handlers/pdfHandler.ts
View file

@ -52,7 +52,7 @@ export class PDFHandler extends BaseFormatHandler {
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
const totalBytes = buffer.length
// v4.5.0: Report start
// Report start
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(0)
}
@ -74,7 +74,7 @@ export class PDFHandler extends BaseFormatHandler {
const metadata = await pdfDoc.getMetadata()
const numPages = pdfDoc.numPages
// v4.5.0: Report document loaded
// Report document loaded
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
}
@ -85,7 +85,7 @@ export class PDFHandler extends BaseFormatHandler {
let detectedTables = 0
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
// v4.5.0: Report current page
// Report current page
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
}
@ -131,19 +131,19 @@ export class PDFHandler extends BaseFormatHandler {
}
}
// v4.5.0: Estimate bytes processed (pages are sequential)
// Estimate bytes processed (pages are sequential)
const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(bytesProcessed)
}
// v4.5.0: Report extraction progress
// Report extraction progress
if (progressHooks?.onDataExtracted) {
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
}
}
// v4.5.0: Final progress - all bytes processed
// Final progress - all bytes processed
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(totalBytes)
}
@ -153,7 +153,7 @@ export class PDFHandler extends BaseFormatHandler {
const processingTime = Date.now() - startTime
// v4.5.0: Report completion
// Report completion
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(
`PDF complete: ${numPages} pages, ${allData.length} items extracted`

4
src/augmentations/intelligentImport/index.ts
View file

@ -18,14 +18,14 @@ export { ExcelHandler } from './handlers/excelHandler.js'
export { PDFHandler } from './handlers/pdfHandler.js'
export { ImageHandler } from './handlers/imageHandler.js'
// Format Handler Registry (v5.2.0)
// Format Handler Registry
export {
FormatHandlerRegistry,
globalHandlerRegistry
} from './FormatHandlerRegistry.js'
export type { HandlerRegistration } from './FormatHandlerRegistry.js'
// Image Handler Types (v5.2.0)
// Image Handler Types
export type {
ImageMetadata,
EXIFData,

8
src/augmentations/intelligentImport/types.ts
View file

@ -88,13 +88,13 @@ export interface FormatHandlerOptions {
streaming?: boolean
/**
* Progress hooks (v4.5.0)
* Progress hooks
* Handlers call these to report progress during processing
*/
progressHooks?: FormatHandlerProgressHooks
/**
* Total file size in bytes (v4.5.0)
* Total file size in bytes
* Used for progress percentage calculation
*/
totalBytes?: number
@ -168,7 +168,7 @@ export interface IntelligentImportConfig {
/** Enable PDF handler */
enablePDF: boolean
/** Enable Image handler (v5.2.0) */
/** Enable Image handler */
enableImage: boolean
/** Default options for CSV */
@ -180,7 +180,7 @@ export interface IntelligentImportConfig {
/** Default options for PDF */
pdfDefaults?: Partial<FormatHandlerOptions>
/** Default options for Image (v5.2.0) */
/** Default options for Image */
imageDefaults?: Partial<FormatHandlerOptions>
/** Maximum file size to process (bytes) */

2
src/augmentations/typeMatching/brainyTypes.ts
View file

@ -168,7 +168,7 @@ export interface TypeMatchResult {
/**
* BrainyTypes - Intelligent type detection for nouns and verbs
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings
* PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings
* Type embeddings are loaded instantly; only input objects are embedded at runtime
*/
export class BrainyTypes {

4
src/cli/commands/core.ts
View file

@ -172,7 +172,7 @@ export const coreCommands = {
metadata
}
// v4.3.x: Add confidence and weight if provided
// Add confidence and weight if provided
if (options.confidence) {
addParams.confidence = parseFloat(options.confidence)
}
@ -347,7 +347,7 @@ export const coreCommands = {
searchParams.includeRelations = true
}
// v4.7.0: VFS is now part of the knowledge graph (included by default)
// VFS is now part of the knowledge graph (included by default)
// Users can exclude VFS with --where vfsType exists:false if needed
// Triple Intelligence Fusion - custom weighting

4
src/cli/commands/cow.ts
View file

@ -312,7 +312,7 @@ ${chalk.cyan('Fork Statistics:')}
},
/**
* Migrate from v4.x to v5.0.0 (one-time)
* Migrate storage format (one-time)
*/
async migrate(options: MigrateOptions) {
let spinner: any = null
@ -388,7 +388,7 @@ ${chalk.cyan('Fork Statistics:')}
await oldBrain.init()
// Create new brain (v5.0.0)
// Create new brain
const newBrain = new Brainy({
storage: {
type: 'filesystem',

4
src/cli/commands/neural.ts
View file

@ -117,7 +117,7 @@ export const neuralCommand = {
await handleNeighborsCommand(neural, argv)
break
case 'path':
console.log(chalk.yellow('\n⚠️ Semantic path finding coming in v3.21.0'))
console.log(chalk.yellow('\n⚠️ Semantic path finding coming soon'))
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
console.log(chalk.dim('Use "neighbors" and "hierarchy" commands to explore connections'))
break
@ -148,7 +148,7 @@ async function promptForAction(): Promise<string> {
{ name: '🎯 Find semantic clusters', value: 'clusters' },
{ name: '🌳 Show item hierarchy', value: 'hierarchy' },
{ name: '🕸️ Find semantic neighbors', value: 'neighbors' },
{ name: '🛣️ Find semantic path between items (v3.21.0)', value: 'path', disabled: true },
{ name: '🛣️ Find semantic path between items (coming soon)', value: 'path', disabled: true },
{ name: '🚨 Detect outliers', value: 'outliers' },
{ name: '📊 Generate visualization data', value: 'visualize' }
]

2
src/cli/commands/storage.ts
View file

@ -1,5 +1,5 @@
/**
* 💾 Storage Management Commands - v4.0.0
* 💾 Storage Management Commands
*
* Modern interactive CLI for storage lifecycle, cost optimization, and management
*/

2
src/cli/commands/utility.ts
View file

@ -133,7 +133,7 @@ export const utilityCommands = {
// removeOrphans and rebuildIndex would require new Brainy APIs
if (options.removeOrphans || options.rebuildIndex) {
spinner.warn('Advanced cleanup options not yet implemented')
console.log(chalk.yellow('\n⚠️ Advanced cleanup features coming in v3.21.0:'))
console.log(chalk.yellow('\n⚠️ Advanced cleanup features coming soon:'))
console.log(chalk.dim(' • --remove-orphans: Remove disconnected items'))
console.log(chalk.dim(' • --rebuild-index: Rebuild vector index'))
console.log(chalk.dim('\nUse "brainy clean" without options to clear the database'))

6
src/cli/commands/vfs.ts
View file

@ -21,7 +21,7 @@ let brainyInstance: Brainy | null = null
const getBrainy = async (): Promise<Brainy> => {
if (!brainyInstance) {
brainyInstance = new Brainy()
await brainyInstance.init() // v5.0.1: Initialize brain (VFS auto-initialized here!)
await brainyInstance.init() // Initialize brain (VFS auto-initialized here!)
}
return brainyInstance
}
@ -52,8 +52,8 @@ export const vfsCommands = {
const spinner = ora('Reading file...').start()
try {
const brain = await getBrainy() // v5.0.1: Await async getBrainy
// v5.0.1: VFS auto-initialized, no need for vfs.init()
const brain = await getBrainy() // Await async getBrainy
// VFS auto-initialized, no need for vfs.init()
const buffer = await brain.vfs.readFile(path, {
encoding: options.encoding as any
})

10
src/cli/index.ts
View file

@ -68,7 +68,7 @@ ${chalk.cyan('Examples:')}
$ brainy vfs search "React components"
$ brainy vfs similar /code/Button.tsx
${chalk.dim('# Storage management (v4.0.0)')}
${chalk.dim('# Storage management')}
$ brainy storage status --quota
$ brainy storage lifecycle set ${chalk.dim('# Interactive mode')}
$ brainy storage cost-estimate
@ -217,11 +217,11 @@ program
program
.command('path <from> <to>')
.description('Find semantic path between items (v3.21.0)')
.description('Find semantic path between items')
.option('--steps', 'Show step-by-step path')
.option('--max-hops <number>', 'Maximum path length', '5')
.action(() => {
console.log(chalk.yellow('\n⚠️ Semantic path finding coming in v3.21.0'))
console.log(chalk.yellow('\n⚠️ Semantic path finding coming soon'))
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
console.log(chalk.dim('Use "brainy neighbors" and "brainy hierarchy" to explore connections'))
})
@ -446,7 +446,7 @@ program
vfsCommands.tree(path, options)
})
// ===== Storage Management Commands (v4.0.0) =====
// ===== Storage Management Commands =====
program
.command('storage')
@ -610,7 +610,7 @@ program
.option('--iterations <n>', 'Number of iterations', '100')
.action(utilityCommands.benchmark)
// ===== COW Commands (v5.0.0) - Instant Fork & Branching =====
// ===== COW Commands - Instant Fork & Branching =====
program
.command('fork [name]')

2
src/cli/interactive.ts
View file

@ -625,7 +625,7 @@ export async function promptCommand(): Promise<string> {
*/
export async function startInteractiveMode() {
console.log(chalk.cyan('\n🧠 Brainy Interactive Mode\n'))
console.log(chalk.yellow('Interactive REPL mode coming in v3.20.0\n'))
console.log(chalk.yellow('Interactive REPL mode coming soon\n'))
console.log(chalk.dim('Use specific commands for now: brainy add, brainy search, etc.'))
process.exit(0)
}

68
src/coreTypes.ts
View file

@ -57,7 +57,7 @@ export type DistanceFunction = (a: Vector, b: Vector) => number
/**
* Embedding function for converting data to vectors
* v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
* Now properly typed - accepts string, string array (batch), or object, no `any`
*/
export type EmbeddingFunction = (data: string | string[] | Record<string, unknown>) => Promise<Vector>
@ -72,7 +72,7 @@ export interface EmbeddingModel {
/**
* Embed data into a vector
* v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
* Now properly typed - accepts string, string array (batch), or object, no `any`
*/
embed(data: string | string[] | Record<string, unknown>): Promise<Vector>
@ -83,9 +83,9 @@ export interface EmbeddingModel {
}
/**
* HNSW graph noun - Pure vector structure (v4.0.0)
* HNSW graph noun - Pure vector structure
*
* v4.0.0 BREAKING CHANGE: metadata field removed
* metadata field removed
* - Stores ONLY vector data for optimal memory usage
* - Metadata stored separately and combined on retrieval
* - 25% memory reduction @ 1B scale (no in-memory metadata)
@ -100,15 +100,15 @@ export interface HNSWNoun {
}
/**
* Lightweight verb for HNSW index storage - Core relational structure (v4.0.0)
* Lightweight verb for HNSW index storage - Core relational structure
*
* Core fields (v3.50.1+): verb/sourceId/targetId are first-class fields
* Core fields: verb/sourceId/targetId are first-class fields
* These are NOT metadata - they're the essence of what a verb IS:
* - verb: The relationship type (creates, contains, etc.) - needed for routing & display
* - sourceId: What entity this verb connects FROM - needed for graph traversal
* - targetId: What entity this verb connects TO - needed for graph traversal
*
* v4.0.0 BREAKING CHANGE: metadata field removed
* metadata field removed
* - Stores ONLY vector + core relational data
* - User metadata (weight, custom fields) stored separately
* - 10x faster metadata-only updates (skip HNSW rebuild)
@ -134,9 +134,9 @@ export interface HNSWVerb {
}
/**
* Noun metadata structure (v4.8.0)
* Noun metadata structure
*
* v4.8.0 BREAKING CHANGE: Now contains ONLY custom user-defined fields
* Now contains ONLY custom user-defined fields
* - Standard fields (confidence, weight, timestamps, etc.) moved to top-level in HNSWNounWithMetadata
* - This interface represents custom metadata stored separately from vector data
* - Storage format unchanged (backward compatible at storage layer)
@ -162,9 +162,9 @@ export interface NounMetadata {
}
/**
* Verb metadata structure (v4.8.0)
* Verb metadata structure
*
* v4.8.0 BREAKING CHANGE: Now contains ONLY custom user-defined fields
* Now contains ONLY custom user-defined fields
* - Standard fields (weight, confidence, timestamps, etc.) moved to top-level in HNSWVerbWithMetadata
* - This interface represents custom metadata stored separately from vector + core relational data
* - Storage format unchanged (backward compatible at storage layer)
@ -190,9 +190,9 @@ export interface VerbMetadata {
}
/**
* Combined noun structure for transport/API boundaries (v4.8.0)
* Combined noun structure for transport/API boundaries
*
* v4.8.0 BREAKING CHANGE: Standard fields moved to top-level
* Standard fields moved to top-level
* - ALL standard fields (confidence, weight, timestamps, etc.) are now at top-level
* - metadata contains ONLY custom user-defined fields
* - Provides clean, predictable API: entity.confidence always works
@ -230,9 +230,9 @@ export interface HNSWNounWithMetadata {
}
/**
* Combined verb structure for transport/API boundaries (v4.8.0)
* Combined verb structure for transport/API boundaries
*
* v4.8.0 BREAKING CHANGE: Standard fields moved to top-level
* Standard fields moved to top-level
* - ALL standard fields (weight, confidence, timestamps, etc.) are now at top-level
* - metadata contains ONLY custom user-defined fields
* - Provides clean, predictable API: verb.weight always works
@ -308,7 +308,7 @@ export interface HNSWConfig {
efSearch: number // Size of the dynamic candidate list during search
ml: number // Maximum level
useDiskBasedIndex?: boolean // Whether to use disk-based index
maxConcurrentNeighborWrites?: number // Maximum concurrent neighbor updates during insert (v4.10.0+). Default: unlimited (full concurrency)
maxConcurrentNeighborWrites?: number // Maximum concurrent neighbor updates during insert. Default: unlimited (full concurrency)
}
/**
@ -548,7 +548,7 @@ export interface StatisticsData {
}
/**
* Change record for getChangesSince (v4.0.0)
* Change record for getChangesSince
* Replaces `any[]` with properly typed structure
*/
export interface Change {
@ -563,27 +563,27 @@ export interface StorageAdapter {
init(): Promise<void>
/**
* Save noun - Pure HNSW vector data only (v4.0.0)
* Save noun - Pure HNSW vector data only
* @param noun Pure HNSW vector data (no metadata)
* Note: Use saveNounMetadata() to save metadata separately
*/
saveNoun(noun: HNSWNoun): Promise<void>
/**
* Save noun metadata separately (v4.0.0)
* Save noun metadata separately
* @param id Noun ID
* @param metadata Noun metadata
*/
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
/**
* Delete noun metadata (v4.0.0)
* Delete noun metadata
* @param id Noun ID
*/
deleteNounMetadata(id: string): Promise<void>
/**
* Get noun with metadata combined (v4.0.0)
* Get noun with metadata combined
* @returns Combined HNSWNounWithMetadata or null
*/
getNoun(id: string): Promise<HNSWNounWithMetadata | null>
@ -622,14 +622,14 @@ export interface StorageAdapter {
deleteNoun(id: string): Promise<void>
/**
* Save verb - Pure HNSW verb with core fields only (v4.0.0)
* Save verb - Pure HNSW verb with core fields only
* @param verb Pure HNSW verb data (vector + core fields, no user metadata)
* Note: Use saveVerbMetadata() to save metadata separately
*/
saveVerb(verb: HNSWVerb): Promise<void>
/**
* Get verb with metadata combined (v4.0.0)
* Get verb with metadata combined
* @returns Combined HNSWVerbWithMetadata or null
*/
getVerb(id: string): Promise<HNSWVerbWithMetadata | null>
@ -686,14 +686,14 @@ export interface StorageAdapter {
deleteVerb(id: string): Promise<void>
/**
* Save metadata (v4.0.0: now typed)
* Save metadata
* @param id Entity ID
* @param metadata Typed noun metadata
*/
saveMetadata(id: string, metadata: NounMetadata): Promise<void>
/**
* Get metadata (v4.0.0: now typed)
* Get metadata
* @param id Entity ID
* @returns Typed noun metadata or null
*/
@ -702,26 +702,26 @@ export interface StorageAdapter {
/**
* Get multiple metadata objects in batches (prevents socket exhaustion)
* @param ids Array of IDs to get metadata for
* @returns Promise that resolves to a Map of id -> metadata (v4.0.0: typed)
* @returns Promise that resolves to a Map of id -> metadata
*/
getMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>
/**
* Get noun metadata from storage (v4.0.0: now typed)
* Get noun metadata from storage
* @param id The ID of the noun
* @returns Promise that resolves to the metadata or null if not found
*/
getNounMetadata(id: string): Promise<NounMetadata | null>
/**
* Batch get multiple nouns with vectors (v6.2.0 - N+1 fix)
* Batch get multiple nouns with vectors
* @param ids Array of noun IDs to fetch
* @returns Map of id → HNSWNounWithMetadata (only successful reads included)
*/
getNounBatch?(ids: string[]): Promise<Map<string, HNSWNounWithMetadata>>
/**
* Save verb metadata to storage (v4.0.0: now typed)
* Save verb metadata to storage
* @param id The ID of the verb
* @param metadata The metadata to save
* @returns Promise that resolves when the metadata is saved
@ -729,14 +729,14 @@ export interface StorageAdapter {
saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void>
/**
* Get verb metadata from storage (v4.0.0: now typed)
* Get verb metadata from storage
* @param id The ID of the verb
* @returns Promise that resolves to the metadata or null if not found
*/
getVerbMetadata(id: string): Promise<VerbMetadata | null>
/**
* Batch get multiple verbs (v6.2.0 - N+1 fix)
* Batch get multiple verbs
* @param ids Array of verb IDs to fetch
* @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
*/
@ -745,7 +745,7 @@ export interface StorageAdapter {
clear(): Promise<void>
/**
* Batch delete multiple objects from storage (v4.0.0)
* Batch delete multiple objects from storage
* Efficient deletion of large numbers of entities using cloud provider batch APIs.
* Significantly faster and cheaper than individual deletes (up to 1000x speedup).
*
@ -853,7 +853,7 @@ export interface StorageAdapter {
flushStatisticsToStorage(): Promise<void>
/**
* Track field names from a JSON document (v4.0.0: now typed)
* Track field names from a JSON document
* @param jsonDocument The JSON document to extract field names from
* @param service The service that inserted the data
*/
@ -872,7 +872,7 @@ export interface StorageAdapter {
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>
/**
* Get changes since a specific timestamp (v4.0.0: now typed)
* Get changes since a specific timestamp
* @param timestamp The timestamp to get changes since
* @param limit Optional limit on the number of changes to return
* @returns Promise that resolves to an array of properly typed changes

4
src/embeddings/candle-wasm/src/lib.rs
View file

@ -26,7 +26,7 @@ use js_sys::{Array, Float32Array};
use tokenizers::Tokenizer;
use wasm_bindgen::prelude::*;
// v7.2.0: Model weights are NO LONGER embedded in WASM
// Model weights are NO LONGER embedded in WASM
//
// Previous design: 90MB WASM with model weights embedded via include_bytes!()
// Problem: WASM parsing/compilation took 139+ seconds on throttled CPU (Cloud Run)
@ -78,7 +78,7 @@ impl EmbeddingEngine {
/// Load the model and tokenizer from bytes
///
/// v7.2.0: This is now the ONLY way to initialize the engine.
/// This is now the ONLY way to initialize the engine.
/// Model weights are no longer embedded in WASM for faster initialization.
///
/// # Arguments

10
src/embeddings/wasm/CandleEmbeddingEngine.ts
View file

@ -5,7 +5,7 @@
* Pure Rust/WASM implementation for sentence embeddings.
* Works with Bun, Node.js, Bun --compile, and browsers.
*
* v7.2.0 Architecture (20x faster initialization):
* Architecture (20x faster initialization):
* - WASM file: ~2.4MB (inference code only)
* - Model files: ~88MB (loaded separately as raw bytes)
* - Init time: ~5-7 seconds (vs 139 seconds with embedded model)
@ -34,7 +34,7 @@ interface CandleWasmModule {
}
interface CandleEngineInstance {
// v7.2.0: load() is the only initialization method (no more embedded model)
// load() is the only initialization method (no more embedded model)
load(modelBytes: Uint8Array, tokenizerBytes: Uint8Array, configBytes: Uint8Array): void
is_ready(): boolean
embed(text: string): Float32Array
@ -54,7 +54,7 @@ let globalInitPromise: Promise<void> | null = null
* Uses the Candle ML framework (Rust/WASM) for inference.
* Supports all-MiniLM-L6-v2 with 384-dimensional embeddings.
*
* v7.2.0: Model weights are loaded separately from WASM for 20x faster init.
* Model weights are loaded separately from WASM for 20x faster init.
* For bun --compile deployments, both WASM and model files are automatically
* embedded in the binary - single file deployment still works.
*/
@ -104,7 +104,7 @@ export class CandleEmbeddingEngine {
/**
* Perform actual initialization
*
* v7.2.0: WASM and model files are loaded separately for 20x faster init.
* WASM and model files are loaded separately for 20x faster init.
* - WASM (~2.4MB): Compiles in ~3-5 seconds
* - Model (~88MB): Loads as raw bytes in ~1-2 seconds
* - Total: ~5-7 seconds (vs 139 seconds with embedded model)
@ -153,7 +153,7 @@ export class CandleEmbeddingEngine {
/**
* Load the WASM module
*
* v7.2.0: WASM is now only ~2.4MB (inference code only, no model weights).
* WASM is now only ~2.4MB (inference code only, no model weights).
* Uses wasmLoader.ts for cross-environment compatibility.
*/
private async loadWasmModule(): Promise<CandleWasmModule> {

2
src/embeddings/wasm/modelLoader.ts
View file

@ -1,7 +1,7 @@
/**
* Universal Model Loader for Candle Embeddings
*
* v7.2.0: Model weights are now loaded separately from WASM for faster initialization.
* Model weights are now loaded separately from WASM for faster initialization.
* This reduces WASM compilation time from 139 seconds to ~3-5 seconds.
*
* Loads model files from appropriate source based on environment:

37
src/graph/graphAdjacencyIndex.ts
View file

@ -45,7 +45,7 @@ export class GraphAdjacencyIndex {
private lsmTreeVerbsBySource: LSMTree // sourceId -> verbIds
private lsmTreeVerbsByTarget: LSMTree // targetId -> verbIds
// v5.7.0: ID-only tracking for billion-scale memory optimization
// ID-only tracking for billion-scale memory optimization
// Previous: Map<string, GraphVerb> stored full objects (128GB @ 1B verbs)
// Now: Set<string> stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction
private verbIdSet = new Set<string>()
@ -117,7 +117,7 @@ export class GraphAdjacencyIndex {
/**
* Initialize the graph index (lazy initialization)
* v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency
* Added defensive auto-rebuild check for verbIdSet consistency
*/
private async ensureInitialized(): Promise<void> {
if (this.initialized) {
@ -129,7 +129,7 @@ export class GraphAdjacencyIndex {
await this.lsmTreeVerbsBySource.init()
await this.lsmTreeVerbsByTarget.init()
// v6.3.0: Defensive check - if LSM-trees have data but verbIdSet is empty,
// Defensive check - if LSM-trees have data but verbIdSet is empty,
// the index was created without proper rebuild (shouldn't happen with singleton
// pattern but protects against edge cases and future refactoring)
const lsmTreeSize = this.lsmTreeVerbsBySource.size()
@ -151,7 +151,7 @@ export class GraphAdjacencyIndex {
}
/**
* Populate verbIdSet from storage without full rebuild (v6.3.0)
* Populate verbIdSet from storage without full rebuild
* Lighter weight than full rebuild - only loads verb IDs, not all verb data
* @private
*/
@ -192,7 +192,7 @@ export class GraphAdjacencyIndex {
* Core API - Neighbor lookup with LSM-tree storage
*
* O(log n) with bloom filter optimization (90% of queries skip disk I/O)
* v5.8.0: Added pagination support for high-degree nodes
* Added pagination support for high-degree nodes
*
* @param id Entity ID to get neighbors for
* @param optionsOrDirection Optional: direction string OR options object
@ -252,7 +252,7 @@ export class GraphAdjacencyIndex {
// Convert to array for pagination
let result = Array.from(neighbors)
// Apply pagination if requested (v5.8.0)
// Apply pagination if requested
if (options?.limit !== undefined || options?.offset !== undefined) {
const offset = options.offset || 0
const limit = options.limit !== undefined ? options.limit : result.length
@ -273,8 +273,8 @@ export class GraphAdjacencyIndex {
* Get verb IDs by source - Billion-scale optimization for getVerbsBySource
*
* O(log n) LSM-tree lookup with bloom filter optimization
* v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround)
* v5.8.0: Added pagination support for entities with many relationships
* Filters out deleted verb IDs (tombstone deletion workaround)
* Added pagination support for entities with many relationships
*
* @param sourceId Source entity ID
* @param options Optional configuration
@ -318,7 +318,7 @@ export class GraphAdjacencyIndex {
const allIds = verbIds || []
let result = allIds.filter(id => this.verbIdSet.has(id))
// Apply pagination if requested (v5.8.0)
// Apply pagination if requested
if (options?.limit !== undefined || options?.offset !== undefined) {
const offset = options.offset || 0
const limit = options.limit !== undefined ? options.limit : result.length
@ -332,8 +332,8 @@ export class GraphAdjacencyIndex {
* Get verb IDs by target - Billion-scale optimization for getVerbsByTarget
*
* O(log n) LSM-tree lookup with bloom filter optimization
* v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround)
* v5.8.0: Added pagination support for popular target entities
* Filters out deleted verb IDs (tombstone deletion workaround)
* Added pagination support for popular target entities
*
* @param targetId Target entity ID
* @param options Optional configuration
@ -377,7 +377,7 @@ export class GraphAdjacencyIndex {
const allIds = verbIds || []
let result = allIds.filter(id => this.verbIdSet.has(id))
// Apply pagination if requested (v5.8.0)
// Apply pagination if requested
if (options?.limit !== undefined || options?.offset !== undefined) {
const offset = options.offset || 0
const limit = options.limit !== undefined ? options.limit : result.length
@ -414,7 +414,7 @@ export class GraphAdjacencyIndex {
}
/**
* Batch get multiple verbs with caching (v6.2.0 - N+1 fix)
* Batch get multiple verbs with caching
*
* **Performance**: Eliminates N+1 pattern for verb loading
* - Current: N × getVerbCached() = N × 50ms on GCS = 250ms for 5 verbs
@ -433,7 +433,6 @@ export class GraphAdjacencyIndex {
* @param verbIds Array of verb IDs to fetch
* @returns Map of verbId → GraphVerb (only successful reads included)
*
* @since v6.2.0
*/
async getVerbsBatchCached(verbIds: string[]): Promise<Map<string, GraphVerb>> {
const results = new Map<string, GraphVerb>()
@ -514,7 +513,7 @@ export class GraphAdjacencyIndex {
const targetStats = this.lsmTreeTarget.getStats()
// Note: Exact unique node counts would require full LSM-tree scan
// v5.7.0: Using verbIdSet (ID-only tracking) for memory efficiency
// Using verbIdSet (ID-only tracking) for memory efficiency
const uniqueSourceNodes = this.verbIdSet.size
const uniqueTargetNodes = this.verbIdSet.size
const totalNodes = this.verbIdSet.size
@ -622,13 +621,13 @@ export class GraphAdjacencyIndex {
// Clear current index
this.verbIdSet.clear()
this.totalRelationshipsIndexed = 0
// v6.2.4: CRITICAL FIX - Clear relationship counts to prevent accumulation
// CRITICAL FIX - Clear relationship counts to prevent accumulation
this.relationshipCountsByType.clear()
// Note: LSM-trees will be recreated from storage via their own initialization
// Verb data will be loaded on-demand via UnifiedCache
// Adaptive loading strategy based on storage type (v4.2.4)
// Adaptive loading strategy based on storage type
const storageType = this.storage?.constructor.name || ''
const isLocalStorage =
storageType === 'FileSystemStorage' ||
@ -754,7 +753,7 @@ export class GraphAdjacencyIndex {
bytes += targetStats.memTableMemory
// Verb ID set (memory-efficient: IDs only, ~8 bytes per ID pointer)
// v5.7.0: Previous verbIndex Map stored full objects (128 bytes each = 128GB @ 1B verbs)
// Previous verbIndex Map stored full objects (128 bytes each = 128GB @ 1B verbs)
// Now: verbIdSet stores only IDs (~8 bytes each = ~100KB @ 1B verbs) = 1,280,000x reduction
bytes += this.verbIdSet.size * 8
@ -792,7 +791,7 @@ export class GraphAdjacencyIndex {
/**
* Flush LSM-tree MemTables to disk
* CRITICAL FIX (v3.43.2): Now public so it can be called from brain.flush()
* CRITICAL FIX: Now public so it can be called from brain.flush()
*/
async flush(): Promise<void> {
if (!this.initialized) {

61
src/hnsw/hnswIndex.ts
View file

@ -35,18 +35,18 @@ export class HNSWIndex {
private distanceFunction: DistanceFunction
private dimension: number | null = null
private useParallelization: boolean = true // Whether to use parallelization for performance-critical operations
private storage: BaseStorage | null = null // Storage adapter for HNSW persistence (v3.35.0+)
private storage: BaseStorage | null = null // Storage adapter for HNSW persistence
// Universal memory management (v3.36.0+)
// Universal memory management
private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes
// Always-adaptive caching (v3.36.0+) - no "mode" concept, system adapts automatically
// Always-adaptive caching - no "mode" concept, system adapts automatically
// COW (Copy-on-Write) support - v5.0.0
// COW (Copy-on-Write) support
private cowEnabled: boolean = false
private cowModifiedNodes: Set<string> = new Set()
private cowParent: HNSWIndex | null = null
// v6.2.8: Deferred HNSW persistence for cloud storage performance
// Deferred HNSW persistence for cloud storage performance
// In deferred mode, HNSW connections are only persisted on flush/close
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
private persistMode: 'immediate' | 'deferred' = 'immediate'
@ -86,7 +86,7 @@ export class HNSWIndex {
}
/**
* v6.2.8: Flush dirty HNSW data to storage
* Flush dirty HNSW data to storage
*
* In deferred persistence mode, HNSW connections are tracked as dirty but not
* immediately persisted. Call flush() to persist all pending changes.
@ -361,6 +361,19 @@ export class HNSWIndex {
this.entryPointId = id
this.maxLevel = nounLevel
this.nouns.set(id, noun)
// Persist system data for first noun (previously skipped)
if (this.storage && this.persistMode === 'immediate') {
await this.storage.saveHNSWSystem({
entryPointId: this.entryPointId,
maxLevel: this.maxLevel
}).catch(error => {
console.error('Failed to persist initial HNSW system data:', error)
})
} else if (this.persistMode === 'deferred') {
this.dirtySystem = true
}
return id
}
@ -438,7 +451,7 @@ export class HNSWIndex {
)
// Add bidirectional connections
// PERFORMANCE OPTIMIZATION (v4.10.0): Collect all neighbor updates for concurrent execution
// PERFORMANCE OPTIMIZATION: Collect all neighbor updates for concurrent execution
const neighborUpdates: Array<{
neighborId: string
promise: Promise<void>
@ -467,9 +480,9 @@ export class HNSWIndex {
await this.pruneConnections(neighbor, level)
}
// Persist updated neighbor HNSW data (v3.35.0+)
// Persist updated neighbor HNSW data
//
// v6.2.8: Deferred persistence mode for cloud storage performance
// Deferred persistence mode for cloud storage performance
// In deferred mode, we track dirty nodes instead of persisting immediately
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
if (this.storage && this.persistMode === 'immediate') {
@ -561,8 +574,8 @@ export class HNSWIndex {
this.highLevelNodes.get(nounLevel)!.add(id)
}
// Persist HNSW graph data to storage (v3.35.0+)
// v6.2.8: Respect persistMode setting
// Persist HNSW graph data to storage
// Respect persistMode setting
if (this.storage && this.persistMode === 'immediate') {
// IMMEDIATE MODE: Original behavior - persist new entity and system data
const connectionsObj: Record<string, string[]> = {}
@ -594,7 +607,7 @@ export class HNSWIndex {
}
/**
* O(1) entry point recovery using highLevelNodes index (v6.2.3).
* O(1) entry point recovery using highLevelNodes index.
* At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
* For tiny indexes with only level 0-1 nodes, any node works as entry point.
*/
@ -640,7 +653,7 @@ export class HNSWIndex {
}
// Start from the entry point
// If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3)
// If entry point is null but nouns exist, attempt O(1) recovery
if (!this.entryPointId && this.nouns.size > 0) {
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
if (recoveredId) {
@ -656,7 +669,7 @@ export class HNSWIndex {
let entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) {
// Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3)
// Entry point ID exists but noun was deleted - O(1) recovery
if (this.nouns.size > 0) {
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
if (recoveredId) {
@ -944,7 +957,7 @@ export class HNSWIndex {
/**
* Get vector safely (always uses adaptive caching via UnifiedCache)
*
* Production-grade adaptive caching (v3.36.0+):
* Production-grade adaptive caching:
* - Vector already loaded: Returns immediately (O(1))
* - Vector in cache: Loads from UnifiedCache (O(1) hash lookup)
* - Vector on disk: Loads from storage → UnifiedCache (O(disk))
@ -990,7 +1003,7 @@ export class HNSWIndex {
}
/**
* Get vector synchronously if available in memory (v3.36.0+)
* Get vector synchronously if available in memory
*
* Sync fast path optimization:
* - Vector in memory: Returns immediately (zero overhead)
@ -1048,7 +1061,7 @@ export class HNSWIndex {
}
/**
* Calculate distance with sync fast path (v3.36.0+)
* Calculate distance with sync fast path
*
* Eliminates async overhead when vectors are in memory:
* - Sync path: Vector in memory → returns number (zero overhead)
@ -1093,7 +1106,7 @@ export class HNSWIndex {
}
/**
* Rebuild HNSW index from persisted graph data (v3.35.0+)
* Rebuild HNSW index from persisted graph data
*
* This is a production-grade O(N) rebuild that restores the pre-computed graph structure
* from storage. Much faster than re-building which is O(N log N).
@ -1157,7 +1170,7 @@ export class HNSWIndex {
)
}
// Step 4: Adaptive loading strategy based on storage type (v4.2.4)
// Step 4: Adaptive loading strategy based on storage type
// FileSystem/Memory/OPFS: Load all at once (avoids repeated getAllShardedFiles() calls)
// Cloud (GCS/S3/R2): Use pagination (efficient native cloud APIs)
const storageType = this.storage?.constructor.name || ''
@ -1238,7 +1251,7 @@ export class HNSWIndex {
prodLog.info(`HNSW: Using cloud pagination strategy (${storageType})`)
let hasMore = true
let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop)
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
while (hasMore) {
// Fetch batch of nouns from storage (cast needed as method is not in base interface)
@ -1249,7 +1262,7 @@ export class HNSWIndex {
nextCursor?: string
} = await (this.storage as any).getNounsWithPagination({
limit: batchSize,
offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored)
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
})
// Set total count on first batch
@ -1307,11 +1320,11 @@ export class HNSWIndex {
// Check for more data
hasMore = result.hasMore
offset += batchSize // v5.7.11: Increment offset for next page
offset += batchSize // Increment offset for next page
}
}
// Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1))
// Step 5: CRITICAL - Recover entry point if missing)
// This ensures consistency even if getHNSWSystem() returned null
if (this.nouns.size > 0 && this.entryPointId === null) {
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup')
@ -1426,7 +1439,7 @@ export class HNSWIndex {
}
/**
* Get cache performance statistics for monitoring and diagnostics (v3.36.0+)
* Get cache performance statistics for monitoring and diagnostics
*
* Production-grade monitoring:
* - Adaptive caching strategy (preloading vs on-demand)

26
src/hnsw/typeAwareHNSWIndex.ts
View file

@ -118,7 +118,7 @@ export class TypeAwareHNSWIndex {
}
/**
* v6.2.8: Flush dirty HNSW data to storage for all type-specific indexes
* Flush dirty HNSW data to storage for all type-specific indexes
*
* In deferred persistence mode, HNSW connections are tracked as dirty but not
* immediately persisted. Call flush() to persist all pending changes across
@ -502,7 +502,7 @@ export class TypeAwareHNSWIndex {
// Load ALL nouns ONCE and route to correct type indexes
// This is O(N) instead of O(42*N) from the previous parallel approach
let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop)
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
let hasMore = true
let totalLoaded = 0
const loadedByType = new Map<NounType, number>()
@ -515,13 +515,13 @@ export class TypeAwareHNSWIndex {
totalCount?: number
} = await (this.storage as any).getNounsWithPagination({
limit: batchSize,
offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored)
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
})
// Route each noun to its type index
for (const nounData of result.items) {
try {
// v7.3.0: Use 'type' property from HNSWNounWithMetadata (not 'nounType')
// Use 'type' property from HNSWNounWithMetadata (not 'nounType')
// Previously accessed wrong property, causing N+1 metadata fetches
// getNounsWithPagination already includes type in response
const nounType = nounData.type
@ -578,7 +578,7 @@ export class TypeAwareHNSWIndex {
}
hasMore = result.hasMore
offset += batchSize // v5.7.11: Increment offset for next page
offset += batchSize // Increment offset for next page
// Progress logging
if (totalLoaded % 1000 === 0) {
@ -593,12 +593,26 @@ export class TypeAwareHNSWIndex {
let entryPointId: string | null = null
for (const [id, noun] of (index as any).nouns.entries()) {
if (noun.level > maxLevel) {
if (entryPointId === null || noun.level > maxLevel) {
maxLevel = noun.level
entryPointId = id
}
}
// Recovery: if still null after loop but nouns exist
if ((index as any).nouns.size > 0 && !entryPointId) {
const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1()
entryPointId = recoveredId
maxLevel = recoveredLevel
}
// Validation: if entry point doesn't exist in loaded nouns
if (entryPointId && !(index as any).nouns.has(entryPointId)) {
const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1()
entryPointId = recoveredId
maxLevel = recoveredLevel
}
;(index as any).entryPointId = entryPointId
;(index as any).maxLevel = maxLevel

8
src/import/FormatDetector.ts
View file

@ -111,7 +111,7 @@ export class FormatDetector {
return 'docx'
}
// Images (v5.2.0: ImageHandler support)
// Images (ImageHandler support)
if (mimeType.startsWith('image/')) {
return 'image'
}
@ -134,7 +134,7 @@ export class FormatDetector {
}
}
// YAML detection (v4.2.0)
// YAML detection
if (this.looksLikeYAML(trimmed)) {
return {
format: 'yaml',
@ -206,7 +206,7 @@ export class FormatDetector {
}
}
// Image formats (v5.2.0)
// Image formats
// JPEG: FF D8 FF
if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) {
return {
@ -384,7 +384,7 @@ export class FormatDetector {
/**
* Check if content looks like YAML
* v4.2.0: Added YAML detection
* Added YAML detection
*/
private looksLikeYAML(content: string): boolean {
const lines = content.split('\n').filter(l => l.trim()).slice(0, 20)

110
src/import/ImportCoordinator.ts
View file

@ -37,10 +37,10 @@ export interface ImportSource {
/** Optional filename hint */
filename?: string
/** HTTP headers for URL imports (v4.2.0) */
/** HTTP headers for URL imports */
headers?: Record<string, string>
/** Basic authentication for URL imports (v4.2.0) */
/** Basic authentication for URL imports */
auth?: {
username: string
password: string
@ -93,7 +93,7 @@ export interface ValidImportOptions {
/** Create relationships in knowledge graph */
createRelationships?: boolean
/** Create provenance relationships (document → entity) [v4.9.0] */
/** Create provenance relationships (document → entity) */
createProvenanceLinks?: boolean
/** Preserve source file in VFS */
@ -144,7 +144,7 @@ export interface ValidImportOptions {
customMetadata?: Record<string, any>
/**
* Progress callback for tracking import progress (v4.2.0+)
* Progress callback for tracking import progress
*
* **Streaming Architecture** (always enabled):
* - Indexes are flushed periodically during import (adaptive intervals)
@ -227,21 +227,21 @@ export type ImportOptions = ValidImportOptions & DeprecatedImportOptions
export interface ImportProgress {
stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete'
/** Phase of import - extraction or relationship building (v3.49.0) */
/** Phase of import - extraction or relationship building */
phase?: 'extraction' | 'relationships'
message: string
processed?: number
/** Alias for processed, used in relationship phase (v3.49.0) */
/** Alias for processed, used in relationship phase */
current?: number
total?: number
entities?: number
relationships?: number
/** Rows per second (v3.38.0) */
/** Rows per second */
throughput?: number
/** Estimated time remaining in ms (v3.38.0) */
/** Estimated time remaining in ms */
eta?: number
/**
* Whether data is queryable at this point (v4.2.0+)
* Whether data is queryable at this point
*
* When true, indexes have been flushed and queries will return up-to-date results.
* When false, data exists in storage but indexes may not be current (queries may be slower/incomplete).
@ -357,7 +357,7 @@ export class ImportCoordinator {
/**
* Import from any source with auto-detection
* v4.2.0: Now supports URL imports with authentication
* Now supports URL imports with authentication
*/
async import(
source: Buffer | string | object | ImportSource,
@ -365,10 +365,10 @@ export class ImportCoordinator {
): Promise<ImportResult> {
const startTime = Date.now()
// Validate options (v4.0.0+: Reject deprecated v3.x options)
// Validate options (Reject deprecated options)
this.validateOptions(options)
// Normalize source (v4.2.0: handles URL fetching)
// Normalize source (handles URL fetching)
const normalizedSource = await this.normalizeSource(source, options.format)
// Report detection stage
@ -387,10 +387,10 @@ export class ImportCoordinator {
}
// Set defaults early (needed for tracking context)
// CRITICAL FIX (v4.3.2): Spread options FIRST, then apply defaults
// CRITICAL FIX: Spread options FIRST, then apply defaults
// Previously: ...options at the end overwrote normalized defaults with undefined
// Now: Defaults properly override undefined values
// v4.4.0: Enable AI features by default for smarter imports
// Enable AI features by default for smarter imports
const opts = {
...options, // Spread first to get all options
vfsPath: options.vfsPath || `/imports/${Date.now()}`,
@ -399,13 +399,13 @@ export class ImportCoordinator {
createRelationships: options.createRelationships !== false,
preserveSource: options.preserveSource !== false,
enableDeduplication: options.enableDeduplication !== false,
enableNeuralExtraction: options.enableNeuralExtraction !== false, // v4.4.0: Default true
enableRelationshipInference: options.enableRelationshipInference !== false, // v4.4.0: Default true
enableNeuralExtraction: options.enableNeuralExtraction !== false, // Default true
enableRelationshipInference: options.enableRelationshipInference !== false, // Default true
enableConceptExtraction: options.enableConceptExtraction !== false, // Already defaults to true
deduplicationThreshold: options.deduplicationThreshold || 0.85
}
// Generate tracking context (v4.10.0+: Unified import/project tracking)
// Generate tracking context (Unified import/project tracking)
const importId = options.importId || uuidv4()
const projectId = options.projectId || this.deriveProjectId(opts.vfsPath)
const trackingContext: TrackingContext = {
@ -441,13 +441,13 @@ export class ImportCoordinator {
groupBy: opts.groupBy,
customGrouping: opts.customGrouping,
preserveSource: opts.preserveSource,
// v5.1.2: Fix sourceBuffer for file paths - type is 'path' not 'buffer' from normalizeSource()
// Fix sourceBuffer for file paths - type is 'path' not 'buffer' from normalizeSource()
sourceBuffer: Buffer.isBuffer(normalizedSource.data) ? normalizedSource.data as Buffer : undefined,
sourceFilename: normalizedSource.filename || `import.${detection.format}`,
createRelationshipFile: true,
createMetadataFile: true,
trackingContext, // v4.10.0: Pass tracking metadata to VFS
// v4.11.1: Pass progress callback for VFS creation updates
trackingContext, // Pass tracking metadata to VFS
// Pass progress callback for VFS creation updates
onProgress: (vfsProgress) => {
options.onProgress?.({
stage: 'storing-vfs',
@ -473,7 +473,7 @@ export class ImportCoordinator {
sourceFilename: normalizedSource.filename || `import.${detection.format}`,
format: detection.format
},
trackingContext // v4.10.0: Pass tracking metadata to graph creation
trackingContext // Pass tracking metadata to graph creation
)
// Report complete
@ -520,7 +520,7 @@ export class ImportCoordinator {
)
}
// CRITICAL FIX (v3.43.2): Auto-flush all indexes before returning
// CRITICAL FIX: Auto-flush all indexes before returning
// Ensures imported data survives server restarts
// Bug #5: Import data was only in memory, lost on restart
options.onProgress?.({
@ -535,7 +535,7 @@ export class ImportCoordinator {
/**
* Normalize source to ImportSource
* v4.2.0: Now async to support URL fetching
* Now async to support URL fetching
*/
private async normalizeSource(
source: Buffer | string | object | ImportSource,
@ -616,7 +616,7 @@ export class ImportCoordinator {
/**
* Fetch content from URL
* v4.2.0: Supports authentication and custom headers
* Supports authentication and custom headers
*/
private async fetchUrl(source: ImportSource): Promise<ImportSource> {
const url = typeof source.data === 'string' ? source.data : String(source.data)
@ -722,7 +722,7 @@ export class ImportCoordinator {
format: SupportedFormat,
options: ImportOptions
): Promise<any> {
// v5.2.0: Check if IntelligentImportAugmentation already extracted data
// Check if IntelligentImportAugmentation already extracted data
if ((options as any)._intelligentImport && (options as any)._extractedData) {
const extractedData = (options as any)._extractedData
// Convert extracted data to ExtractedRow format
@ -760,7 +760,7 @@ export class ImportCoordinator {
enableConceptExtraction: options.enableConceptExtraction !== false,
confidenceThreshold: options.confidenceThreshold || 0.6,
onProgress: (stats: any) => {
// Enhanced progress reporting (v3.38.0) with throughput and ETA
// Enhanced progress reporting with throughput and ETA
const message = stats.throughput
? `Extracting entities from ${format} (${stats.throughput} rows/sec, ETA: ${Math.round(stats.eta / 1000)}s)...`
: `Extracting entities from ${format}...`
@ -825,7 +825,7 @@ export class ImportCoordinator {
return await this.docxImporter.extract(docxBuffer, extractOptions)
case 'image':
// v5.2.0: Images are handled by IntelligentImportAugmentation
// Images are handled by IntelligentImportAugmentation
// If we reach here, augmentation didn't process it - return minimal result
const imageName = source.filename || 'image'
const imageId = `image-${Date.now()}`
@ -867,7 +867,7 @@ export class ImportCoordinator {
/**
* Create entities and relationships in knowledge graph
* v4.9.0: Added sourceInfo parameter for document entity creation
* Added sourceInfo parameter for document entity creation
*/
private async createGraphEntities(
extractionResult: any,
@ -877,7 +877,7 @@ export class ImportCoordinator {
sourceFilename: string
format: string
},
trackingContext?: TrackingContext // v4.10.0: Import/project tracking
trackingContext?: TrackingContext // Import/project tracking
): Promise<{
entities: Array<{ id: string; name: string; type: NounType; vfsPath?: string; metadata?: Record<string, any> }>
relationships: Array<{ id: string; from: string; to: string; type: VerbType }>
@ -891,7 +891,7 @@ export class ImportCoordinator {
let mergedCount = 0
let newCount = 0
// CRITICAL FIX (v4.3.2): Default to true when undefined
// CRITICAL FIX: Default to true when undefined
// Previously: if (!options.createEntities) treated undefined as false
// Now: Only skip when explicitly set to false
if (options.createEntities === false) {
@ -908,7 +908,7 @@ export class ImportCoordinator {
// Extract rows/sections/entities from result (unified across formats)
const rows = extractionResult.rows || extractionResult.sections || extractionResult.entities || []
// Progressive flush interval - adjusts based on current count (v4.2.0+)
// Progressive flush interval - adjusts based on current count
// Starts at 100, increases to 1000 at 1K entities, then 5000 at 10K
// This works for both known totals (files) and unknown totals (streaming APIs)
let currentFlushInterval = 100 // Start with frequent updates for better UX
@ -938,7 +938,7 @@ export class ImportCoordinator {
}
// ============================================
// v4.9.0: Create document entity for import source
// Create document entity for import source
// ============================================
let documentEntityId: string | null = null
let provenanceCount = 0
@ -957,7 +957,7 @@ export class ImportCoordinator {
vfsPath: vfsResult.rootPath,
totalRows: rows.length,
byType: this.countByType(rows),
// v4.10.0: Import tracking metadata
// Import tracking metadata
...(trackingContext && {
importIds: [trackingContext.importId],
projectId: trackingContext.projectId,
@ -973,7 +973,7 @@ export class ImportCoordinator {
}
// ============================================
// v4.11.0: Batch entity creation using addMany()
// Batch entity creation using addMany()
// Replaces entity-by-entity loop for 10-100x performance improvement on cloud storage
// ============================================
@ -1038,7 +1038,7 @@ export class ImportCoordinator {
name: entity.name,
type: entity.type,
vfsPath: vfsFile?.path,
metadata: entity.metadata // v5.2.0: Include metadata in return (for ImageHandler, etc)
metadata: entity.metadata // Include metadata in return (for ImageHandler, etc)
})
newCount++
}
@ -1090,7 +1090,7 @@ export class ImportCoordinator {
const importSource = vfsResult.rootPath
let entityId: string
// v5.7.0: No deduplication during import (12-24x speedup)
// No deduplication during import (12-24x speedup)
// Background deduplication runs 5 minutes after import completes
entityId = await this.brain.add({
data: entity.description || entity.name,
@ -1101,7 +1101,7 @@ export class ImportCoordinator {
confidence: entity.confidence,
vfsPath: vfsFile?.path,
importedFrom: 'import-coordinator',
// v4.10.0: Import tracking metadata
// Import tracking metadata
...(trackingContext && {
importId: trackingContext.importId, // Used for background dedup
importIds: [trackingContext.importId],
@ -1126,11 +1126,11 @@ export class ImportCoordinator {
name: entity.name,
type: entity.type,
vfsPath: vfsFile?.path,
metadata: entity.metadata // v5.2.0: Include metadata in return (for ImageHandler, etc)
metadata: entity.metadata // Include metadata in return (for ImageHandler, etc)
})
// ============================================
// v4.9.0: Create provenance relationship (document → entity)
// Create provenance relationship (document → entity)
// ============================================
if (documentEntityId && options.createProvenanceLinks !== false) {
await this.brain.relate({
@ -1144,7 +1144,7 @@ export class ImportCoordinator {
rowNumber: row.rowNumber,
extractedAt: Date.now(),
format: sourceInfo?.format,
// v4.10.0: Import tracking metadata
// Import tracking metadata
...(trackingContext && {
importIds: [trackingContext.importId],
projectId: trackingContext.projectId,
@ -1161,7 +1161,7 @@ export class ImportCoordinator {
if (options.createRelationships && row.relationships) {
for (const rel of row.relationships) {
try {
// CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop
// CRITICAL FIX: Prevent infinite placeholder creation loop
// Find or create target entity using EXACT matching only
let targetEntityId: string | undefined
@ -1195,7 +1195,7 @@ export class ImportCoordinator {
name: rel.to,
placeholder: true,
inferredFrom: entity.name,
// v4.10.0: Import tracking metadata
// Import tracking metadata
...(trackingContext && {
importIds: [trackingContext.importId],
projectId: trackingContext.projectId,
@ -1221,11 +1221,11 @@ export class ImportCoordinator {
from: entityId,
to: targetEntityId,
type: rel.type,
confidence: rel.confidence, // v4.2.0: Top-level field
weight: rel.weight || 1.0, // v4.2.0: Top-level field
confidence: rel.confidence, // Top-level field
weight: rel.weight || 1.0, // Top-level field
metadata: {
evidence: rel.evidence,
// v4.10.0: Import tracking metadata (will be merged in batch creation)
// Import tracking metadata (will be merged in batch creation)
...(trackingContext && {
importIds: [trackingContext.importId],
projectId: trackingContext.projectId,
@ -1242,7 +1242,7 @@ export class ImportCoordinator {
}
}
// Streaming import: Progressive flush with dynamic interval adjustment (v4.2.0+)
// Streaming import: Progressive flush with dynamic interval adjustment
entitiesSinceFlush++
if (entitiesSinceFlush >= currentFlushInterval) {
@ -1307,7 +1307,7 @@ export class ImportCoordinator {
}
// Batch create all relationships using brain.relateMany() for performance
// v4.9.0: Enhanced with type-based inference and semantic metadata
// Enhanced with type-based inference and semantic metadata
if (options.createRelationships && relationships.length > 0) {
try {
const relationshipParams = relationships.map(rel => {
@ -1331,7 +1331,7 @@ export class ImportCoordinator {
type: verbType, // Enhanced type
metadata: {
...((rel as any).metadata || {}),
relationshipType: 'semantic', // v4.9.0: Distinguish from VFS/provenance
relationshipType: 'semantic', // Distinguish from VFS/provenance
inferredType: verbType !== rel.type, // Track if type was enhanced
originalType: rel.type
}
@ -1369,7 +1369,7 @@ export class ImportCoordinator {
}
}
// v5.7.0: Schedule background deduplication (debounced 5 minutes)
// Schedule background deduplication (debounced 5 minutes)
if (trackingContext && trackingContext.importId) {
this.backgroundDedup.scheduleDedup(trackingContext.importId)
}
@ -1457,7 +1457,7 @@ export class ImportCoordinator {
}
}
// YAML: entities -> rows (v4.2.0)
// YAML: entities -> rows
if (format === 'yaml') {
const rows = result.entities.map((entity: any) => ({
entity,
@ -1477,7 +1477,7 @@ export class ImportCoordinator {
}
}
// DOCX: entities -> rows (v4.2.0)
// DOCX: entities -> rows
if (format === 'docx') {
const rows = result.entities.map((entity: any) => ({
entity,
@ -1502,7 +1502,7 @@ export class ImportCoordinator {
}
/**
* Validate options and reject deprecated v3.x options (v4.0.0+)
* Validate options and reject deprecated v3.x options
* Throws clear errors with migration guidance
*/
private validateOptions(options: any): void {
@ -1657,7 +1657,7 @@ ${optionDetails}
}
/**
* Get progressive flush interval based on CURRENT entity count (v4.2.0+)
* Get progressive flush interval based on CURRENT entity count
*
* Unlike adaptive intervals (which require knowing total count upfront),
* progressive intervals adjust dynamically as import proceeds.
@ -1699,7 +1699,7 @@ ${optionDetails}
/**
* Infer relationship type based on entity types and context
* v4.9.0: Semantic relationship enhancement
* Semantic relationship enhancement
*
* @param sourceType - Type of source entity
* @param targetType - Type of target entity
@ -1769,7 +1769,7 @@ ${optionDetails}
/**
* Count entities by type for document metadata
* v4.9.0: Used for document entity statistics
* Used for document entity statistics
*
* @param rows - Extracted rows from import
* @returns Record of entity type counts

14
src/importers/SmartCSVImporter.ts
View file

@ -42,17 +42,17 @@ export interface SmartCSVOptions extends FormatHandlerOptions {
csvDelimiter?: string
csvHeaders?: boolean
/** Progress callback (v3.39.0: Enhanced with performance metrics) */
/** Progress callback (Enhanced with performance metrics) */
onProgress?: (stats: {
processed: number
total: number
entities: number
relationships: number
/** Rows per second (v3.39.0) */
/** Rows per second */
throughput?: number
/** Estimated time remaining in ms (v3.39.0) */
/** Estimated time remaining in ms */
eta?: number
/** Current phase (v3.39.0) */
/** Current phase */
phase?: string
}) => void
}
@ -169,7 +169,7 @@ export class SmartCSVImporter {
}
// Parse CSV using existing handler
// v4.5.0: Pass progress hooks to handler for file parsing progress
// Pass progress hooks to handler for file parsing progress
const processedData = await this.csvHandler.process(buffer, {
...options,
csvDelimiter: opts.csvDelimiter,
@ -217,7 +217,7 @@ export class SmartCSVImporter {
// Detect column names
const columns = this.detectColumns(rows[0], opts)
// Process each row with BATCHED PARALLEL PROCESSING (v3.39.0)
// Process each row with BATCHED PARALLEL PROCESSING
const extractedRows: ExtractedRow[] = []
const entityMap = new Map<string, string>()
const stats = {
@ -375,7 +375,7 @@ export class SmartCSVImporter {
total: rows.length,
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
// Additional performance metrics (v3.39.0)
// Additional performance metrics
throughput: Math.round(rowsPerSecond * 10) / 10,
eta: Math.round(estimatedTimeRemaining),
phase: 'extracting'

6
src/importers/SmartDOCXImporter.ts
View file

@ -9,7 +9,7 @@
* - NaturalLanguageProcessor for relationship inference
* - Hierarchical relationship creation based on heading hierarchy
*
* v4.2.0: New format handler
* New format handler
* NO MOCKS - Production-ready implementation
*/
@ -187,7 +187,7 @@ export class SmartDOCXImporter {
await this.init()
}
// v4.5.0: Report parsing start
// Report parsing start
options.onProgress?.({
processed: 0,
entities: 0,
@ -200,7 +200,7 @@ export class SmartDOCXImporter {
// Extract HTML for structure analysis (headings, tables)
const htmlResult = await mammoth.convertToHtml({ buffer })
// v4.5.0: Report parsing complete
// Report parsing complete
options.onProgress?.({
processed: 0,
entities: 0,

28
src/importers/SmartExcelImporter.ts
View file

@ -36,17 +36,17 @@ export interface SmartExcelOptions extends FormatHandlerOptions {
typeColumn?: string // e.g., "Type", "Category"
relatedColumn?: string // e.g., "Related Terms", "See Also"
/** Progress callback (v3.38.0: Enhanced with performance metrics) */
/** Progress callback (Enhanced with performance metrics) */
onProgress?: (stats: {
processed: number
total: number
entities: number
relationships: number
/** Rows per second (v3.38.0) */
/** Rows per second */
throughput?: number
/** Estimated time remaining in ms (v3.38.0) */
/** Estimated time remaining in ms */
eta?: number
/** Current phase (v3.38.0) */
/** Current phase */
phase?: string
}) => void
}
@ -111,7 +111,7 @@ export interface SmartExcelResult {
}
}
/** Sheet-specific data for VFS extraction (v4.2.0) */
/** Sheet-specific data for VFS extraction */
sheets?: Array<{
name: string
rows: ExtractedRow[]
@ -161,7 +161,7 @@ export class SmartExcelImporter {
const opts = {
enableNeuralExtraction: true,
enableRelationshipInference: true,
// CONCEPT EXTRACTION PRODUCTION-READY (v3.33.0+):
// CONCEPT EXTRACTION PRODUCTION-READY:
// Type embeddings are now pre-computed at build time - zero runtime cost!
// All 42 noun types + 127 verb types instantly available
//
@ -184,7 +184,7 @@ export class SmartExcelImporter {
}
// Parse Excel using existing handler
// v4.5.0: Pass progress hooks to handler for file parsing progress
// Pass progress hooks to handler for file parsing progress
const processedData = await this.excelHandler.process(buffer, {
...options,
totalBytes: buffer.length,
@ -227,7 +227,7 @@ export class SmartExcelImporter {
return this.emptyResult(startTime)
}
// CRITICAL FIX (v4.8.6): Detect columns per-sheet, not globally
// CRITICAL FIX: Detect columns per-sheet, not globally
// Different sheets may have different column structures (Term vs Name, etc.)
// Group rows by sheet and detect columns for each sheet separately
const rowsBySheet = new Map<string, typeof rows>()
@ -247,7 +247,7 @@ export class SmartExcelImporter {
}
}
// Process each row with BATCHED PARALLEL PROCESSING (v3.38.0)
// Process each row with BATCHED PARALLEL PROCESSING
const extractedRows: ExtractedRow[] = []
const entityMap = new Map<string, string>()
const stats = {
@ -269,7 +269,7 @@ export class SmartExcelImporter {
chunk.map(async (row, chunkIndex) => {
const i = chunkStart + chunkIndex
// CRITICAL FIX (v4.8.6): Use sheet-specific column mapping
// CRITICAL FIX: Use sheet-specific column mapping
const sheet = row._sheet || 'default'
const columns = columnsBySheet.get(sheet) || this.detectColumns(row, opts)
@ -353,7 +353,7 @@ export class SmartExcelImporter {
}
// ============================================
// v4.9.0: Enhanced column-based relationship detection
// Enhanced column-based relationship detection
// ============================================
// Parse explicit "Related Terms" column
if (relatedTerms) {
@ -379,7 +379,7 @@ export class SmartExcelImporter {
}
}
// v4.9.0: Check for additional relationship-indicating columns
// Check for additional relationship-indicating columns
// Expanded patterns for various relationship types
const relationshipColumnPatterns = [
{ pattern: /^(location|home|lives in|resides|dwelling|place)$/i, defaultType: VerbType.LocatedAt },
@ -485,14 +485,14 @@ export class SmartExcelImporter {
total: rows.length,
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
// Additional performance metrics (v3.38.0)
// Additional performance metrics
throughput: Math.round(rowsPerSecond * 10) / 10,
eta: Math.round(estimatedTimeRemaining),
phase: 'extracting'
} as any)
}
// Group rows by sheet for VFS extraction (v4.2.0)
// Group rows by sheet for VFS extraction
const sheetGroups = new Map<string, ExtractedRow[]>()
extractedRows.forEach((extractedRow, index) => {
const originalRow = rows[index]

6
src/importers/SmartJSONImporter.ts
View file

@ -167,7 +167,7 @@ export class SmartJSONImporter {
...options
}
// v4.5.0: Report parsing start
// Report parsing start
opts.onProgress({
processed: 0,
entities: 0,
@ -186,7 +186,7 @@ export class SmartJSONImporter {
jsonData = data
}
// v4.5.0: Report parsing complete, starting traversal
// Report parsing complete, starting traversal
opts.onProgress({
processed: 0,
entities: 0,
@ -228,7 +228,7 @@ export class SmartJSONImporter {
}
)
// v4.5.0: Report completion
// Report completion
opts.onProgress({
processed: nodesProcessed,
entities: entities.length,

6
src/importers/SmartMarkdownImporter.ts
View file

@ -174,7 +174,7 @@ export class SmartMarkdownImporter {
...options
}
// v4.5.0: Report parsing start
// Report parsing start
opts.onProgress({
processed: 0,
total: 0,
@ -185,7 +185,7 @@ export class SmartMarkdownImporter {
// Parse markdown into sections
const parsedSections = this.parseMarkdown(markdown, opts)
// v4.5.0: Report parsing complete
// Report parsing complete
opts.onProgress({
processed: 0,
total: parsedSections.length,
@ -218,7 +218,7 @@ export class SmartMarkdownImporter {
})
}
// v4.5.0: Report completion
// Report completion
const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0)
const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0)
opts.onProgress({

16
src/importers/SmartPDFImporter.ts
View file

@ -40,17 +40,17 @@ export interface SmartPDFOptions extends FormatHandlerOptions {
/** Group by page or full document */
groupBy?: 'page' | 'document'
/** Progress callback (v3.39.0: Enhanced with performance metrics) */
/** Progress callback (Enhanced with performance metrics) */
onProgress?: (stats: {
processed: number
total: number
entities: number
relationships: number
/** Sections per second (v3.39.0) */
/** Sections per second */
throughput?: number
/** Estimated time remaining in ms (v3.39.0) */
/** Estimated time remaining in ms */
eta?: number
/** Current phase (v3.39.0) */
/** Current phase */
phase?: string
}) => void
}
@ -181,7 +181,7 @@ export class SmartPDFImporter {
}
// Parse PDF using existing handler
// v4.5.0: Pass progress hooks to handler for file parsing progress
// Pass progress hooks to handler for file parsing progress
const processedData = await this.pdfHandler.process(buffer, {
...options,
totalBytes: buffer.length,
@ -228,7 +228,7 @@ export class SmartPDFImporter {
// Group data by page or combine into single document
const grouped = this.groupData(data, opts)
// Process each group with BATCHED PARALLEL PROCESSING (v3.39.0)
// Process each group with BATCHED PARALLEL PROCESSING
const sections: ExtractedSection[] = []
const entityMap = new Map<string, string>()
const stats = {
@ -270,7 +270,7 @@ export class SmartPDFImporter {
total: totalGroups,
entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0),
// Additional performance metrics (v3.39.0)
// Additional performance metrics
throughput: Math.round(sectionsPerSecond * 10) / 10,
eta: Math.round(estimatedTimeRemaining),
phase: 'extracting'
@ -367,7 +367,7 @@ export class SmartPDFImporter {
const combinedText = texts.join('\n\n')
// Parallel extraction: entities AND concepts at the same time (v3.39.0)
// Parallel extraction: entities AND concepts at the same time
const [extractedEntities, concepts] = await Promise.all([
// Extract entities if enabled
options.enableNeuralExtraction && combinedText.length > 0

6
src/importers/SmartYAMLImporter.ts
View file

@ -8,7 +8,7 @@
* - NaturalLanguageProcessor for relationship inference
* - Hierarchical relationship creation (parent-child, contains, etc.)
*
* v4.2.0: New format handler
* New format handler
* NO MOCKS - Production-ready implementation
*/
@ -159,7 +159,7 @@ export class SmartYAMLImporter {
): Promise<SmartYAMLResult> {
const startTime = Date.now()
// v4.5.0: Report parsing start
// Report parsing start
options.onProgress?.({
processed: 0,
entities: 0,
@ -178,7 +178,7 @@ export class SmartYAMLImporter {
throw new Error(`Failed to parse YAML: ${error.message}`)
}
// v4.5.0: Report parsing complete
// Report parsing complete
options.onProgress?.({
processed: 0,
entities: 0,

26
src/importers/VFSStructureGenerator.ts
View file

@ -40,10 +40,10 @@ export interface VFSStructureOptions {
/** Create metadata file */
createMetadataFile?: boolean
/** Import tracking context (v4.10.0) */
/** Import tracking context */
trackingContext?: TrackingContext
/** Progress callback (v4.11.1) - Reports VFS creation progress */
/** Progress callback - Reports VFS creation progress */
onProgress?: (progress: {
stage: 'directories' | 'entities' | 'metadata'
message: string
@ -98,7 +98,7 @@ export class VFSStructureGenerator {
// Get brain's cached VFS instance (creates if doesn't exist)
this.vfs = this.brain.vfs
// CRITICAL FIX (v4.10.2): Always call vfs.init() explicitly
// CRITICAL FIX: Always call vfs.init() explicitly
// The previous code tried to check if initialized via stat('/') but this was unreliable
// vfs.init() is idempotent, so calling it multiple times is safe
await this.vfs.init()
@ -123,7 +123,7 @@ export class VFSStructureGenerator {
// Ensure VFS is initialized
await this.init()
// v4.11.1: Calculate total operations for progress tracking
// Calculate total operations for progress tracking
const groups = this.groupEntities(importResult, options)
const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0)
const totalOperations =
@ -161,7 +161,7 @@ export class VFSStructureGenerator {
try {
await this.vfs.mkdir(options.rootPath, {
recursive: true,
metadata: trackingMetadata // v4.10.0: Add tracking metadata
metadata: trackingMetadata // Add tracking metadata
})
result.directories.push(options.rootPath)
result.operations++
@ -179,7 +179,7 @@ export class VFSStructureGenerator {
if (options.preserveSource && options.sourceBuffer && options.sourceFilename) {
const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}`
await this.vfs.writeFile(sourcePath, options.sourceBuffer, {
metadata: trackingMetadata // v4.10.0: Add tracking metadata
metadata: trackingMetadata // Add tracking metadata
})
result.files.push({
path: sourcePath,
@ -200,7 +200,7 @@ export class VFSStructureGenerator {
try {
await this.vfs.mkdir(groupPath, {
recursive: true,
metadata: trackingMetadata // v4.10.0: Add tracking metadata
metadata: trackingMetadata // Add tracking metadata
})
result.directories.push(groupPath)
result.operations++
@ -242,7 +242,7 @@ export class VFSStructureGenerator {
await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2), {
metadata: {
...trackingMetadata, // v4.10.0: Add tracking metadata
...trackingMetadata, // Add tracking metadata
entityId: extracted.entity.id
}
})
@ -253,7 +253,7 @@ export class VFSStructureGenerator {
})
result.operations++
// v4.11.1: Report progress every 10 entities (or on last entity)
// Report progress every 10 entities (or on last entity)
if (entityCount % 10 === 0 || entityCount === entities.length) {
reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`)
}
@ -280,7 +280,7 @@ export class VFSStructureGenerator {
}
await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2), {
metadata: trackingMetadata // v4.10.0: Add tracking metadata
metadata: trackingMetadata // Add tracking metadata
})
result.files.push({
path: relationshipsPath,
@ -322,7 +322,7 @@ export class VFSStructureGenerator {
}
await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2), {
metadata: trackingMetadata // v4.10.0: Add tracking metadata
metadata: trackingMetadata // Add tracking metadata
})
result.files.push({
path: metadataPath,
@ -332,7 +332,7 @@ export class VFSStructureGenerator {
reportProgress('metadata', 'Created metadata file')
}
// v4.11.1: Final progress update
// Final progress update
if (options.onProgress) {
options.onProgress({
stage: 'metadata',
@ -355,7 +355,7 @@ export class VFSStructureGenerator {
): Map<string, typeof importResult.rows> {
const groups = new Map<string, typeof importResult.rows>()
// Handle sheet-based grouping (v4.2.0)
// Handle sheet-based grouping
if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) {
for (const sheet of importResult.sheets) {
groups.set(sheet.name, sheet.rows)

6
src/index.ts
View file

@ -70,7 +70,7 @@ export type {
NeuralImportOptions
} from './cortex/neuralImport.js'
// Export Neural Entity Extraction (v5.7.6 - Workshop request)
// Export Neural Entity Extraction
export { NeuralEntityExtractor } from './neural/entityExtractor.js'
export { SmartExtractor } from './neural/SmartExtractor.js'
export { SmartRelationshipExtractor } from './neural/SmartRelationshipExtractor.js'
@ -220,7 +220,7 @@ export {
// FileSystemStorage is exported separately to avoid browser build issues
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
// Export COW (Copy-on-Write) infrastructure for v5.0.0
// Export COW (Copy-on-Write) infrastructure
// Enables premium augmentations to implement temporal features
import { CommitLog } from './storage/cow/CommitLog.js'
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
@ -543,7 +543,7 @@ export type {
MCPTool
}
// ============= Integration Hub (v7.4.0) =============
// ============= Integration Hub =============
// Connect Brainy to Excel, Power BI, Google Sheets, and more
// Enable with: new Brainy({ integrations: true })

2
src/integrations/index.ts
View file

@ -7,7 +7,7 @@
* - Real-time dashboards (SSE streaming)
* - External webhooks (push notifications)
*
* @example Enable integrations (v7.4.0 - recommended)
* @example Enable integrations (recommended)
* ```typescript
* import { Brainy } from '@soulcraft/brainy'
*

6
src/interfaces/IIndex.ts
View file

@ -1,5 +1,5 @@
/**
* Unified Index Interface (v3.35.0+)
* Unified Index Interface
*
* Standardizes index lifecycle across all index types in Brainy.
* All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
@ -57,7 +57,7 @@ export interface RebuildOptions {
* - On-demand: Large datasets loaded adaptively via UnifiedCache
*
* This option is kept for backwards compatibility but is ignored.
* The system always uses adaptive caching (v3.36.0+).
* The system always uses adaptive caching.
*/
lazy?: boolean
@ -104,7 +104,7 @@ export interface IIndex {
* - Provide progress reporting for large datasets
* - Recover gracefully from partial failures
*
* Adaptive Caching (v3.36.0+):
* Adaptive Caching:
* System automatically chooses optimal strategy:
* - Small datasets: Preload all data at init for zero-latency access
* - Large datasets: Load on-demand via UnifiedCache for memory efficiency

20
src/neural/entityExtractor.ts
View file

@ -2,7 +2,7 @@
* Neural Entity Extractor using Brainy's NounTypes
* Uses embeddings and similarity matching for accurate type detection
*
* v4.2.0: Now powered by SmartExtractor for ultra-neural classification
* Now powered by SmartExtractor for ultra-neural classification
* PRODUCTION-READY with caching support
*/
@ -24,7 +24,7 @@ export interface ExtractedEntity {
type: NounType
position: { start: number; end: number }
confidence: number
weight?: number // v4.2.0: Entity importance/salience
weight?: number // Entity importance/salience
vector?: Vector
metadata?: any
}
@ -39,7 +39,7 @@ export class NeuralEntityExtractor {
// Entity extraction cache
private cache: EntityExtractionCache
// Runtime embedding cache for performance (v3.38.0)
// Runtime embedding cache for performance
// Caches candidate embeddings during an extraction session to avoid redundant model calls
private embeddingCache: Map<string, Vector> = new Map()
private embeddingCacheStats = {
@ -48,7 +48,7 @@ export class NeuralEntityExtractor {
size: 0
}
// v4.2.0: SmartExtractor for ultra-neural classification
// SmartExtractor for ultra-neural classification
private smartExtractor: SmartExtractor
constructor(brain: Brainy | Brainy<any>, cacheOptions?: EntityCacheOptions) {
@ -63,7 +63,7 @@ export class NeuralEntityExtractor {
/**
* Initialize type embeddings for neural matching
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed embeddings from build time
* PRODUCTION OPTIMIZATION: Uses pre-computed embeddings from build time
* Zero runtime cost - embeddings are loaded instantly from embedded data
*/
private async initializeTypeEmbeddings(requestedTypes?: NounType[]): Promise<void> {
@ -109,7 +109,7 @@ export class NeuralEntityExtractor {
}
}
): Promise<ExtractedEntity[]> {
// PRODUCTION OPTIMIZATION (v3.33.0): Load pre-computed type embeddings
// PRODUCTION OPTIMIZATION: Load pre-computed type embeddings
// Zero runtime cost - embeddings were computed at build time
await this.initializeTypeEmbeddings(options?.types)
@ -138,7 +138,7 @@ export class NeuralEntityExtractor {
// Step 1: Extract potential entities using patterns
const candidates = await this.extractCandidates(text)
// Step 2: Classify each candidate using SmartExtractor (v4.2.0)
// Step 2: Classify each candidate using SmartExtractor
for (const candidate of candidates) {
// Use SmartExtractor for unified neural + rule-based classification
const classification = await this.smartExtractor.extract(candidate.text, {
@ -357,7 +357,7 @@ export class NeuralEntityExtractor {
}
/**
* Get embedding for text with caching (v3.38.0)
* Get embedding for text with caching
*
* PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
* to avoid redundant model calls for repeated text (common in large imports)
@ -509,7 +509,7 @@ export class NeuralEntityExtractor {
}
/**
* Clear embedding cache (v3.38.0)
* Clear embedding cache
*
* Clears the runtime embedding cache. Useful for:
* - Freeing memory after large imports
@ -525,7 +525,7 @@ export class NeuralEntityExtractor {
}
/**
* Get embedding cache statistics (v3.38.0)
* Get embedding cache statistics
*
* Returns performance metrics for the embedding cache:
* - hits: Number of cache hits (avoided model calls)

2
src/neural/naturalLanguageProcessor.ts
View file

@ -99,7 +99,7 @@ export class NaturalLanguageProcessor {
/**
* Initialize embeddings for all NounTypes and VerbTypes
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings
* PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings
* Zero runtime cost - embeddings are loaded instantly from embedded data
*/
private async initializeTypeEmbeddings(): Promise<void> {

4
src/neural/signals/PatternSignal.ts
View file

@ -122,7 +122,7 @@ export class PatternSignal {
])
// Location patterns - MEDIUM PRIORITY (city/country format - requires more context)
// v4.11.2: Lower priority to avoid matching person names with commas
// Lower priority to avoid matching person names with commas
this.addPatterns(NounType.Location, 0.75, [
/\b[A-Z][a-z]+,\s*(?:Japan|China|France|Germany|Italy|Spain|Canada|Mexico|Brazil|India|Australia|Russia|UK|USA)\b/
])
@ -170,7 +170,7 @@ export class PatternSignal {
// Technology patterns (Thing type)
this.addPatterns(NounType.Thing, 0.82, [
/\b(?:JavaScript|TypeScript|Python|Java|Go|Rust|Swift|Kotlin)\b/,
/\bC\+\+(?!\w)/, // v4.11.2: Special handling for C++ (word boundary doesn't work with +)
/\bC\+\+(?!\w)/, // Special handling for C++ (word boundary doesn't work with +)
/\b(?:React|Vue|Angular|Node|Express|Django|Flask|Rails)\b/,
/\b(?:AWS|Azure|GCP|Docker|Kubernetes|Git|GitHub|GitLab)\b/,
/\b(?:API|SDK|CLI|IDE|framework|library|package|module)\b/i,

2
src/setup.ts
View file

@ -1,7 +1,7 @@
/**
* Brainy Setup - Minimal Polyfills
*
* ARCHITECTURE (v7.0.0):
* ARCHITECTURE:
* Brainy uses Candle WASM (Rust-based) for embeddings.
* No transformers.js or ONNX Runtime dependency, no hacks required.
*

108
src/storage/adapters/azureBlobStorage.ts
View file

@ -9,7 +9,7 @@
* 4. SAS Token
* 5. Azure AD (OAuth2) via DefaultAzureCredential
*
* v4.0.0: Fully compatible with metadata/vector separation architecture
* Fully compatible with metadata/vector separation architecture
*/
import {
@ -65,7 +65,7 @@ const MAX_AZURE_PAGE_SIZE = 5000
* 3. Storage Account Key - if accountName + accountKey provided
* 4. SAS Token - if accountName + sasToken provided
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed pagination overrides
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -105,7 +105,7 @@ export class AzureBlobStorage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Multi-level cache manager for efficient data access
@ -115,7 +115,7 @@ export class AzureBlobStorage extends BaseStorage {
// Module logger
private logger = createModuleLogger('AzureBlobStorage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
/**
@ -162,7 +162,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
* strict locally. Zero-config optimization.
@ -171,7 +171,6 @@ export class AzureBlobStorage extends BaseStorage {
* - `'strict'`: Traditional blocking init. Validates container and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: InitMode
@ -185,7 +184,7 @@ export class AzureBlobStorage extends BaseStorage {
this.sasToken = options.sasToken
this.readOnly = options.readOnly || false
// v7.3.0: Handle initMode
// Handle initMode
if (options.initMode) {
this.initMode = options.initMode
}
@ -201,7 +200,7 @@ export class AzureBlobStorage extends BaseStorage {
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// v6.2.7: Write buffering always enabled - no env var check needed
// Write buffering always enabled - no env var check needed
}
/**
@ -216,7 +215,7 @@ export class AzureBlobStorage extends BaseStorage {
* Recent Azure improvements make parallel downloads very efficient
*
* @returns Azure Blob-optimized batch configuration
* @since v5.12.0 - Updated for native batch API
* Updated for native batch API
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -241,7 +240,6 @@ export class AzureBlobStorage extends BaseStorage {
*
* @param paths - Array of Azure blob paths to read
* @returns Map of path -> parsed JSON data (only successful reads)
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -297,7 +295,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Initialize the storage adapter
*
* v7.3.0: Supports progressive initialization for fast cold starts
* Supports progressive initialization for fast cold starts
*
* | Mode | Init Time | When |
* |------|-----------|------|
@ -404,7 +402,7 @@ export class AzureBlobStorage extends BaseStorage {
this.verbCacheManager.clear()
prodLog.info('✅ Cache cleared - starting fresh')
// v7.3.0: Progressive vs Strict initialization
// Progressive vs Strict initialization
if (effectiveMode === 'progressive') {
// PROGRESSIVE MODE: Fast init, background validation
// Mark as initialized immediately - ready to accept operations
@ -413,7 +411,7 @@ export class AzureBlobStorage extends BaseStorage {
prodLog.info(`✅ Azure progressive init complete: ${this.containerName} (validation deferred)`)
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Schedule background tasks (non-blocking)
@ -434,7 +432,7 @@ export class AzureBlobStorage extends BaseStorage {
await this.initializeCounts()
this.countsLoaded = true
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Mark background tasks as complete (nothing to do in background)
@ -447,7 +445,7 @@ export class AzureBlobStorage extends BaseStorage {
}
// =============================================
// Progressive Initialization (v7.3.0+)
// Progressive Initialization
// =============================================
/**
@ -461,7 +459,6 @@ export class AzureBlobStorage extends BaseStorage {
*
* @protected
* @override
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
const startTime = Date.now()
@ -485,7 +482,6 @@ export class AzureBlobStorage extends BaseStorage {
* bucketValidated/bucketValidationError for lazy use.
*
* @private
* @since v7.3.0
*/
private async validateContainerInBackground(): Promise<void> {
try {
@ -517,7 +513,6 @@ export class AzureBlobStorage extends BaseStorage {
* Load counts from storage in background.
*
* @private
* @since v7.3.0
*/
private async loadCountsInBackground(): Promise<void> {
try {
@ -541,7 +536,6 @@ export class AzureBlobStorage extends BaseStorage {
* @throws Error if container validation fails
* @protected
* @override
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do
@ -665,7 +659,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Flush noun buffer to Azure
@ -697,20 +691,20 @@ export class AzureBlobStorage extends BaseStorage {
await Promise.all(writes)
}
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -791,7 +785,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Get a node from storage
@ -889,18 +883,18 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Write an object to a specific path in Azure
* Primitive operation required by base class
*
* v7.3.0: Performs lazy container validation on first write in progressive mode.
* Performs lazy container validation on first write in progressive mode.
* @protected
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy container validation for progressive init
// Lazy container validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -954,12 +948,12 @@ export class AzureBlobStorage extends BaseStorage {
* Delete an object from a specific path in Azure
* Primitive operation required by base class
*
* v7.3.0: Performs lazy container validation on first delete in progressive mode.
* Performs lazy container validation on first delete in progressive mode.
* @protected
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy container validation for progressive init
// Lazy container validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -1209,20 +1203,20 @@ export class AzureBlobStorage extends BaseStorage {
})
}
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Save an edge to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.verbWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
this.verbCacheManager.set(edge.id, edge)
await this.verbWriteBuffer.add(edge.id, edge)
@ -1255,7 +1249,7 @@ export class AzureBlobStorage extends BaseStorage {
])
),
// CORE RELATIONAL DATA (v4.0.0)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -1280,7 +1274,7 @@ export class AzureBlobStorage extends BaseStorage {
// Update cache
this.verbCacheManager.set(edge.id, edge)
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.logger.trace(`Edge ${edge.id} saved successfully`)
@ -1298,7 +1292,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Get an edge from storage
@ -1335,7 +1329,7 @@ export class AzureBlobStorage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
const edge: Edge = {
id: data.id,
vector: data.vector,
@ -1346,7 +1340,7 @@ export class AzureBlobStorage extends BaseStorage {
sourceId: data.sourceId,
targetId: data.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
@ -1375,13 +1369,13 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation
// Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation
// Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation
// Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation
/**
* Clear all data from storage
@ -1393,7 +1387,7 @@ export class AzureBlobStorage extends BaseStorage {
this.logger.info('🧹 Clearing all data from Azure container...')
// Delete all blobs in container
// v5.6.1: listBlobsFlat() returns ALL blobs including _cow/ prefix
// listBlobsFlat() returns ALL blobs including _cow/ prefix
// This correctly deletes COW version control data (commits, trees, blobs, refs)
for await (const blob of this.containerClient!.listBlobsFlat()) {
if (blob.name) {
@ -1402,7 +1396,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1461,12 +1455,12 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker blob exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -1643,7 +1637,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Get a noun's vector for HNSW rebuild
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -1653,7 +1647,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Uses mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -1662,7 +1656,7 @@ export class AzureBlobStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -1682,7 +1676,7 @@ export class AzureBlobStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -1704,7 +1698,7 @@ export class AzureBlobStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -1715,7 +1709,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -1744,7 +1738,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Save HNSW system data (entry point, max level)
*
* CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
* CRITICAL FIX: Optimistic locking with ETags to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -1840,7 +1834,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Set the access tier for a specific blob (v4.0.0 cost optimization)
* Set the access tier for a specific blob (cost optimization)
* Azure Blob Storage tiers:
* - Hot: $0.0184/GB/month - Frequently accessed data
* - Cool: $0.01/GB/month - Infrequently accessed data (45% cheaper)
@ -1906,7 +1900,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Set access tier for multiple blobs in batch (v4.0.0 cost optimization)
* Set access tier for multiple blobs in batch (cost optimization)
* Efficiently move large numbers of blobs between tiers for cost optimization
*
* @param blobs - Array of blob names and their target tiers
@ -2128,7 +2122,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Set lifecycle management policy for automatic tier transitions and deletions (v4.0.0)
* Set lifecycle management policy for automatic tier transitions and deletions
* Automates cost optimization by moving old data to cheaper tiers or deleting it
*
* Azure Lifecycle Management rules run once per day and apply to the entire container.

43
src/storage/adapters/baseStorageAdapter.ts
View file

@ -19,7 +19,7 @@ import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/field
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
// =============================================================================
// Progressive Initialization Types (v7.3.0+)
// Progressive Initialization Types
// =============================================================================
/**
@ -29,7 +29,6 @@ import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
* fast cold starts in serverless environments while maintaining strict
* validation for local development.
*
* @since v7.3.0
*
* | Mode | Description | Use Case |
* |------|-------------|----------|
@ -96,7 +95,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
abstract getVerbMetadata(id: string): Promise<any | null>
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
// These methods enable HNSW index rebuilding after container restarts
abstract getNounVector(id: string): Promise<number[] | null>
@ -139,7 +138,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* relateMany(), and import operations to automatically adapt to storage capabilities.
*
* @returns Batch configuration optimized for this storage type
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
// Conservative defaults that work safely across all storage types
@ -295,7 +293,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}> = new Map()
// =============================================
// Progressive Initialization State (v7.3.0+)
// Progressive Initialization State
// =============================================
// These properties enable fast cold starts in cloud environments
// by deferring validation and count loading to background tasks.
@ -308,7 +306,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* - `'strict'`: Always validate during init (traditional behavior)
*
* @protected
* @since v7.3.0
*/
protected initMode: InitMode = 'auto'
@ -319,7 +316,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* after the first successful write operation or background validation.
*
* @protected
* @since v7.3.0
*/
protected bucketValidated = false
@ -329,7 +325,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Stored here so subsequent operations can fail fast with the same error.
*
* @protected
* @since v7.3.0
*/
protected bucketValidationError: Error | null = null
@ -340,7 +335,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Operations work immediately; counts become accurate after background load.
*
* @protected
* @since v7.3.0
*/
protected countsLoaded = false
@ -350,7 +344,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Useful for tests and diagnostics to ensure full initialization.
*
* @protected
* @since v7.3.0
*/
protected backgroundTasksComplete = false
@ -361,7 +354,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* before proceeding (e.g., in tests).
*
* @protected
* @since v7.3.0
*/
protected backgroundInitPromise: Promise<void> | null = null
@ -1057,7 +1049,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
protected readonly COUNT_CACHE_TTL = 60000 // 1 minute cache TTL
// =============================================
// Smart Count Batching (v3.32.3+)
// Smart Count Batching
// =============================================
// Count batching state - mirrors statistics batching pattern
@ -1112,7 +1104,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.incrementEntityCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
// - Cloud storage (GCS, S3): Batches 10 ops OR 5 seconds
// - Local storage (File, Memory): Persists immediately
await this.scheduleCountPersist()
@ -1147,13 +1139,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.decrementEntityCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
await this.scheduleCountPersist()
})
}
/**
* Increment verb count - O(1) operation (v4.1.2: now synchronous)
* Increment verb count - O(1) operation (now synchronous)
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
* @param type The verb type
*/
@ -1168,7 +1160,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Thread-safe increment for verb counts (v4.1.2)
* Thread-safe increment for verb counts
* Uses mutex for single-node, distributed consensus for multi-node
* @param type The verb type
*/
@ -1176,13 +1168,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-verb-${type}`, async () => {
this.incrementVerbCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
await this.scheduleCountPersist()
})
}
/**
* Decrement verb count - O(1) operation (v4.1.2: now synchronous)
* Decrement verb count - O(1) operation (now synchronous)
* @param type The verb type
*/
protected decrementVerbCount(type: string): void {
@ -1203,20 +1195,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Thread-safe decrement for verb counts (v4.1.2)
* Thread-safe decrement for verb counts
* @param type The verb type
*/
protected async decrementVerbCountSafe(type: string): Promise<void> {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-verb-${type}`, async () => {
this.decrementVerbCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
await this.scheduleCountPersist()
})
}
// =============================================
// Smart Batching Methods (v3.32.3+)
// Smart Batching Methods
// =============================================
/**
@ -1235,7 +1227,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
// =============================================
// Progressive Initialization Methods (v7.3.0+)
// Progressive Initialization Methods
// =============================================
/**
@ -1254,7 +1246,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @returns `true` if running in a detected cloud environment
* @protected
* @since v7.3.0
*/
protected detectCloudEnvironment(): boolean {
return !!(
@ -1274,7 +1265,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @returns The resolved init mode (`'progressive'` or `'strict'`)
* @protected
* @since v7.3.0
*/
protected resolveInitMode(): 'progressive' | 'strict' {
if (this.initMode === 'auto') {
@ -1295,7 +1285,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Always call `super.scheduleBackgroundInit()` first.
*
* @protected
* @since v7.3.0
*/
protected scheduleBackgroundInit(): void {
// Create a promise that tracks all background work
@ -1320,7 +1309,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* The default implementation does nothing (for local storage adapters).
*
* @protected
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
// Default implementation: nothing to do for local storage
@ -1348,7 +1336,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* ```
*
* @public
* @since v7.3.0
*/
public async awaitBackgroundInit(): Promise<void> {
if (this.backgroundInitPromise) {
@ -1361,7 +1348,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @returns `true` if all background tasks are complete
* @public
* @since v7.3.0
*/
public isBackgroundInitComplete(): boolean {
return this.backgroundTasksComplete
@ -1379,7 +1365,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @throws Error if bucket validation fails
* @protected
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do

109
src/storage/adapters/fileSystemStorage.ts
View file

@ -59,7 +59,7 @@ try {
* File system storage adapter for Node.js environments
* Uses the file system to store data in the specified directory structure
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -91,12 +91,12 @@ export class FileSystemStorage extends BaseStorage {
private lockTimers: Map<string, NodeJS.Timeout> = new Map() // Track timers for cleanup
private allTimers: Set<NodeJS.Timeout> = new Set() // Track all timers for cleanup
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
// CRITICAL FIX: Mutex locks for HNSW concurrency control
// Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops)
// Matches MemoryStorage and OPFSStorage behavior (tested in production)
private hnswLocks = new Map<string, Promise<void>>()
// Compression configuration (v4.0.0)
// Compression configuration
private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings
private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced)
@ -136,7 +136,6 @@ export class FileSystemStorage extends BaseStorage {
* - Parallel processing supported
*
* @returns FileSystem-optimized batch configuration
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -179,7 +178,7 @@ export class FileSystemStorage extends BaseStorage {
try {
// Initialize directory paths now that path module is loaded
// Clean directory structure (v4.7.2+)
// Clean directory structure
this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw')
this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw')
this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference
@ -245,7 +244,7 @@ export class FileSystemStorage extends BaseStorage {
// Always use fixed depth after migration/detection
this.cachedShardingDepth = this.SHARDING_DEPTH
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
} catch (error) {
console.error('Error initializing FileSystemStorage:', error)
@ -281,7 +280,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save a node to storage
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
@ -303,7 +302,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Write to temp file
await this.ensureDirectoryExists(path.dirname(tempPath))
await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2))
@ -320,7 +319,7 @@ export class FileSystemStorage extends BaseStorage {
throw error
}
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveNounMetadata_internal
// This fixes the race condition where metadata didn't exist yet
}
@ -361,7 +360,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get all nodes from storage
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
* Previously only scanned flat directory, causing rebuild to find 0 entities
*/
protected async getAllNodes(): Promise<HNSWNode[]> {
@ -406,7 +405,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get nodes by noun type
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
@ -462,7 +461,7 @@ export class FileSystemStorage extends BaseStorage {
const filePath = this.getNodePath(id)
// Load metadata to get type for count update (v4.0.0: separate storage)
// Load metadata to get type for count update (separate storage)
try {
const metadata = await this.getNounMetadata(id)
if (metadata) {
@ -490,13 +489,13 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save an edge to storage
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// Convert connections Map to a serializable format
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
@ -505,7 +504,7 @@ export class FileSystemStorage extends BaseStorage {
Array.from(set as Set<string>)
),
// CORE RELATIONAL DATA (v3.50.1+)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -518,7 +517,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Write to temp file
await this.ensureDirectoryExists(path.dirname(tempPath))
await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2))
@ -535,7 +534,7 @@ export class FileSystemStorage extends BaseStorage {
throw error
}
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
}
@ -556,7 +555,7 @@ export class FileSystemStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
@ -567,7 +566,7 @@ export class FileSystemStorage extends BaseStorage {
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
} catch (error: any) {
@ -580,7 +579,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get all edges from storage
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
* Previously only scanned flat directory, causing rebuild to find 0 relationships
*/
protected async getAllEdges(): Promise<Edge[]> {
@ -608,7 +607,7 @@ export class FileSystemStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// v4.0.0: Include core relational fields (NO metadata field)
// Include core relational fields (NO metadata field)
allEdges.push({
id: parsedEdge.id,
vector: parsedEdge.vector,
@ -619,7 +618,7 @@ export class FileSystemStorage extends BaseStorage {
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
})
}
@ -696,8 +695,8 @@ export class FileSystemStorage extends BaseStorage {
/**
* Primitive operation: Write object to path
* All metadata operations use this internally via base class routing
* v4.0.0: Supports gzip compression for 60-80% disk savings
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
* Supports gzip compression for 60-80% disk savings
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
*/
protected async writeObjectToPath(pathStr: string, data: any): Promise<void> {
await this.ensureInitialized()
@ -711,7 +710,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${compressedPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Compress and write to temp file
const jsonString = JSON.stringify(data, null, 2)
const compressed = await new Promise<Buffer>((resolve, reject) => {
@ -748,7 +747,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Write to temp file
await fs.promises.writeFile(tempPath, JSON.stringify(data, null, 2))
@ -770,7 +769,7 @@ export class FileSystemStorage extends BaseStorage {
* Primitive operation: Read object from path
* All metadata operations use this internally via base class routing
* Enhanced error handling for corrupted metadata files (Bug #3 mitigation)
* v4.0.0: Supports reading both compressed (.gz) and uncompressed files for backward compatibility
* Supports reading both compressed (.gz) and uncompressed files for backward compatibility
*/
protected async readObjectFromPath(pathStr: string): Promise<any | null> {
await this.ensureInitialized()
@ -822,7 +821,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Primitive operation: Delete object from path
* All metadata operations use this internally via base class routing
* v4.0.0: Deletes both compressed and uncompressed versions (for cleanup)
* Deletes both compressed and uncompressed versions (for cleanup)
*/
protected async deleteObjectFromPath(pathStr: string): Promise<void> {
await this.ensureInitialized()
@ -863,7 +862,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Primitive operation: List objects under path prefix
* All metadata operations use this internally via base class routing
* v4.0.0: Handles both .json and .json.gz files, normalizes paths
* Handles both .json and .json.gz files, normalizes paths
*/
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
await this.ensureInitialized()
@ -877,7 +876,7 @@ export class FileSystemStorage extends BaseStorage {
for (const entry of entries) {
if (entry.isFile()) {
// v5.3.5: Handle multiple compression formats for broad compatibility
// Handle multiple compression formats for broad compatibility
// - .json.gz: Standard entity/metadata files (JSON compressed)
// - .gz: COW files (refs, blobs, commits - raw compressed)
// - .json: Uncompressed JSON files
@ -890,7 +889,7 @@ export class FileSystemStorage extends BaseStorage {
seen.add(normalizedPath)
}
} else if (entry.name.endsWith('.gz')) {
// v5.3.5 fix: COW files stored as .gz (not .json.gz)
// COW files stored as .gz (not .json.gz)
// Strip .gz extension and return path
const normalizedName = entry.name.slice(0, -3) // Remove .gz
const normalizedPath = path.join(prefix, normalizedName)
@ -966,7 +965,7 @@ export class FileSystemStorage extends BaseStorage {
* Get nouns with pagination support
* @param options Pagination options
*/
// v5.4.0: Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
// Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
/**
* Clear all data from storage
@ -1002,7 +1001,7 @@ export class FileSystemStorage extends BaseStorage {
}
}
// v5.10.4: Clear the entire branches/ directory (branch-based storage)
// Clear the entire branches/ directory (branch-based storage)
// Bug fix: Data is stored in branches/main/entities/, not just entities/
// The branch-based structure was introduced for COW support
const branchesDir = path.join(this.rootDir, 'branches')
@ -1016,7 +1015,7 @@ export class FileSystemStorage extends BaseStorage {
await removeDirectoryContents(this.indexDir)
}
// v5.6.1: Remove COW (copy-on-write) version control data
// Remove COW (copy-on-write) version control data
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
// Must be deleted to fully clear all data including version history
const cowDir = path.join(this.rootDir, '_cow')
@ -1024,7 +1023,7 @@ export class FileSystemStorage extends BaseStorage {
// Delete the entire _cow/ directory (not just contents)
await fs.promises.rm(cowDir, { recursive: true, force: true })
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1035,12 +1034,12 @@ export class FileSystemStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
// Reset entity counters (inherited from BaseStorageAdapter)
// These in-memory counters must be reset to 0 after clearing all data
;(this as any).totalNounCount = 0
;(this as any).totalVerbCount = 0
// v7.3.1: Clear write-through cache (inherited from BaseStorage)
// Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data
// after clear(), causing "ghost" entities to appear
this.clearWriteCache()
@ -1074,12 +1073,12 @@ export class FileSystemStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker file exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -1152,7 +1151,7 @@ export class FileSystemStorage extends BaseStorage {
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
// CRITICAL FIX (v3.43.2): Use persisted counts instead of directory reads
// CRITICAL FIX: Use persisted counts instead of directory reads
// This is O(1) instead of O(n), and handles sharded structure correctly
const nounsCount = this.totalNounCount
const verbsCount = this.totalVerbCount
@ -1210,8 +1209,8 @@ export class FileSystemStorage extends BaseStorage {
}
}
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
/**
* Acquire a file-based lock for coordinating operations across multiple processes
@ -1503,14 +1502,14 @@ export class FileSystemStorage extends BaseStorage {
this.totalVerbCount = validVerbFiles.length
// Sample some files to get type distribution (don't read all)
// v4.0.0: Load metadata separately for type information
// Load metadata separately for type information
const sampleSize = Math.min(100, validNounFiles.length)
for (let i = 0; i < sampleSize; i++) {
try {
const file = validNounFiles[i]
const id = file.replace('.json', '')
// v4.0.0: Load metadata from separate storage for type info
// Load metadata from separate storage for type info
const metadata = await this.getNounMetadata(id)
if (metadata) {
const type = metadata.noun || 'default'
@ -2148,7 +2147,7 @@ export class FileSystemStorage extends BaseStorage {
const edge = JSON.parse(data)
const metadata = await this.getVerbMetadata(id)
// v4.8.1: Don't skip verbs without metadata - metadata is optional
// Don't skip verbs without metadata - metadata is optional
// FIX: This was the root cause of the VFS bug (11 versions)
// Verbs can exist without metadata files (e.g., from imports/migrations)
@ -2162,7 +2161,7 @@ export class FileSystemStorage extends BaseStorage {
connections = connectionsMap
}
// v4.8.0: Extract standard fields from metadata to top-level
// Extract standard fields from metadata to top-level
const metadataObj = (metadata || {}) as VerbMetadata
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj
@ -2309,12 +2308,12 @@ export class FileSystemStorage extends BaseStorage {
}
// =============================================
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
// =============================================
/**
* Get vector for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -2324,7 +2323,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -2333,7 +2332,7 @@ export class FileSystemStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -2353,7 +2352,7 @@ export class FileSystemStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -2375,7 +2374,7 @@ export class FileSystemStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write)
// Use BaseStorage's saveNoun (type-first paths, atomic write)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -2386,7 +2385,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -2415,7 +2414,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save HNSW system data (entry point, max level)
*
* CRITICAL FIX (v4.10.1): Mutex lock + atomic write to prevent race conditions
* CRITICAL FIX: Mutex lock + atomic write to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -2425,7 +2424,7 @@ export class FileSystemStorage extends BaseStorage {
const lockKey = 'hnsw/system'
// CRITICAL FIX (v4.10.1): Mutex lock to serialize system updates
// CRITICAL FIX: Mutex lock to serialize system updates
// System data (entry point, max level) updated frequently during HNSW construction
// Without mutex, concurrent updates can lose data (same as entity-level problem)

135
src/storage/adapters/gcsStorage.ts
View file

@ -65,7 +65,7 @@ const MAX_GCS_PAGE_SIZE = 5000
* 3. Service Account Credentials Object (if credentials provided)
* 4. HMAC Keys (if accessKeyId/secretAccessKey provided)
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -110,7 +110,7 @@ export class GcsStorage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Multi-level cache manager for efficient data access
@ -120,7 +120,7 @@ export class GcsStorage extends BaseStorage {
// Module logger
private logger = createModuleLogger('GcsStorage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
// Configuration options
@ -166,7 +166,7 @@ export class GcsStorage extends BaseStorage {
secretAccessKey?: string
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Cloud Run, Lambda),
* strict locally. Zero-config optimization.
@ -175,19 +175,18 @@ export class GcsStorage extends BaseStorage {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: InitMode
/**
* @deprecated Use `initMode: 'progressive'` instead.
* Will be removed in v8.0.0.
* Will be removed in a future version.
*/
skipInitialScan?: boolean
/**
* @deprecated Use `initMode: 'progressive'` instead.
* Will be removed in v8.0.0.
* Will be removed in a future version.
*/
skipCountsFile?: boolean
@ -206,7 +205,7 @@ export class GcsStorage extends BaseStorage {
this.accessKeyId = options.accessKeyId
this.secretAccessKey = options.secretAccessKey
// v7.3.0: Handle initMode and deprecated skip* flags
// Handle initMode and deprecated skip* flags
if (options.initMode) {
this.initMode = options.initMode
}
@ -215,14 +214,14 @@ export class GcsStorage extends BaseStorage {
if (options.skipInitialScan) {
console.warn(
'[GcsStorage] DEPRECATION WARNING: skipInitialScan is deprecated. ' +
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
'Use initMode: "progressive" instead. Will be removed in a future version.'
)
this.skipInitialScan = true
}
if (options.skipCountsFile) {
console.warn(
'[GcsStorage] DEPRECATION WARNING: skipCountsFile is deprecated. ' +
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
'Use initMode: "progressive" instead. Will be removed in a future version.'
)
this.skipCountsFile = true
}
@ -240,13 +239,13 @@ export class GcsStorage extends BaseStorage {
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// v6.2.7: Write buffering always enabled - no env var check needed
// Write buffering always enabled - no env var check needed
}
/**
* Initialize the storage adapter
*
* v7.3.0: Supports progressive initialization for fast cold starts
* Supports progressive initialization for fast cold starts
*
* | Mode | Init Time | When |
* |------|-----------|------|
@ -337,14 +336,14 @@ export class GcsStorage extends BaseStorage {
}
)
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
// CRITICAL FIX: Clear any stale cache entries from previous runs
// This prevents cache poisoning from causing silent failures on container restart
prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning')
this.nounCacheManager.clear()
this.verbCacheManager.clear()
prodLog.info('✅ Cache cleared - starting fresh')
// v7.3.0: Progressive vs Strict initialization
// Progressive vs Strict initialization
if (effectiveMode === 'progressive') {
// PROGRESSIVE MODE: Fast init, background validation
// Mark as initialized immediately - ready to accept operations
@ -353,7 +352,7 @@ export class GcsStorage extends BaseStorage {
prodLog.info(`✅ GCS progressive init complete: ${this.bucketName} (validation deferred)`)
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Schedule background tasks (non-blocking)
@ -373,7 +372,7 @@ export class GcsStorage extends BaseStorage {
await this.initializeCounts()
this.countsLoaded = true
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Mark background tasks as complete (nothing to do in background)
@ -386,7 +385,7 @@ export class GcsStorage extends BaseStorage {
}
// =============================================
// Progressive Initialization (v7.3.0+)
// Progressive Initialization
// =============================================
/**
@ -400,7 +399,6 @@ export class GcsStorage extends BaseStorage {
*
* @protected
* @override
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
const startTime = Date.now()
@ -423,7 +421,6 @@ export class GcsStorage extends BaseStorage {
* Stores result in bucketValidated/bucketValidationError for lazy use.
*
* @private
* @since v7.3.0
*/
private async validateBucketInBackground(): Promise<void> {
try {
@ -451,7 +448,6 @@ export class GcsStorage extends BaseStorage {
* Uses the existing initializeCounts() logic but in background.
*
* @private
* @since v7.3.0
*/
private async loadCountsInBackground(): Promise<void> {
try {
@ -475,7 +471,6 @@ export class GcsStorage extends BaseStorage {
* @throws Error if bucket does not exist or is not accessible
* @protected
* @override
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do
@ -562,7 +557,7 @@ export class GcsStorage extends BaseStorage {
}
/**
* Override base class to enable smart batching for cloud storage (v3.32.3+)
* Override base class to enable smart batching for cloud storage
*
* GCS is cloud storage with network latency (~50ms per write).
* Smart batching reduces writes from 1000 ops → 100 batches.
@ -596,7 +591,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Flush noun buffer to GCS
@ -628,20 +623,20 @@ export class GcsStorage extends BaseStorage {
await Promise.all(writes)
}
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -657,10 +652,10 @@ export class GcsStorage extends BaseStorage {
/**
* Save a node directly to GCS (bypass buffer)
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
*/
private async saveNodeDirect(node: HNSWNode): Promise<void> {
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
// Apply backpressure before starting operation
@ -695,14 +690,14 @@ export class GcsStorage extends BaseStorage {
resumable: false // For small objects, non-resumable is faster
})
// CRITICAL FIX (v3.37.8): Only cache nodes with non-empty vectors
// CRITICAL FIX: Only cache nodes with non-empty vectors
// This prevents cache pollution from HNSW's lazy-loading nodes (vector: [])
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
// Note: Empty vectors are intentional during HNSW lazy mode - not logged
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveNounMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.logger.trace(`Node ${node.id} saved successfully`)
@ -721,7 +716,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Get a node from storage
@ -732,7 +727,7 @@ export class GcsStorage extends BaseStorage {
// Check cache first
const cached: HNSWNode | null = await this.nounCacheManager.get(id)
// Validate cached object before returning (v3.37.8+)
// Validate cached object before returning
if (cached !== undefined && cached !== null) {
// Validate cached object has required fields (including non-empty vector!)
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
@ -831,18 +826,18 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Write an object to a specific path in GCS
* Primitive operation required by base class
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
* @protected
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -892,7 +887,7 @@ export class GcsStorage extends BaseStorage {
}
/**
* Batch read multiple objects from GCS (v5.12.0 - Cloud Storage Optimization)
* Batch read multiple objects from GCS
*
* **Performance**: GCS-optimized parallel downloads
* - Uses Promise.all() for concurrent requests
@ -908,7 +903,6 @@ export class GcsStorage extends BaseStorage {
* @returns Map of path → data (only successful reads included)
*
* @public - Called by baseStorage.readBatchFromAdapter()
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -960,12 +954,11 @@ export class GcsStorage extends BaseStorage {
}
/**
* Get GCS-specific batch configuration (v5.12.0)
* Get GCS-specific batch configuration
*
* GCS performs well with high concurrency due to HTTP/2 multiplexing
*
* @public - Overrides BaseStorage.getBatchConfig()
* @since v5.12.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -984,12 +977,12 @@ export class GcsStorage extends BaseStorage {
* Delete an object from a specific path in GCS
* Primitive operation required by base class
*
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
* Performs lazy bucket validation on first delete in progressive mode.
* @protected
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -1034,20 +1027,20 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Save an edge to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.verbWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
this.verbCacheManager.set(edge.id, edge)
await this.verbWriteBuffer.add(edge.id, edge)
@ -1061,10 +1054,10 @@ export class GcsStorage extends BaseStorage {
/**
* Save an edge directly to GCS (bypass buffer)
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
*/
private async saveEdgeDirect(edge: Edge): Promise<void> {
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
const requestId = await this.applyBackpressure()
@ -1073,7 +1066,7 @@ export class GcsStorage extends BaseStorage {
this.logger.trace(`Saving edge ${edge.id}`)
// Convert connections Map to serializable format
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
@ -1085,7 +1078,7 @@ export class GcsStorage extends BaseStorage {
])
),
// CORE RELATIONAL DATA (v3.50.1+)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -1107,7 +1100,7 @@ export class GcsStorage extends BaseStorage {
// Update cache
this.verbCacheManager.set(edge.id, edge)
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.logger.trace(`Edge ${edge.id} saved successfully`)
@ -1125,7 +1118,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Get an edge from storage
@ -1161,7 +1154,7 @@ export class GcsStorage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
const edge: Edge = {
id: data.id,
vector: data.vector,
@ -1172,7 +1165,7 @@ export class GcsStorage extends BaseStorage {
sourceId: data.sourceId,
targetId: data.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
@ -1201,13 +1194,13 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed pagination overrides - use BaseStorage's type-first implementation
// Removed pagination overrides - use BaseStorage's type-first implementation
// - getNounsWithPagination, getNodesWithPagination, getVerbsWithPagination
// - getNouns, getVerbs (public wrappers)
// v5.4.0: Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation
// Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation
// (getNounsByNounType_internal, getVerbsBySource_internal, getVerbsByTarget_internal, getVerbsByType_internal)
/**
@ -1280,7 +1273,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.11.0: Clear ALL data using correct paths
// Clear ALL data using correct paths
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
await deleteObjectsWithPrefix('branches/')
@ -1290,7 +1283,7 @@ export class GcsStorage extends BaseStorage {
// Delete system metadata
await deleteObjectsWithPrefix('_system/')
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1352,12 +1345,12 @@ export class GcsStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker object exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -1413,7 +1406,7 @@ export class GcsStorage extends BaseStorage {
}
} catch (error: any) {
if (error.code === 404) {
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
// CRITICAL FIX: Statistics file doesn't exist yet (first restart)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
this.logger.trace('Statistics file not found - returning minimal stats with counts')
@ -1607,11 +1600,11 @@ export class GcsStorage extends BaseStorage {
}
}
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
/**
* Get a noun's vector for HNSW rebuild
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -1621,7 +1614,7 @@ export class GcsStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Uses mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -1630,7 +1623,7 @@ export class GcsStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -1650,7 +1643,7 @@ export class GcsStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -1672,7 +1665,7 @@ export class GcsStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -1683,7 +1676,7 @@ export class GcsStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -1713,7 +1706,7 @@ export class GcsStorage extends BaseStorage {
* Save HNSW system data (entry point, max level)
* Storage path: system/hnsw-system.json
*
* CRITICAL FIX (v4.10.1): Optimistic locking with generation numbers to prevent race conditions
* CRITICAL FIX: Optimistic locking with generation numbers to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -1807,7 +1800,7 @@ export class GcsStorage extends BaseStorage {
}
// ============================================================================
// GCS Lifecycle Management & Autoclass (v4.0.0)
// GCS Lifecycle Management & Autoclass
// Cost optimization through automatic tier transitions and Autoclass
// ============================================================================

10
src/storage/adapters/historicalStorageAdapter.ts
View file

@ -25,7 +25,7 @@
* - Lazy loading: only loads entities when accessed
* - No eager-loading of entire commit state
*
* v5.4.0: Production-ready, billion-scale historical queries
* Production-ready, billion-scale historical queries
*/
import { BaseStorage } from '../baseStorage.js'
@ -144,7 +144,7 @@ export class HistoricalStorageAdapter extends BaseStorage {
*/
public async init(): Promise<void> {
// Get COW components from underlying storage
// v6.2.4: Fixed property names - use public properties without underscore prefix
// Fixed property names - use public properties without underscore prefix
this.commitLog = this.underlyingStorage.commitLog
this.blobStorage = this.underlyingStorage.blobStorage
@ -161,7 +161,7 @@ export class HistoricalStorageAdapter extends BaseStorage {
throw new Error(`Commit not found: ${this.commitId}`)
}
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
}
@ -310,12 +310,12 @@ export class HistoricalStorageAdapter extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
* No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
* @returns Always false (read-only adapter doesn't manage COW state)
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/

49
src/storage/adapters/memoryStorage.ts
View file

@ -24,7 +24,7 @@ import { PaginatedResult } from '../../types/paginationTypes.js'
* Uses Maps to store data in memory
*/
export class MemoryStorage extends BaseStorage {
// v5.4.0: Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths
// Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths
private statistics: StatisticsData | null = null
// Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata)
@ -55,7 +55,6 @@ export class MemoryStorage extends BaseStorage {
* - Parallel processing maximizes throughput
*
* @returns Memory-optimized batch configuration
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -72,27 +71,27 @@ export class MemoryStorage extends BaseStorage {
/**
* Initialize the storage adapter
* v6.0.0: Calls super.init() to initialize GraphAdjacencyIndex and type statistics
* Calls super.init() to initialize GraphAdjacencyIndex and type statistics
*/
public async init(): Promise<void> {
await super.init()
}
// v5.4.0: Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation
// Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation
/**
* Get nouns with pagination and filtering
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
* Returns HNSWNounWithMetadata[] (includes metadata field)
* @param options Pagination and filtering options
* @returns Promise that resolves to a paginated result of nouns with metadata
*/
// v5.4.0: Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation
// Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation
// v5.4.0: Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation
// Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation
// v5.4.0: Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation
// Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation
// v5.4.0: Removed verb *_internal method overrides - using BaseStorage's type-first implementation
// Removed verb *_internal method overrides - using BaseStorage's type-first implementation
/**
* Primitive operation: Write object to path
@ -159,8 +158,8 @@ export class MemoryStorage extends BaseStorage {
/**
* Clear all data from storage
* v5.4.0: Clears objectStore (type-first paths)
* v7.3.1: Also clears writeCache to prevent stale data after clear
* Clears objectStore (type-first paths)
* Also clears writeCache to prevent stale data after clear
*/
public async clear(): Promise<void> {
this.objectStore.clear()
@ -174,7 +173,7 @@ export class MemoryStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v7.3.1: Clear write-through cache (inherited from BaseStorage)
// Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data
// after clear(), causing "ghost" entities to appear
this.clearWriteCache()
@ -182,7 +181,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Get information about storage usage and capacity
* v5.4.0: Uses BaseStorage counts
* Uses BaseStorage counts
*/
public async getStorageStatus(): Promise<{
type: string
@ -204,12 +203,12 @@ export class MemoryStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: No-op for MemoryStorage (doesn't persist)
* No-op for MemoryStorage (doesn't persist)
* @returns Always false (marker doesn't persist in memory)
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -252,7 +251,7 @@ export class MemoryStorage extends BaseStorage {
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
if (!this.statistics) {
// CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init)
// CRITICAL FIX: Statistics don't exist yet (first init)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
@ -299,10 +298,10 @@ export class MemoryStorage extends BaseStorage {
}
/**
* Initialize counts from in-memory storage - O(1) operation (v4.0.0)
* Initialize counts from in-memory storage - O(1) operation
*/
protected async initializeCounts(): Promise<void> {
// v6.0.0: Scan objectStore paths (ID-first structure) to count entities
// Scan objectStore paths (ID-first structure) to count entities
this.entityCounts.clear()
this.verbCounts.clear()
@ -314,14 +313,14 @@ export class MemoryStorage extends BaseStorage {
// Count nouns (entities/nouns/{shard}/{id}/vectors.json)
const nounMatch = path.match(/^entities\/nouns\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
if (nounMatch) {
// v6.0.0: Type is in metadata, not path - just count total
// Type is in metadata, not path - just count total
totalNouns++
}
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
const verbMatch = path.match(/^entities\/verbs\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
if (verbMatch) {
// v6.0.0: Type is in metadata, not path - just count total
// Type is in metadata, not path - just count total
totalVerbs++
}
}
@ -339,26 +338,26 @@ export class MemoryStorage extends BaseStorage {
}
// =============================================
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
// =============================================
/**
* Get vector for a noun
* v5.4.0: Uses BaseStorage's type-first implementation
* Uses BaseStorage's type-first implementation
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
return noun ? [...noun.vector] : null
}
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
// CRITICAL FIX: Mutex locks for HNSW concurrency control
// Even in-memory operations need serialization to prevent async race conditions
private hnswLocks = new Map<string, Promise<void>>()
/**
* Save HNSW graph data for a noun
*
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions during concurrent HNSW updates
* CRITICAL FIX: Mutex locking to prevent race conditions during concurrent HNSW updates
* Even in-memory operations can race due to async/await interleaving
* Prevents data corruption when multiple entities connect to same neighbor simultaneously
*/
@ -417,7 +416,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Save HNSW system data (entry point, max level)
*
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions
* CRITICAL FIX: Mutex locking to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null

43
src/storage/adapters/opfsStorage.ts
View file

@ -57,7 +57,7 @@ const ROOT_DIR = 'opfs-vector-db'
* OPFS storage adapter for browser environments
* Uses the Origin Private File System API to store data persistently
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -97,7 +97,6 @@ export class OPFSStorage extends BaseStorage {
* - Sequential processing preferred for stability
*
* @returns OPFS-optimized batch configuration
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -172,7 +171,7 @@ export class OPFSStorage extends BaseStorage {
// Initialize counts from storage
await this.initializeCounts()
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
} catch (error) {
console.error('Failed to initialize OPFS storage:', error)
@ -232,7 +231,7 @@ export class OPFSStorage extends BaseStorage {
}
}
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
/**
* Delete an edge from storage
@ -482,14 +481,14 @@ export class OPFSStorage extends BaseStorage {
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir!)
// v5.6.1: Remove COW (copy-on-write) version control data
// Remove COW (copy-on-write) version control data
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
// Must be deleted to fully clear all data including version history
try {
// Delete the entire _cow/ directory (not just contents)
await this.rootDir!.removeEntry('_cow', { recursive: true })
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -505,7 +504,7 @@ export class OPFSStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
// Reset entity counters (inherited from BaseStorageAdapter)
// These in-memory counters must be reset to 0 after clearing all data
;(this as any).totalNounCount = 0
;(this as any).totalVerbCount = 0
@ -517,16 +516,16 @@ export class OPFSStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker file exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
// Quota monitoring configuration (v4.0.0)
// Quota monitoring configuration
private quotaWarningThreshold = 0.8 // Warn at 80% usage
private quotaCriticalThreshold = 0.95 // Critical at 95% usage
private lastQuotaCheck: number = 0
@ -675,7 +674,7 @@ export class OPFSStorage extends BaseStorage {
}
/**
* Get detailed quota status with warnings (v4.0.0)
* Get detailed quota status with warnings
* Monitors storage usage and warns when approaching quota limits
*
* @returns Promise that resolves to quota status with warning levels
@ -769,7 +768,7 @@ export class OPFSStorage extends BaseStorage {
}
/**
* Monitor quota during operations (v4.0.0)
* Monitor quota during operations
* Automatically checks quota at regular intervals and warns if approaching limits
* Call this before write operations to ensure quota is available
*
@ -1185,7 +1184,7 @@ export class OPFSStorage extends BaseStorage {
}
}
} catch (error) {
// CRITICAL FIX (v3.37.4): No statistics files exist (first init)
// CRITICAL FIX: No statistics files exist (first init)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
@ -1228,7 +1227,7 @@ export class OPFSStorage extends BaseStorage {
* @param options Pagination and filter options
* @returns Promise that resolves to a paginated result of nouns
*/
// v5.4.0: Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
// Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
/**
* Initialize counts from OPFS storage
@ -1313,28 +1312,28 @@ export class OPFSStorage extends BaseStorage {
}
}
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
/**
* Get a noun's vector for HNSW rebuild
*/
/**
* Get vector for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
return noun ? noun.vector : null
}
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
// CRITICAL FIX: Mutex locks for HNSW concurrency control
// Browser environments are single-threaded but async operations can still interleave
private hnswLocks = new Map<string, Promise<void>>()
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -1354,7 +1353,7 @@ export class OPFSStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
const existingNoun = await this.getNoun(nounId)
if (!existingNoun) {
@ -1374,7 +1373,7 @@ export class OPFSStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths)
// Use BaseStorage's saveNoun (type-first paths)
await this.saveNoun(updatedNoun)
} finally {
// Release lock
@ -1385,7 +1384,7 @@ export class OPFSStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -1415,7 +1414,7 @@ export class OPFSStorage extends BaseStorage {
* Save HNSW system data (entry point, max level)
* Storage path: index/hnsw-system.json
*
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions
* CRITICAL FIX: Mutex locking to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null

4
src/storage/adapters/optimizedS3Search.ts
View file

@ -107,7 +107,7 @@ export class OptimizedS3Search {
}
// Determine if there are more items
const hasMore = listResult.hasMore || nouns.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop)
const hasMore = listResult.hasMore || nouns.length > limit // Fixed >= to > (was causing infinite loop)
// Set next cursor
let nextCursor: string | undefined
@ -190,7 +190,7 @@ export class OptimizedS3Search {
}
// Determine if there are more items
const hasMore = listResult.hasMore || verbs.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop)
const hasMore = listResult.hasMore || verbs.length > limit // Fixed >= to > (was causing infinite loop)
// Set next cursor
let nextCursor: string | undefined

49
src/storage/adapters/r2Storage.ts
View file

@ -58,7 +58,7 @@ const MAX_R2_PAGE_SIZE = 1000
* Dedicated Cloudflare R2 storage adapter
* Optimized for R2's unique characteristics and global edge network
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed getNounsWithPagination override
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -112,7 +112,7 @@ export class R2Storage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Multi-level cache manager for efficient data access
@ -122,7 +122,7 @@ export class R2Storage extends BaseStorage {
// Module logger
private logger = createModuleLogger('R2Storage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
/**
@ -168,7 +168,7 @@ export class R2Storage extends BaseStorage {
})
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// v6.2.7: Write buffering always enabled - no env var check needed
// Write buffering always enabled - no env var check needed
}
/**
@ -183,7 +183,7 @@ export class R2Storage extends BaseStorage {
* Zero egress fees enable aggressive caching and parallel downloads
*
* @returns R2-optimized batch configuration
* @since v5.12.0 - Updated for native batch API
* Updated for native batch API
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -208,7 +208,6 @@ export class R2Storage extends BaseStorage {
*
* @param paths - Array of R2 object keys to read
* @returns Map of path -> parsed JSON data (only successful reads)
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -336,7 +335,7 @@ export class R2Storage extends BaseStorage {
this.nounCacheManager.clear()
this.verbCacheManager.clear()
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
} catch (error) {
this.logger.error('Failed to initialize R2 storage:', error)
@ -409,7 +408,7 @@ export class R2Storage extends BaseStorage {
}
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Flush noun buffer to R2
@ -444,16 +443,16 @@ export class R2Storage extends BaseStorage {
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -728,14 +727,14 @@ export class R2Storage extends BaseStorage {
/**
* Save an edge to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.verbWriteBuffer) {
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
this.verbCacheManager.set(edge.id, edge)
await this.verbWriteBuffer.add(edge.id, edge)
@ -750,7 +749,7 @@ export class R2Storage extends BaseStorage {
const requestId = await this.applyBackpressure()
try {
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
@ -762,7 +761,7 @@ export class R2Storage extends BaseStorage {
])
),
// CORE RELATIONAL DATA (v3.50.1+)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -785,7 +784,7 @@ export class R2Storage extends BaseStorage {
this.verbCacheManager.set(edge.id, edge)
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.releaseBackpressure(true, requestId)
@ -831,7 +830,7 @@ export class R2Storage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
const edge: Edge = {
id: data.id,
vector: data.vector,
@ -842,7 +841,7 @@ export class R2Storage extends BaseStorage {
sourceId: data.sourceId,
targetId: data.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
@ -1081,7 +1080,7 @@ export class R2Storage extends BaseStorage {
prodLog.info('🧹 R2: Clearing all data from bucket...')
// v5.11.0: Clear ALL data using correct paths
// Clear ALL data using correct paths
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
const branchObjects = await this.listObjectsUnderPath('branches/')
for (const key of branchObjects) {
@ -1100,7 +1099,7 @@ export class R2Storage extends BaseStorage {
await this.deleteObjectFromPath(key)
}
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1143,17 +1142,17 @@ export class R2Storage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker object exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
// Removed getNounsWithPagination override - use BaseStorage's type-first implementation
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
}

98
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -85,7 +85,7 @@ type S3Command = any
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
* - bucketName: GCS bucket name
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -152,7 +152,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Operation executors for timeout and retry handling
@ -165,7 +165,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Module logger
private logger = createModuleLogger('S3Storage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
/**
@ -210,7 +210,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Lambda, Cloud Run),
* strict locally. Zero-config optimization.
@ -219,7 +219,6 @@ export class S3CompatibleStorage extends BaseStorage {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: InitMode
@ -236,7 +235,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.serviceType = options.serviceType || 's3'
this.readOnly = options.readOnly || false
// v7.3.0: Handle initMode
// Handle initMode
if (options.initMode) {
this.initMode = options.initMode
}
@ -270,7 +269,7 @@ export class S3CompatibleStorage extends BaseStorage {
* S3 supports ~5000 operations/second with burst capacity up to 10,000
*
* @returns S3-optimized batch configuration
* @since v5.12.0 - Updated for native batch API
* Updated for native batch API
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -295,7 +294,6 @@ export class S3CompatibleStorage extends BaseStorage {
*
* @param paths - Array of S3 object keys to read
* @returns Map of path -> parsed JSON data (only successful reads)
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -358,7 +356,7 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Initialize the storage adapter
*
* v7.3.0: Supports progressive initialization for fast cold starts
* Supports progressive initialization for fast cold starts
*
* | Mode | Init Time | When |
* |------|-----------|------|
@ -514,7 +512,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Initialize request coalescer
this.initializeCoalescer()
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
// CRITICAL FIX: Clear any stale cache entries from previous runs
// This prevents cache poisoning from causing silent failures on container restart
const nodeCacheSize = this.nodeCache?.size || 0
if (nodeCacheSize > 0) {
@ -524,7 +522,7 @@ export class S3CompatibleStorage extends BaseStorage {
prodLog.info('🧹 Node cache is empty - starting fresh')
}
// v7.3.0: Progressive vs Strict initialization
// Progressive vs Strict initialization
if (effectiveMode === 'progressive') {
// PROGRESSIVE MODE: Fast init, background validation
// Mark as initialized immediately - ready to accept operations
@ -533,7 +531,7 @@ export class S3CompatibleStorage extends BaseStorage {
prodLog.info(`✅ S3 progressive init complete: ${this.bucketName} (validation deferred)`)
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Schedule background tasks (non-blocking)
@ -556,7 +554,7 @@ export class S3CompatibleStorage extends BaseStorage {
await this.initializeCounts()
this.countsLoaded = true
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Mark background tasks as complete (nothing to do in background)
@ -573,7 +571,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
// =============================================
// Progressive Initialization (v7.3.0+)
// Progressive Initialization
// =============================================
/**
@ -588,7 +586,6 @@ export class S3CompatibleStorage extends BaseStorage {
*
* @protected
* @override
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
const startTime = Date.now()
@ -614,7 +611,6 @@ export class S3CompatibleStorage extends BaseStorage {
* Stores result in bucketValidated/bucketValidationError for lazy use.
*
* @private
* @since v7.3.0
*/
private async validateBucketInBackground(): Promise<void> {
try {
@ -638,7 +634,6 @@ export class S3CompatibleStorage extends BaseStorage {
* Load counts from storage in background.
*
* @private
* @since v7.3.0
*/
private async loadCountsInBackground(): Promise<void> {
try {
@ -662,7 +657,6 @@ export class S3CompatibleStorage extends BaseStorage {
* @throws Error if bucket does not exist or is not accessible
* @protected
* @override
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do
@ -918,7 +912,7 @@ export class S3CompatibleStorage extends BaseStorage {
)
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Bulk write nouns to S3
@ -1179,20 +1173,20 @@ export class S3CompatibleStorage extends BaseStorage {
return this.socketManager.getBatchSize()
}
// v5.4.0: Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
// Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -1242,7 +1236,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.logger.debug(`Node ${node.id} saved successfully`)
// Log the change for efficient synchronization (v4.0.0: no metadata on node)
// Log the change for efficient synchronization (no metadata on node)
await this.appendToChangeLog({
timestamp: Date.now(),
operation: 'add', // Could be 'update' if we track existing nodes
@ -1250,7 +1244,7 @@ export class S3CompatibleStorage extends BaseStorage {
entityId: node.id,
data: {
vector: node.vector
// ✅ NO metadata field in v4.0.0 - stored separately
// ✅ NO metadata field - stored separately
}
})
@ -1296,7 +1290,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNoun_internal override - uses BaseStorage type-first implementation
// Removed getNoun_internal override - uses BaseStorage type-first implementation
/**
* Get a node from storage
@ -1307,7 +1301,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Check cache first
const cached = this.nodeCache.get(id)
// Validate cached object before returning (v3.37.8+)
// Validate cached object before returning
if (cached !== undefined && cached !== null) {
// Validate cached object has required fields (including non-empty vector!)
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
@ -1607,7 +1601,7 @@ export class S3CompatibleStorage extends BaseStorage {
return nodes
}
// v5.4.0: Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal)
// Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal)
// Now inherit from BaseStorage's type-first implementation
@ -1788,23 +1782,23 @@ export class S3CompatibleStorage extends BaseStorage {
return true // Return all edges since filtering requires metadata
}
// v5.4.0: Removed getVerbsWithPagination override - use BaseStorage's type-first implementation
// Removed getVerbsWithPagination override - use BaseStorage's type-first implementation
// v5.4.0: Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb)
// Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb)
// Total: 8 *_internal methods removed - all now inherit from BaseStorage's type-first implementation
/**
* Primitive operation: Write object to path
* All metadata operations use this internally via base class routing
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
// Apply backpressure before starting operation
@ -1897,11 +1891,11 @@ export class S3CompatibleStorage extends BaseStorage {
* Primitive operation: Delete object from path
* All metadata operations use this internally via base class routing
*
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
* Performs lazy bucket validation on first delete in progressive mode.
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -2315,7 +2309,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// v5.11.0: Clear ALL data using correct paths
// Clear ALL data using correct paths
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
await deleteObjectsWithPrefix('branches/')
@ -2325,7 +2319,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Delete system metadata
await deleteObjectsWithPrefix('_system/')
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -2335,7 +2329,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
// Reset entity counters (inherited from BaseStorageAdapter)
// These in-memory counters must be reset to 0 after clearing all data
;(this as any).totalNounCount = 0
;(this as any).totalVerbCount = 0
@ -2457,12 +2451,12 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker object exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -2851,7 +2845,7 @@ export class S3CompatibleStorage extends BaseStorage {
totalEdges: this.totalVerbCount
}
}
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
// CRITICAL FIX: Statistics file doesn't exist yet (first restart)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
@ -3415,7 +3409,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
// Removed getNounsWithPagination override - use BaseStorage's type-first implementation
/**
* Estimate total noun count by listing objects across all shards
@ -3549,7 +3543,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Override base class to enable smart batching for cloud storage (v3.32.3+)
* Override base class to enable smart batching for cloud storage
*
* S3 is cloud storage with network latency (~50ms per write).
* Smart batching reduces writes from 1000 ops → 100 batches.
@ -3560,11 +3554,11 @@ export class S3CompatibleStorage extends BaseStorage {
return true // S3 benefits from batching
}
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
/**
* Get a noun's vector for HNSW rebuild
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -3574,7 +3568,7 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Uses mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -3583,7 +3577,7 @@ export class S3CompatibleStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -3603,7 +3597,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -3625,7 +3619,7 @@ export class S3CompatibleStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -3636,7 +3630,7 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -3666,7 +3660,7 @@ export class S3CompatibleStorage extends BaseStorage {
* Save HNSW system data (entry point, max level)
* Storage path: system/hnsw-system.json
*
* CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
* CRITICAL FIX: Optimistic locking with ETags to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -3781,7 +3775,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Set S3 lifecycle policy for automatic tier transitions and deletions (v4.0.0)
* Set S3 lifecycle policy for automatic tier transitions and deletions
* Automates cost optimization by moving old data to cheaper storage classes
*
* S3 Storage Classes:
@ -3976,7 +3970,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Enable S3 Intelligent-Tiering for automatic cost optimization (v4.0.0)
* Enable S3 Intelligent-Tiering for automatic cost optimization
* Automatically moves objects between access tiers based on usage patterns
*
* Intelligent-Tiering automatically saves up to 95% on storage costs:

4
src/storage/backwardCompatibility.ts
View file

@ -1,6 +1,6 @@
/**
* DEPRECATED (v4.7.2): Backward compatibility stubs
* TODO: Remove in v4.7.3 after migrating s3CompatibleStorage
* DEPRECATED: Backward compatibility stubs
* TODO: Remove after migrating s3CompatibleStorage
*/
export class StorageCompatibilityLayer {

339
src/storage/baseStorage.ts
View file

File diff suppressed because it is too large Load diff

14
src/storage/cow/BlobStorage.ts
View file

@ -161,7 +161,7 @@ export class BlobStorage {
}
/**
* v5.7.5: Ensure compression is ready before write operations
* Ensure compression is ready before write operations
* Fixes race condition where write happens before async compression init completes
*/
private async ensureCompressionReady(): Promise<void> {
@ -206,7 +206,7 @@ export class BlobStorage {
return hash
}
// v5.7.5: Ensure compression is initialized before writing
// Ensure compression is initialized before writing
// Fixes race condition where write happens before async init completes
await this.ensureCompressionReady()
@ -223,7 +223,7 @@ export class BlobStorage {
}
// Create metadata
// v5.7.5: Store ACTUAL compression state, not intended
// Store ACTUAL compression state, not intended
// Prevents corruption if compression failed to initialize
const actualCompression = finalData === data ? 'none' : compression
const metadata: BlobMetadata = {
@ -277,7 +277,7 @@ export class BlobStorage {
* @returns Blob data
*/
async read(hash: string, options: BlobReadOptions = {}): Promise<Buffer> {
// v5.3.4 fix: Guard against NULL hash (sentinel value)
// Guard against NULL hash (sentinel value)
// NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree"
// It should NEVER be read as actual blob data
if (isNullHash(hash)) {
@ -309,7 +309,7 @@ export class BlobStorage {
metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`)
if (metadataBuffer) {
prefix = tryPrefix
// v5.10.1: Unwrap metadata before parsing (defense-in-depth)
// Unwrap metadata before parsing (defense-in-depth)
// Metadata should be JSON, but adapter might return wrapped format
const unwrappedMetadata = unwrapBinaryData(metadataBuffer)
metadata = JSON.parse(unwrappedMetadata.toString())
@ -338,8 +338,8 @@ export class BlobStorage {
finalData = await this.zstdDecompress(data)
}
// v5.10.1: Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
// Even though COW adapter should unwrap (v5.7.5), verify it happened and re-unwrap if needed
// Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
// Even though COW adapter should unwrap, verify it happened and re-unwrap if needed
// This prevents "Blob integrity check failed" errors if adapter returns wrapped data
// Uses shared binaryDataCodec utility (single source of truth for unwrap logic)
const unwrappedData = unwrapBinaryData(finalData)

2
src/storage/cow/CommitLog.ts
View file

@ -43,7 +43,7 @@ export interface CommitLogStats {
/**
* CommitLog: Efficient commit history traversal and querying
*
* Pure v5.0.0 implementation - modern, clean, fast
* Pure implementation - modern, clean, fast
*/
export class CommitLog {
private blobStorage: BlobStorage

2
src/storage/cow/CommitObject.ts
View file

@ -337,7 +337,7 @@ export class CommitObject {
let currentHash: string | null = startHash
let depth = 0
// v5.3.4 fix: Guard against NULL hash (sentinel for "no parent")
// Guard against NULL hash (sentinel for "no parent")
// The initial commit has parent = null or NULL_HASH ('0000...0000')
// We must stop walking when we reach it, not try to read it
while (currentHash && !isNullHash(currentHash)) {

2
src/storage/cow/RefManager.ts
View file

@ -55,7 +55,7 @@ export interface RefUpdateOptions {
/**
* RefManager: Manages branches, tags, and HEAD pointer
*
* Pure implementation for v5.0.0 - no backward compatibility
* Pure implementation - no backward compatibility
*/
export class RefManager {
private adapter: COWStorageAdapter

4
src/storage/cow/binaryDataCodec.ts
View file

@ -86,7 +86,7 @@ export function unwrapBinaryData(data: any): Buffer {
/**
* Wrap binary data for JSON storage
*
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH!
* ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
* ⚠️ This function exists for legacy/compatibility only.
*
@ -94,7 +94,7 @@ export function unwrapBinaryData(data: any): Buffer {
* This is FRAGILE because compressed binary can accidentally parse as valid JSON,
* causing blob integrity failures.
*
* v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
* SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
* - Keys with '-meta:' or 'ref:' prefix → Always JSON
* - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
* No guessing needed!

23
src/storage/storageFactory.ts
View file

@ -10,7 +10,7 @@ import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js'
import { R2Storage } from './adapters/r2Storage.js'
import { GcsStorage } from './adapters/gcsStorage.js'
import { AzureBlobStorage } from './adapters/azureBlobStorage.js'
// TypeAwareStorageAdapter removed in v5.4.0 - type-aware now built into BaseStorage
// TypeAwareStorageAdapter removed - type-aware now built into BaseStorage
// FileSystemStorage is dynamically imported to avoid issues in browser environments
import { isBrowser } from '../utils/environment.js'
import { OperationConfig } from '../utils/operationUtils.js'
@ -94,7 +94,7 @@ export interface StorageOptions {
sessionToken?: string
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Lambda),
* strict locally. Zero-config optimization.
@ -103,7 +103,6 @@ export interface StorageOptions {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: 'progressive' | 'strict' | 'auto'
}
@ -215,7 +214,7 @@ export interface StorageOptions {
skipCountsFile?: boolean
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Cloud Run),
* strict locally. Zero-config optimization.
@ -224,7 +223,6 @@ export interface StorageOptions {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: 'progressive' | 'strict' | 'auto'
}
@ -259,7 +257,7 @@ export interface StorageOptions {
sasToken?: string
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
* strict locally. Zero-config optimization.
@ -268,7 +266,6 @@ export interface StorageOptions {
* - `'strict'`: Traditional blocking init. Validates container and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: 'progressive' | 'strict' | 'auto'
}
@ -388,7 +385,7 @@ export interface StorageOptions {
/**
* COW (Copy-on-Write) configuration for instant fork() capability
* v5.0.1: COW is now always enabled (automatic, zero-config)
* COW is now always enabled (automatic, zero-config)
*/
branch?: string // Current branch name (default: 'main')
enableCompression?: boolean // Enable zstd compression for COW blobs (default: true)
@ -410,14 +407,14 @@ function getFileSystemPath(options: StorageOptions): string {
}
/**
* Configure COW (Copy-on-Write) options on a storage adapter (v5.4.0)
* Configure COW (Copy-on-Write) options on a storage adapter
* TypeAware is now built-in to all adapters, no wrapper needed!
*
* @param storage - The storage adapter
* @param options - Storage options (for COW configuration)
*/
function configureCOW(storage: any, options?: StorageOptions): void {
// v5.0.1: COW will be initialized AFTER storage.init() in Brainy
// COW will be initialized AFTER storage.init() in Brainy
// Store COW options for later initialization
if (typeof storage.initializeCOW === 'function') {
storage._cowOptions = {
@ -672,10 +669,10 @@ export async function createStorage(
}
case 'type-aware':
// v5.0.0: TypeAware is now the default for ALL adapters
// TypeAware is now the default for ALL adapters
// Redirect to the underlying type instead
console.warn(
'⚠️ type-aware is deprecated in v5.0.0 - TypeAware is now always enabled.'
'⚠️ type-aware is deprecated - TypeAware is now always enabled.'
)
console.warn(
' Just use the underlying storage type (e.g., "filesystem", "s3", etc.)'
@ -866,7 +863,7 @@ export async function createStorage(
}
/**
* Export storage adapters (v5.4.0: TypeAware is now built-in, no separate export)
* Export storage adapters (TypeAware is now built-in, no separate export)
*/
export {
MemoryStorage,

6
src/types/brainyInterface.ts
View file

@ -18,7 +18,7 @@ export interface BrainyInterface<T = unknown> {
init(): Promise<void>
/**
* Promise that resolves when initialization is complete (v7.3.0+)
* Promise that resolves when initialization is complete
* Can be awaited multiple times safely.
*/
readonly ready: Promise<void>
@ -29,12 +29,12 @@ export interface BrainyInterface<T = unknown> {
readonly isInitialized: boolean
/**
* Check if all initialization including background tasks is complete (v7.3.0+)
* Check if all initialization including background tasks is complete
*/
isFullyInitialized(): boolean
/**
* Wait for all background initialization tasks to complete (v7.3.0+)
* Wait for all background initialization tasks to complete
* For cloud storage adapters, this waits for bucket validation and count sync.
*/
awaitBackgroundInit(): Promise<void>

4
src/types/graphTypes.ts
View file

@ -303,7 +303,7 @@
* }
* ```
*
* ## Stage 3 Changes from v5.4.0
* ## Stage 3 Changes
*
* ### Nouns Added (+11)
* agent, quality, timeInterval, function, proposition, socialGroup, institution,
@ -323,7 +323,7 @@
* createdBy (use inverse of creates), supervises (use inverse of reportsTo)
*
* **Net Change**: +9 nouns (31 → 40), +48 verbs (40 → 88) = +57 types total
* **Coverage**: 60% (v5.4.0) → 95% (Stage 3)
* **Coverage**: 60% → 95% (Stage 3)
*/
// Common metadata types

8
src/types/typeMigration.ts
View file

@ -1,5 +1,5 @@
/**
* Type Migration Utilities for Stage 3 Taxonomy (v6.0.0)
* Type Migration Utilities for Stage 3 Taxonomy
*
* Provides migration helpers for code using removed types from v5.x
*
@ -52,7 +52,7 @@ export function isRemovedVerbType(type: string): type is keyof typeof REMOVED_VE
}
/**
* Migrate a noun type from v5.x to v6.0 Stage 3
* Migrate a noun type (Stage 3)
* Returns the migrated type or the original if no migration needed
*/
export function migrateNounType(type: string): NounType {
@ -64,7 +64,7 @@ export function migrateNounType(type: string): NounType {
}
/**
* Migrate a verb type from v5.x to v6.0 Stage 3
* Migrate a verb type (Stage 3)
* Returns the migrated type or the original if no migration needed
*
* WARNING: Some verbs require inverting source/target relationships!
@ -138,7 +138,7 @@ export function migrateRelationship(params: {
/**
* Stage 3 Type Compatibility Check
* Helps developers identify code that needs updating for v6.0
* Helps developers identify code that needs updating
*/
export function checkTypeCompatibility(nounTypes: string[], verbTypes: string[]): {
valid: boolean

6
src/utils/entityIdMapper.ts
View file

@ -54,7 +54,7 @@ export class EntityIdMapper {
async init(): Promise<void> {
try {
const metadata = await this.storage.getMetadata(this.storageKey)
// v4.8.0: metadata IS the data (no nested 'data' property)
// metadata IS the data (no nested 'data' property)
if (metadata && (metadata as any).nextId !== undefined) {
const data = metadata as any as EntityIdMapperData
this.nextId = data.nextId
@ -87,7 +87,7 @@ export class EntityIdMapper {
}
/**
* v7.5.0: Get integer ID for UUID with immediate persistence guarantee
* Get integer ID for UUID with immediate persistence guarantee
* Unlike getOrAssign(), this method flushes to storage immediately after assigning
* a new ID. This prevents UUID→int mapping divergence if the process crashes
* before a normal flush() occurs.
@ -194,7 +194,7 @@ export class EntityIdMapper {
}
// Convert maps to plain objects for serialization
// v4.0.0: Add required 'noun' property for NounMetadata
// Add required 'noun' property for NounMetadata
const data = {
noun: 'EntityIdMapper',
nextId: this.nextId,

6
src/utils/fieldTypeInference.ts
View file

@ -388,7 +388,7 @@ export class FieldTypeInference {
const data = await this.storage.getMetadata(cacheKey)
if (data) {
// v4.0.0: Double cast for type boundary crossing
// Double cast for type boundary crossing
return data as unknown as FieldTypeInfo
}
} catch (error) {
@ -406,7 +406,7 @@ export class FieldTypeInference {
this.typeCache.set(field, typeInfo)
// Save to persistent storage (async, non-blocking)
// v4.0.0: Add required 'noun' property for NounMetadata
// Add required 'noun' property for NounMetadata
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
const metadataObj = {
noun: 'FieldTypeCache',
@ -487,7 +487,7 @@ export class FieldTypeInference {
if (field) {
this.typeCache.delete(field)
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
// v4.0.0: null signals deletion to storage adapter
// null signals deletion to storage adapter
await this.storage.saveMetadata(cacheKey, null as any)
} else {
this.typeCache.clear()

3
src/utils/import-progress-tracker.ts
View file

@ -1,5 +1,5 @@
/**
* Import Progress Tracker (v4.5.0)
* Import Progress Tracker
*
* Comprehensive progress tracking for imports with:
* - Multi-dimensional progress (bytes, entities, stages, timing)
@ -8,7 +8,6 @@
* - Throttled callbacks (avoid spam)
* - Weighted overall progress
*
* @since v4.5.0
*/
import {

6
src/utils/memoryDetection.ts
View file

@ -49,7 +49,7 @@ export interface CacheAllocationStrategy {
/** Environment type detected */
environment: 'production' | 'development' | 'container' | 'unknown'
/** Model memory reserved (bytes) - v3.36.0+ */
/** Model memory reserved (bytes) */
modelMemory: number
/** Model precision (q8 or fp32) */
@ -195,7 +195,7 @@ function detectCgroupV1Memory(): number | null {
* Calculate optimal cache size based on available memory
* Scales intelligently from 2GB to 128GB+
*
* v3.36.0+: Accounts for embedding model memory (150MB Q8, 250MB FP32)
* Accounts for embedding model memory (150MB Q8, 250MB FP32)
*/
export function calculateOptimalCacheSize(
memoryInfo: MemoryInfo,
@ -219,7 +219,7 @@ export function calculateOptimalCacheSize(
const minSize = options.minSize || 256 * 1024 * 1024 // 256MB minimum
const maxSize = options.maxSize || null
// Detect model memory usage (v3.36.0+)
// Detect model memory usage
const modelInfo = detectModelMemory({ precision: options.modelPrecision || 'q8' })
const modelMemory = modelInfo.bytes

2
src/utils/metadataFilter.ts
View file

@ -323,7 +323,7 @@ export function filterSearchResultsByMetadata<T>(
/**
* Filter nouns by metadata before search
* v4.0.0: Takes HNSWNounWithMetadata which includes metadata field
* Takes HNSWNounWithMetadata which includes metadata field
*/
export function filterNounsByMetadata(
nouns: HNSWNounWithMetadata[],

209
src/utils/metadataIndex.ts
View file

@ -76,7 +76,7 @@ interface FieldStats {
rangeQueryCount: number
exactQueryCount: number
avgQueryTime: number
indexType: 'hash' // v3.42.0: Only 'hash' since all fields use chunked sparse indices with zone maps
indexType: 'hash' // Only 'hash' since all fields use chunked sparse indices with zone maps
normalizationStrategy?: 'none' | 'precision' | 'bucket'
}
@ -121,19 +121,19 @@ export class MetadataIndexManager {
private lockPromises = new Map<string, Promise<boolean>>()
private lockTimers = new Map<string, NodeJS.Timeout>() // Track timers for cleanup
// Adaptive Chunked Sparse Indexing (v3.42.0 → v3.44.1)
// Adaptive Chunked Sparse Indexing
// Reduces file count from 560k → 89 files (630x reduction)
// ALL fields now use chunking - no more flat files
// v3.44.1: Removed sparseIndices Map - now lazy-loaded via UnifiedCache only
// Removed sparseIndices Map - now lazy-loaded via UnifiedCache only
// PROJECTED: Reduces metadata memory from 35GB → 5GB @ 1B scale (86% reduction from chunking strategy, not yet benchmarked)
private chunkManager: ChunkManager
private chunkingStrategy: AdaptiveChunkingStrategy
// Roaring Bitmap Support (v3.43.0)
// Roaring Bitmap Support
// EntityIdMapper for UUID ↔ integer conversion
private idMapper: EntityIdMapper
// Field Type Inference (v3.48.0 - Production-ready value-based type detection)
// Field Type Inference (Production-ready value-based type detection)
// Replaces unreliable pattern matching with DuckDB-inspired value analysis
private fieldTypeInference: FieldTypeInference
@ -179,20 +179,20 @@ export class MetadataIndexManager {
// Get global unified cache for coordinated memory management
this.unifiedCache = getGlobalCache()
// Initialize EntityIdMapper for roaring bitmap UUID ↔ integer mapping (v3.43.0)
// Initialize EntityIdMapper for roaring bitmap UUID ↔ integer mapping
this.idMapper = new EntityIdMapper({
storage,
storageKey: 'brainy:entityIdMapper'
})
// Initialize chunking system (v3.42.0) with roaring bitmap support
// Initialize chunking system with roaring bitmap support
this.chunkManager = new ChunkManager(storage, this.idMapper)
this.chunkingStrategy = new AdaptiveChunkingStrategy()
// Initialize Field Type Inference (v3.48.0)
// Initialize Field Type Inference
this.fieldTypeInference = new FieldTypeInference(storage)
// v6.2.2: Removed lazyLoadCounts() call from constructor
// Removed lazyLoadCounts() call from constructor
// It was a race condition (not awaited) and read from wrong source.
// Now properly called in init() after warmCache() loads the sparse index.
}
@ -205,18 +205,18 @@ export class MetadataIndexManager {
// Initialize roaring-wasm library (browser bundle requires async init)
await roaringLibraryInitialize()
// Load field registry to discover persisted indices (v4.2.1)
// Load field registry to discover persisted indices
// Must run first to populate fieldIndexes directory before warming cache
await this.loadFieldRegistry()
// Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
await this.idMapper.init()
// Warm the cache with common fields (v3.44.1 - lazy loading optimization)
// Warm the cache with common fields (lazy loading optimization)
// This loads the 'noun' sparse index which is needed for type counts
await this.warmCache()
// v6.2.2: Load type counts AFTER warmCache (sparse index is now cached)
// Load type counts AFTER warmCache (sparse index is now cached)
// Previously called in constructor without await and read from wrong source
await this.lazyLoadCounts()
@ -224,16 +224,16 @@ export class MetadataIndexManager {
// Now correctly happens AFTER lazyLoadCounts() finishes
this.syncTypeCountsToFixed()
// v7.5.0: Detect index corruption and auto-rebuild if necessary
// Detect index corruption and auto-rebuild if necessary
// The update() field asymmetry bug caused indexes to accumulate stale entries
// This check runs on startup to detect and repair corrupted indexes automatically
await this.detectAndRepairCorruption()
}
/**
* v7.5.0: Detect index corruption and automatically repair via rebuild
* Detect index corruption and automatically repair via rebuild
* This catches the update() field asymmetry bug that causes 7 fields to accumulate per update
* Corruption threshold: 100 avg entries/entity (expected ~30)
* Corruption threshold: 100 avg metadata entries/entity, excluding __words__ (expected ~30)
*/
private async detectAndRepairCorruption(): Promise<void> {
const validation = await this.validateConsistency()
@ -260,7 +260,7 @@ export class MetadataIndexManager {
}
/**
* Warm the cache by preloading common field sparse indices (v3.44.1)
* Warm the cache by preloading common field sparse indices
* This improves cache hit rates by loading frequently-accessed fields at startup
* Target: >80% cache hit rate for typical workloads
*/
@ -413,18 +413,18 @@ export class MetadataIndexManager {
/**
* Lazy load entity counts from the 'noun' field sparse index (O(n) where n = number of types)
* v6.2.2 FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed
* FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed
* Now computes counts from the sparse index which has the correct type information
*/
private async lazyLoadCounts(): Promise<void> {
try {
// v6.2.4: CRITICAL FIX - Clear counts before loading to prevent accumulation
// CRITICAL FIX - Clear counts before loading to prevent accumulation
// Previously, counts accumulated across restarts causing 100x inflation
this.totalEntitiesByType.clear()
this.entityCountsByTypeFixed.fill(0)
this.verbCountsByTypeFixed.fill(0)
// v6.2.2: Load counts from sparse index (correct source)
// Load counts from sparse index (correct source)
const nounSparseIndex = await this.loadSparseIndex('noun')
if (!nounSparseIndex) {
// No sparse index yet - counts will be populated as entities are added
@ -522,7 +522,7 @@ export class MetadataIndexManager {
const stats = this.fieldStats.get(field)!
const cardinality = stats.cardinality
// Track unique values by checking fieldIndex counts (v3.42.0 - removed indexCache)
// Track unique values by checking fieldIndex counts
const fieldIndex = this.fieldIndexes.get(field)
const normalizedValue = this.normalizeValue(value, field)
const currentCount = fieldIndex?.values[normalizedValue] || 0
@ -581,7 +581,7 @@ export class MetadataIndexManager {
private updateIndexStrategy(field: string, stats: FieldStats): void {
const hasHighCardinality = stats.cardinality.uniqueValues > this.HIGH_CARDINALITY_THRESHOLD
// All fields use chunked sparse indexing with zone maps (v3.42.0)
// All fields use chunked sparse indexing with zone maps
stats.indexType = 'hash'
// Determine normalization strategy for high cardinality NON-temporal fields
@ -603,7 +603,7 @@ export class MetadataIndexManager {
}
// ============================================================================
// Adaptive Chunked Sparse Indexing (v3.42.0)
// Adaptive Chunked Sparse Indexing
// All fields use chunking - simplified implementation
// ============================================================================
@ -652,8 +652,8 @@ export class MetadataIndexManager {
}
/**
* Get IDs for a value using chunked sparse index with roaring bitmaps (v3.43.0)
* v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* Get IDs for a value using chunked sparse index with roaring bitmaps
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
*/
private async getIdsFromChunks(field: string, value: any): Promise<string[]> {
// Load sparse index via UnifiedCache (lazy loading)
@ -690,9 +690,9 @@ export class MetadataIndexManager {
}
/**
* Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps (v3.43.0)
* v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* v4.5.4: Normalize min/max for timestamp bucketing before comparison
* Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* Normalize min/max for timestamp bucketing before comparison
*/
private async getIdsFromChunksForRange(
field: string,
@ -707,7 +707,7 @@ export class MetadataIndexManager {
return [] // No chunked index exists yet
}
// v4.5.4: Normalize min/max for consistent comparison with indexed values
// Normalize min/max for consistent comparison with indexed values
// (indexed values are bucketed for timestamps, so we must bucket the query bounds too)
const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined
const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined
@ -751,9 +751,9 @@ export class MetadataIndexManager {
}
/**
* Get roaring bitmap for a field-value pair without converting to UUIDs (v3.43.0)
* Get roaring bitmap for a field-value pair without converting to UUIDs
* This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
* v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* @returns RoaringBitmap32 containing integer IDs, or null if no matches
*/
private async getBitmapFromChunks(field: string, value: any): Promise<RoaringBitmap32 | null> {
@ -806,7 +806,7 @@ export class MetadataIndexManager {
}
/**
* Get IDs for multiple field-value pairs using fast roaring bitmap intersection (v3.43.0)
* Get IDs for multiple field-value pairs using fast roaring bitmap intersection
*
* This method provides 500-900x faster multi-field queries by:
* - Using hardware-accelerated bitmap AND operations (SIMD: AVX2/SSE4.2)
@ -862,7 +862,7 @@ export class MetadataIndexManager {
/**
* Add value-ID mapping to chunked index
* v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
*/
private async addToChunkedIndex(field: string, value: any, id: string): Promise<void> {
// Load or create sparse index via UnifiedCache (lazy loading)
@ -964,7 +964,7 @@ export class MetadataIndexManager {
/**
* Remove ID from chunked index
* v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
*/
private async removeFromChunkedIndex(field: string, value: any, id: string): Promise<void> {
// Load sparse index via UnifiedCache (lazy loading)
@ -1013,7 +1013,7 @@ export class MetadataIndexManager {
stats.rangeQueryCount++
}
// All fields use chunked sparse index with zone map optimization (v3.42.0)
// All fields use chunked sparse index with zone map optimization
return await this.getIdsFromChunksForRange(field, min, max, includeMin, includeMax)
}
@ -1047,7 +1047,7 @@ export class MetadataIndexManager {
/**
* Normalize value for consistent indexing with VALUE-BASED temporal detection
*
* v3.48.0: Replaced unreliable field name pattern matching with production-ready
* Replaced unreliable field name pattern matching with production-ready
* value-based detection (DuckDB-inspired). Analyzes actual data values, not names.
*
* NO FALLBACKS - Pure value-based detection only.
@ -1153,14 +1153,14 @@ export class MetadataIndexManager {
/**
* Extract indexable field-value pairs from entity or metadata
*
* v4.8.0: Now handles BOTH entity structure (with top-level fields) AND plain metadata
* Now handles BOTH entity structure (with top-level fields) AND plain metadata
* - Extracts from top-level fields (confidence, weight, timestamps, type, service, etc.)
* - Also extracts from nested metadata field (custom user fields)
* - Skips HNSW-specific fields (vector, connections, level, id)
* - Maps 'type' → 'noun' for backward compatibility with existing indexes
*
* BUG FIX (v3.50.1): Exclude vector embeddings and large arrays from indexing
* BUG FIX (v3.50.2): Also exclude purely numeric field names (array indices)
* BUG FIX: Exclude vector embeddings and large arrays from indexing
* BUG FIX: Also exclude purely numeric field names (array indices)
* - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities
* - Arrays converted to objects with numeric keys were still being indexed
*/
@ -1186,7 +1186,7 @@ export class MetadataIndexManager {
if (!this.shouldIndexField(fullKey)) continue
// Special handling for metadata field at top level
// v4.8.0: Flatten metadata fields to top-level (no prefix) for cleaner queries
// Flatten metadata fields to top-level (no prefix) for cleaner queries
// Standard fields are already at top-level, custom fields go in metadata
// By flattening here, queries can use { category: 'B' } instead of { 'metadata.category': 'B' }
if (key === 'metadata' && !prefix && typeof value === 'object' && !Array.isArray(value)) {
@ -1211,7 +1211,7 @@ export class MetadataIndexManager {
}
} else {
// Primitive value: index it
// v4.8.0: Map 'type' → 'noun' for backward compatibility
// Map 'type' → 'noun' for backward compatibility
const indexField = (!prefix && key === 'type') ? 'noun' : fullKey
fields.push({ field: indexField, value })
}
@ -1222,8 +1222,8 @@ export class MetadataIndexManager {
extract(data)
}
// v7.7.0: Extract words for hybrid text search
// v7.8.0: Production-scale word limit (5000 words)
// Extract words for hybrid text search
// Production-scale word limit (5000 words)
// - Handles articles, chapters, and large documents
// - Roaring Bitmaps + Chunked Sparse Index + LRU caching
// - Int32 hashes store words as 4-byte values, not strings
@ -1258,7 +1258,7 @@ export class MetadataIndexManager {
}
/**
* Extract text content from entity data for word indexing (v7.7.0)
* Extract text content from entity data for word indexing
*
* Recursively extracts string values from data, excluding:
* - vector, embedding, connections, level, id (internal fields)
@ -1292,7 +1292,7 @@ export class MetadataIndexManager {
}
/**
* Tokenize text into words for indexing (v7.7.0)
* Tokenize text into words for indexing
*
* - Converts to lowercase
* - Removes punctuation
@ -1314,7 +1314,7 @@ export class MetadataIndexManager {
}
/**
* Hash word to int32 using FNV-1a (v7.7.0)
* Hash word to int32 using FNV-1a
*
* FNV-1a is fast with low collision rate, suitable for word hashing.
* Saves ~10GB at billion scale by avoiding string storage.
@ -1332,7 +1332,7 @@ export class MetadataIndexManager {
}
/**
* Get entity IDs matching a text query (v7.7.0)
* Get entity IDs matching a text query
*
* Performs word-based text search using the __words__ index.
* Returns IDs ranked by match count (entities with more matching words first).
@ -1375,19 +1375,19 @@ export class MetadataIndexManager {
/**
* Add item to metadata indexes
*
* v4.8.0: Now accepts either entity structure or plain metadata
* Now accepts either entity structure or plain metadata
* - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} }
* - Plain metadata: { noun, confidence, weight, createdAt, ... }
*
* @param id - Entity ID
* @param entityOrMetadata - Either full entity structure (v4.8.0+) or plain metadata (backward compat)
* @param entityOrMetadata - Either full entity structure or plain metadata (backward compat)
* @param skipFlush - Skip automatic flush (used during batch operations)
*/
async addToIndex(id: string, entityOrMetadata: any, skipFlush: boolean = false): Promise<void> {
const fields = this.extractIndexableFields(entityOrMetadata)
// v6.7.0: Sanity check for excessive indexed fields (indicates possible data issue)
// v7.8.0: Separate threshold for metadata fields vs word fields
// Sanity check for excessive indexed fields (indicates possible data issue)
// Separate threshold for metadata fields vs word fields
// - Metadata fields: warn if > 100 (indicates deeply nested metadata)
// - Word fields: expected to be many for large documents, warn only for extreme cases
const metadataFields = fields.filter(f => f.field !== '__words__')
@ -1418,7 +1418,7 @@ export class MetadataIndexManager {
for (let i = 0; i < fields.length; i++) {
const { field, value } = fields[i]
// All fields use chunked sparse indexing (v3.42.0)
// All fields use chunked sparse indexing
await this.addToChunkedIndex(field, value, id)
// Update statistics and tracking
@ -1432,7 +1432,7 @@ export class MetadataIndexManager {
}
}
// Adaptive auto-flush based on usage patterns (v3.42.0 - flush field indexes only)
// Adaptive auto-flush based on usage patterns
if (!skipFlush) {
const timeSinceLastFlush = Date.now() - this.lastFlushTime
const shouldAutoFlush =
@ -1494,7 +1494,7 @@ export class MetadataIndexManager {
/**
* Remove item from metadata indexes
*
* v4.8.0: Now accepts either entity structure or plain metadata (same as addToIndex)
* Now accepts either entity structure or plain metadata (same as addToIndex)
* - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} }
* - Plain metadata: { noun, confidence, weight, createdAt, ... }
*
@ -1507,7 +1507,7 @@ export class MetadataIndexManager {
const fields = this.extractIndexableFields(metadata)
for (const { field, value } of fields) {
// All fields use chunked sparse indexing (v3.42.0)
// All fields use chunked sparse indexing
await this.removeFromChunkedIndex(field, value, id)
// Update statistics and tracking
@ -1521,7 +1521,7 @@ export class MetadataIndexManager {
} else {
// Remove from all indexes (slower, requires scanning all field indexes)
// This should be rare - prefer providing metadata when removing
// v3.44.1: Scan via fieldIndexes, load sparse indices on-demand
// Scan via fieldIndexes, load sparse indices on-demand
prodLog.warn(`Removing ID ${id} without metadata requires scanning all fields (slow)`)
// Scan all fields via fieldIndexes
@ -1552,7 +1552,7 @@ export class MetadataIndexManager {
* Get all IDs in the index
*/
async getAllIds(): Promise<string[]> {
// Use storage as the source of truth (v3.42.0 - removed redundant indexCache scan)
// Use storage as the source of truth
const allIds = new Set<string>()
// Storage.getNouns() is the definitive source of all entity IDs
@ -1586,7 +1586,7 @@ export class MetadataIndexManager {
stats.exactQueryCount++
}
// All fields use chunked sparse indexing (v3.42.0)
// All fields use chunked sparse indexing
return await this.getIdsFromChunks(field, value)
}
@ -1739,7 +1739,7 @@ export class MetadataIndexManager {
subIds.forEach(id => unionIds.add(id))
}
// v6.2.1: Fix - Check for outer-level field conditions that need AND application
// Fix - Check for outer-level field conditions that need AND application
// This handles cases like { anyOf: [...], vfsType: { exists: false } }
// where the anyOf results must be intersected with other field conditions
const outerFields = Object.keys(filter).filter(
@ -1770,21 +1770,21 @@ export class MetadataIndexManager {
let fieldResults: string[] = []
if (condition && typeof condition === 'object' && !Array.isArray(condition)) {
// Handle Brainy Field Operators (v4.5.4: canonical operators defined)
// Handle Brainy Field Operators (canonical operators defined)
// See docs/api/README.md for complete operator reference
for (const [op, operand] of Object.entries(condition)) {
switch (op) {
// ===== EQUALITY OPERATORS =====
// Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is' (remove in v5.0.0)
case 'is': // DEPRECATED (v4.5.4): Use 'eq' instead
// Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is'
case 'is': // DEPRECATED: Use 'eq' instead
case 'equals': // Alias for 'eq'
case 'eq':
fieldResults = await this.getIds(field, operand)
break
// ===== NEGATION OPERATORS =====
// Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot' (remove in v5.0.0)
case 'isNot': // DEPRECATED (v4.5.4): Use 'ne' instead
// Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot'
case 'isNot': // DEPRECATED: Use 'ne' instead
case 'notEquals': // Alias for 'ne'
case 'ne':
// For notEquals, we need all IDs EXCEPT those matching the value
@ -1824,8 +1824,8 @@ export class MetadataIndexManager {
break
// ===== GREATER THAN OR EQUAL OPERATORS =====
// Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual' (remove in v5.0.0)
case 'greaterEqual': // DEPRECATED (v4.5.4): Use 'gte' instead
// Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual'
case 'greaterEqual': // DEPRECATED: Use 'gte' instead
case 'greaterThanOrEqual': // Alias for 'gte'
case 'gte':
fieldResults = await this.getIdsForRange(field, operand, undefined, true, true)
@ -1839,8 +1839,8 @@ export class MetadataIndexManager {
break
// ===== LESS THAN OR EQUAL OPERATORS =====
// Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual' (remove in v5.0.0)
case 'lessEqual': // DEPRECATED (v4.5.4): Use 'lte' instead
// Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual'
case 'lessEqual': // DEPRECATED: Use 'lte' instead
case 'lessThanOrEqual': // Alias for 'lte'
case 'lte':
fieldResults = await this.getIdsForRange(field, undefined, operand, true, true)
@ -1865,8 +1865,8 @@ export class MetadataIndexManager {
case 'exists':
if (operand) {
// exists: true - Get all IDs that have this field (any value)
// v3.43.0: From chunked sparse index with roaring bitmaps
// v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
// From chunked sparse index with roaring bitmaps
// Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
const allIntIds = new Set<number>()
// Load sparse index via UnifiedCache (lazy loading)
@ -1890,7 +1890,7 @@ export class MetadataIndexManager {
fieldResults = this.idMapper.intsIterableToUuids(allIntIds)
} else {
// exists: false - Get all IDs that DON'T have this field
// v5.7.9: Fixed excludeVFS bug (was returning empty array)
// Fixed excludeVFS bug (was returning empty array)
const allItemIds = await this.getAllIds()
const existsIntIds = new Set<number>()
@ -1921,7 +1921,7 @@ export class MetadataIndexManager {
case 'missing':
// missing: true is equivalent to exists: false
// missing: false is equivalent to exists: true
// v5.7.9: Added for API consistency with in-memory metadataFilter
// Added for API consistency with in-memory metadataFilter
if (operand) {
// missing: true - field does NOT exist (same as exists: false)
const allItemIds = await this.getAllIds()
@ -2021,7 +2021,6 @@ export class MetadataIndexManager {
* @param order - Sort direction: 'asc' (default) or 'desc'
* @returns Promise<string[]> - Entity IDs sorted by specified field
*
* @since v4.5.4
*/
async getSortedIdsForFilter(
filter: any,
@ -2084,7 +2083,6 @@ export class MetadataIndexManager {
* @returns Promise<any> - Field value or undefined if not found
*
* @public (called from brainy.ts for sorted queries)
* @since v4.5.4
*/
async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
// For timestamp fields, load ACTUAL value from entity metadata
@ -2152,7 +2150,6 @@ export class MetadataIndexManager {
* @returns Denormalized value in original type
*
* @private
* @since v4.5.4
*/
private denormalizeValue(normalized: string, field: string): any {
// Try parsing as number (timestamps, integers, floats)
@ -2233,7 +2230,7 @@ export class MetadataIndexManager {
/**
* Flush dirty entries to storage (non-blocking version)
* NOTE (v3.42.0): Sparse indices are flushed immediately in add/remove operations
* NOTE: Sparse indices are flushed immediately in add/remove operations
*/
async flush(): Promise<void> {
// Check if we have anything to flush
@ -2245,7 +2242,7 @@ export class MetadataIndexManager {
const BATCH_SIZE = 20
const allPromises: Promise<void>[] = []
// Flush field indexes in batches (v3.42.0 - removed flat file flushing)
// Flush field indexes in batches
const dirtyFieldsArray = Array.from(this.dirtyFields)
for (let i = 0; i < dirtyFieldsArray.length; i += BATCH_SIZE) {
const batch = dirtyFieldsArray.slice(i, i + BATCH_SIZE)
@ -2264,10 +2261,10 @@ export class MetadataIndexManager {
// Wait for all operations to complete
await Promise.all(allPromises)
// Flush EntityIdMapper (UUID ↔ integer mappings) (v3.43.0)
// Flush EntityIdMapper (UUID ↔ integer mappings)
await this.idMapper.flush()
// Save field registry for fast cold-start discovery (v4.2.1)
// Save field registry for fast cold-start discovery
await this.saveFieldRegistry()
this.dirtyFields.clear()
@ -2347,7 +2344,7 @@ export class MetadataIndexManager {
const indexId = `__metadata_field_index__${filename}`
const unifiedKey = `metadata:field:${filename}`
// v4.0.0: Add required 'noun' property for NounMetadata
// Add required 'noun' property for NounMetadata
await this.storage.saveMetadata(indexId, {
noun: 'MetadataFieldIndex',
values: fieldIndex.values,
@ -2369,7 +2366,7 @@ export class MetadataIndexManager {
/**
* Save field registry to storage for fast cold-start discovery
* v4.2.1: Solves 100x performance regression by persisting field directory
* Solves 100x performance regression by persisting field directory
*
* This enables instant cold starts by discovering which fields have persisted indices
* without needing to rebuild from scratch. Similar to how HNSW persists system metadata.
@ -2404,7 +2401,7 @@ export class MetadataIndexManager {
/**
* Load field registry from storage to populate fieldIndexes directory
* v4.2.1: Enables O(1) discovery of persisted sparse indices
* Enables O(1) discovery of persisted sparse indices
*
* Called during init() to discover which fields have persisted indices.
* Populates fieldIndexes Map with skeleton entries - actual sparse indices
@ -2449,7 +2446,7 @@ export class MetadataIndexManager {
/**
* Get list of persisted fields from storage (not in-memory)
* v6.7.0: Used during rebuild to discover which chunk files need deletion
* Used during rebuild to discover which chunk files need deletion
*
* @returns Array of field names that have persisted sparse indices
*/
@ -2470,7 +2467,7 @@ export class MetadataIndexManager {
/**
* Delete all chunk files for a specific field
* v6.7.0: Used during rebuild to ensure clean slate
* Used during rebuild to ensure clean slate
*
* @param field Field name whose chunks should be deleted
*/
@ -2500,7 +2497,7 @@ export class MetadataIndexManager {
/**
* Clear ALL metadata index data from storage (for recovery)
* v6.7.0: Nuclear option for recovering from corrupted index state
* Nuclear option for recovering from corrupted index state
*
* WARNING: This deletes all indexed data - requires full rebuild after!
* Use when index is corrupted beyond normal rebuild repair.
@ -2565,7 +2562,7 @@ export class MetadataIndexManager {
/**
* Get all entity types and their counts - O(1) operation
* v6.2.2: Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity
* Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity
* during add operations. lazyLoadCounts was reading wrong data but that doesn't
* affect freshly-added entities within the same session.
*/
@ -2574,7 +2571,7 @@ export class MetadataIndexManager {
}
// ============================================================================
// v6.2.1: VFS Statistics Methods (uses existing Roaring bitmap infrastructure)
// VFS Statistics Methods (uses existing Roaring bitmap infrastructure)
// ============================================================================
/**
@ -2744,14 +2741,14 @@ export class MetadataIndexManager {
* Get count of entities matching field-value criteria - queries chunked sparse index
*/
async getCountForCriteria(field: string, value: any): Promise<number> {
// Use chunked sparse indexing (v3.42.0 - removed indexCache)
// Use chunked sparse indexing
const ids = await this.getIds(field, value)
return ids.length
}
/**
* Get index statistics with enhanced counting information
* v3.44.1: Sparse indices now lazy-loaded via UnifiedCache
* Sparse indices now lazy-loaded via UnifiedCache
* Note: This method may load sparse indices to calculate stats
*/
async getStats(): Promise<MetadataIndexStats> {
@ -2759,8 +2756,9 @@ export class MetadataIndexManager {
let totalEntries = 0
let totalIds = 0
// Collect stats from field indexes (lightweight - always in memory)
// Collect stats from metadata field indexes only (excludes __words__ keyword index)
for (const field of this.fieldIndexes.keys()) {
if (field === '__words__') continue // Keyword index not included in metadata stats
fields.add(field)
// Load sparse index to count entries (may trigger lazy load)
@ -2779,7 +2777,7 @@ export class MetadataIndexManager {
}
}
// v6.7.0: Sanity check for index corruption (77x overcounting bug detection)
// Sanity check for index corruption (only metadata fields, not __words__ keyword index)
const entityCount = this.idMapper.size
if (entityCount > 0) {
const avgIdsPerEntity = totalIds / entityCount
@ -2801,11 +2799,12 @@ export class MetadataIndexManager {
}
/**
* v7.5.0: Validate index consistency and detect corruption
* Validate index consistency and detect corruption
* Returns health status and recommendations for repair
*
* Counts metadata field entries only (excludes __words__ keyword index).
* Corruption typically manifests as high avg entries/entity (expected ~30, corrupted can be 100+)
* caused by the update() field asymmetry bug (fixed in v7.5.0)
* caused by the update() field asymmetry bug
*/
async validateConsistency(): Promise<{
healthy: boolean
@ -2827,9 +2826,10 @@ export class MetadataIndexManager {
}
}
// Count total index entries across all fields
// Count total index entries across all fields (excluding keyword index)
let indexEntryCount = 0
for (const field of this.fieldIndexes.keys()) {
if (field === '__words__') continue // Keyword entries are expected to be high-volume
const sparseIndex = await this.loadSparseIndex(field)
if (sparseIndex) {
for (const chunkId of sparseIndex.getAllChunkIds()) {
@ -2845,7 +2845,8 @@ export class MetadataIndexManager {
const avgEntriesPerEntity = indexEntryCount / entityCount
// Threshold: 100 entries/entity is clearly corrupted (expected ~30)
// Threshold: 100 metadata entries/entity is clearly corrupted (expected ~30)
// __words__ keyword entries are excluded from this count since they can be 50-5000 per entity
// This catches the update() asymmetry bug which causes 7 fields to accumulate per update
const CORRUPTION_THRESHOLD = 100
const healthy = avgEntriesPerEntity <= CORRUPTION_THRESHOLD
@ -2868,7 +2869,7 @@ export class MetadataIndexManager {
/**
* Rebuild entire index from scratch using pagination
* Non-blocking version that yields control back to event loop
* v3.44.1: Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map)
* Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map)
*/
async rebuild(): Promise<void> {
if (this.isRebuilding) return
@ -2879,12 +2880,12 @@ export class MetadataIndexManager {
prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`)
prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`)
// Clear existing indexes (v3.42.0 - use sparse indices instead of flat files)
// v3.44.1: No sparseIndices Map to clear - UnifiedCache handles eviction
// Clear existing indexes
// No sparseIndices Map to clear - UnifiedCache handles eviction
this.fieldIndexes.clear()
this.dirtyFields.clear()
// v6.2.4: CRITICAL FIX - Clear type counts to prevent accumulation
// CRITICAL FIX - Clear type counts to prevent accumulation
// Previously, counts accumulated across rebuilds causing incorrect values
this.totalEntitiesByType.clear()
this.entityCountsByTypeFixed.fill(0)
@ -2892,12 +2893,12 @@ export class MetadataIndexManager {
this.typeFieldAffinity.clear()
// Clear all cached sparse indices in UnifiedCache
// This ensures rebuild starts fresh (v3.44.1)
// This ensures rebuild starts fresh
this.unifiedCache.clear('metadata')
// v6.7.0: CRITICAL FIX - Delete existing chunk files from storage
// CRITICAL FIX - Delete existing chunk files from storage
// Without this, old chunk data accumulates with each rebuild causing 77x overcounting!
// Previous fix (v6.2.4) cleared type counts but missed chunk file accumulation.
// Previous fix cleared type counts but missed chunk file accumulation.
prodLog.info('🗑️ Clearing existing metadata index chunks from storage...')
const existingFields = await this.getPersistedFieldList()
@ -2916,13 +2917,13 @@ export class MetadataIndexManager {
prodLog.info(`✅ Cleared ${existingFields.length} field indexes from storage`)
}
// Clear EntityIdMapper to start fresh (v6.7.0)
// Clear EntityIdMapper to start fresh
await this.idMapper.clear()
// Clear chunk manager cache
this.chunkManager.clearCache()
// Adaptive rebuild strategy based on storage adapter (v4.2.3)
// Adaptive rebuild strategy based on storage adapter
// FileSystem/Memory/OPFS: Load all at once (avoids getAllShardedFiles() overhead on every batch)
// Cloud (GCS/S3/R2): Use pagination with small batches (prevent socket exhaustion)
const storageType = this.storage.constructor.name
@ -3394,7 +3395,7 @@ export class MetadataIndexManager {
// This is the type definition itself
entityType = this.normalizeValue(value, field) // Pass field for bucketing!
} else if (metadata && metadata.noun) {
// Extract entity type from metadata (v3.42.0 - removed indexCache scan)
// Extract entity type from metadata
entityType = this.normalizeValue(metadata.noun, 'noun')
} else {
// No type information available, skip affinity tracking

6
src/utils/metadataIndexChunking.ts
View file

@ -592,7 +592,7 @@ export class ChunkManager {
const data = await this.storage.getMetadata(chunkPath)
if (data) {
// v4.0.0: Cast NounMetadata to chunk data structure
// Cast NounMetadata to chunk data structure
const chunkData = data as unknown as any
// Deserialize: convert serialized roaring bitmaps back to RoaringBitmap32 objects
@ -633,7 +633,7 @@ export class ChunkManager {
this.chunkCache.set(cacheKey, chunk)
// Serialize: convert RoaringBitmap32 to portable format (Buffer)
// v4.0.0: Add required 'noun' property for NounMetadata
// Add required 'noun' property for NounMetadata
const serializable = {
noun: 'IndexChunk', // Required by NounMetadata interface
chunkId: chunk.chunkId,
@ -825,7 +825,7 @@ export class ChunkManager {
this.chunkCache.delete(cacheKey)
const chunkPath = this.getChunkPath(field, chunkId)
// v4.0.0: null signals deletion to storage adapter
// null signals deletion to storage adapter
await this.storage.saveMetadata(chunkPath, null as any)
}

2
src/utils/periodicCleanup.ts
View file

@ -215,7 +215,7 @@ export class PeriodicCleanup {
for (const noun of nounsResult.items) {
try {
// v4.0.0: Cast NounMetadata to NamespacedMetadata for isDeleted check
// Cast NounMetadata to NamespacedMetadata for isDeleted check
if (!noun.metadata || !isDeleted(noun.metadata as any)) {
continue // Not deleted, skip
}

14
src/utils/rebuildCounts.ts
View file

@ -2,7 +2,7 @@
* Rebuild Counts Utility
*
* Scans storage and rebuilds counts.json from actual data
* Use this to fix databases affected by the v4.1.1 count synchronization bug
* Use this to fix databases affected by the count synchronization bug
*
* NO MOCKS - Production-ready implementation
*/
@ -64,12 +64,12 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
}
let hasMore = true
let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop)
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
while (hasMore) {
const result: any = await storageWithPagination.getNounsWithPagination({
limit: 100,
offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored)
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
})
for (const noun of result.items) {
@ -82,7 +82,7 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
}
hasMore = result.hasMore
offset += 100 // v5.7.11: Increment offset for next page
offset += 100 // Increment offset for next page
}
console.log(` Found ${totalNouns} entities across ${entityCounts.size} types`)
@ -95,12 +95,12 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
}
hasMore = true
offset = 0 // v5.7.11: Reset offset for verbs pagination
offset = 0 // Reset offset for verbs pagination
while (hasMore) {
const result: any = await storageWithPagination.getVerbsWithPagination({
limit: 100,
offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored)
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
})
for (const verb of result.items) {
@ -112,7 +112,7 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
}
hasMore = result.hasMore
offset += 100 // v5.7.11: Increment offset for next page
offset += 100 // Increment offset for next page
}
console.log(` Found ${totalVerbs} relationships across ${verbCounts.size} types`)

24
src/utils/unifiedCache.ts
View file

@ -2,7 +2,7 @@
* UnifiedCache - Single cache for both HNSW and MetadataIndex
* Prevents resource competition with cost-aware eviction
*
* Features (v3.36.0+):
* Features:
* - Adaptive sizing: Automatically scales from 2GB to 128GB+ based on available memory
* - Container-aware: Detects Docker/K8s limits (cgroups v1/v2)
* - Environment detection: Production vs development allocation strategies
@ -67,14 +67,14 @@ export class UnifiedCache {
private readonly maxSize: number
private readonly config: UnifiedCacheConfig
// Memory management (v3.36.0+)
// Memory management
private readonly memoryInfo: MemoryInfo
private readonly allocationStrategy: CacheAllocationStrategy
private memoryPressureCheckTimer: NodeJS.Timeout | null = null
private lastMemoryWarning = 0
constructor(config: UnifiedCacheConfig = {}) {
// Adaptive cache sizing (v3.36.0+)
// Adaptive cache sizing
const recommendation = getRecommendedCacheConfig({
manualSize: config.maxSize,
minSize: config.minSize,
@ -85,7 +85,7 @@ export class UnifiedCache {
this.allocationStrategy = recommendation.allocation
this.maxSize = recommendation.allocation.cacheSize
// Log allocation decision (v3.36.0+: includes model memory)
// Log allocation decision (includes model memory)
prodLog.info(
`UnifiedCache initialized: ${formatBytes(this.maxSize)} ` +
`(${this.allocationStrategy.environment} mode, ` +
@ -113,7 +113,7 @@ export class UnifiedCache {
this.config = {
enableRequestCoalescing: true,
enableFairnessCheck: true,
fairnessCheckInterval: 30000, // Check fairness every 30 seconds (v3.40.2: was 60s)
fairnessCheckInterval: 30000, // Check fairness every 30 seconds (was 60s)
persistPatterns: true,
enableMemoryMonitoring: true,
memoryCheckInterval: 30000, // Check memory every 30s
@ -175,7 +175,7 @@ export class UnifiedCache {
}
/**
* Synchronous cache lookup (v3.36.0+)
* Synchronous cache lookup
* Returns cached data immediately or undefined if not cached
* Use for sync fast path optimization - zero async overhead
*/
@ -232,7 +232,7 @@ export class UnifiedCache {
this.typeAccessCounts[type]++
this.totalAccessCount++
// Proactive fairness check (v3.40.2): Check immediately if adding to a dominant type
// Proactive fairness check: Check immediately if adding to a dominant type
// This prevents imbalance formation instead of reacting to it
if (this.config.enableFairnessCheck && this.cache.size > 10) {
this.checkProactiveFairness(type)
@ -341,7 +341,7 @@ export class UnifiedCache {
other: typeSizes.other / totalSize
}
// Check for starvation (v3.40.2: more aggressive - 70% cache with <15% accesses)
// Check for starvation (more aggressive - 70% cache with <15% accesses)
// Previous: 90% cache, <10% access (too lenient, caused thrashing)
for (const type of ['hnsw', 'metadata', 'embedding', 'other'] as const) {
if (sizeRatios[type] > 0.7 && accessRatios[type] < 0.15) {
@ -352,7 +352,7 @@ export class UnifiedCache {
}
/**
* Proactive fairness check (v3.40.2)
* Proactive fairness check
* Called immediately when adding items to prevent imbalance formation
* Uses same thresholds as periodic check but runs on-demand
*/
@ -391,7 +391,7 @@ export class UnifiedCache {
// Sort by score (lower is worse)
candidates.sort((a, b) => a[1] - b[1])
// Evict bottom 50% of this type (v3.40.2: was 20%, too slow to prevent thrashing)
// Evict bottom 50% of this type (was 20%, too slow to prevent thrashing)
const evictCount = Math.max(1, Math.floor(candidates.length * 0.5))
for (let i = 0; i < evictCount && i < candidates.length; i++) {
@ -417,7 +417,7 @@ export class UnifiedCache {
/**
* Delete all items with keys starting with the given prefix
* v6.2.9: Added for VFS cache invalidation (fixes stale parent ID bug)
* Added for VFS cache invalidation (fixes stale parent ID bug)
* @param prefix - The key prefix to match
* @returns Number of items deleted
*/
@ -526,7 +526,7 @@ export class UnifiedCache {
totalAccessCount: this.totalAccessCount,
hitRate,
// Memory management (v3.36.0+)
// Memory management
memory: {
available: this.memoryInfo.available,
source: this.memoryInfo.source,

2
src/versioning/VersionDiff.ts
View file

@ -1,5 +1,5 @@
/**
* VersionDiff - Deep Object Comparison for Entity Versions (v5.3.0)
* VersionDiff - Deep Object Comparison for Entity Versions
*
* Provides deep diff between entity versions:
* - Field-level change detection

2
src/versioning/VersionIndex.ts
View file

@ -1,5 +1,5 @@
/**
* VersionIndex - Pure Key-Value Version Storage (v6.3.0)
* VersionIndex - Pure Key-Value Version Storage
*
* Stores version metadata using simple key-value storage:
* - Version list per entity: __version_meta_{entityId}_{branch}

10
src/versioning/VersionManager.ts
View file

@ -1,14 +1,14 @@
/**
* VersionManager - Entity-Level Versioning Engine (v5.3.0, v6.3.0 fix)
* VersionManager - Entity-Level Versioning Engine
*
* Provides entity-level version control with:
* - save() - Create entity version
* - restore() - Restore entity to specific version (v6.3.0: now updates all indexes)
* - restore() - Restore entity to specific version (now updates all indexes)
* - list() - List all versions of an entity
* - compare() - Deep diff between versions
* - prune() - Remove old versions (retention policies)
*
* Architecture (v6.3.0 - Clean Key-Value Storage):
* Architecture:
* - Versions stored as key-value pairs, NOT as entities (no index pollution)
* - Content-addressable: SHA-256 hashing for deduplication
* - Space-efficient: Only stores unique content
@ -206,7 +206,7 @@ export class VersionManager {
throw new Error(`Entity ${entityId} not found`)
}
// v6.3.2 FIX: For VFS file entities, fetch current content from blob storage
// FIX: For VFS file entities, fetch current content from blob storage
// The entity.data field contains stale embedding text, not actual file content
// VFS files store their real content in BlobStorage (content-addressable)
if (this.isVFSFile(entity)) {
@ -416,7 +416,7 @@ export class VersionManager {
)
}
// v6.3.2 FIX: For VFS file entities, write content back to blob storage
// FIX: For VFS file entities, write content back to blob storage
// The versioned data contains the actual file content (not stale embedding text)
// Using vfs.writeFile() ensures proper blob creation and metadata update
if (this.isVFSFile(versionedEntity)) {

18
src/versioning/VersionStorage.ts
View file

@ -1,13 +1,13 @@
/**
* VersionStorage - Hybrid Storage for Entity Versions (v5.3.0, v6.3.0 fix)
* VersionStorage - Hybrid Storage for Entity Versions
*
* Implements content-addressable storage for entity versions:
* - SHA-256 content hashing for deduplication
* - Uses BaseStorage.saveMetadata/getMetadata for storage (v6.3.0)
* - Uses BaseStorage.saveMetadata/getMetadata for storage
* - Integrates with COW commit system
* - Space-efficient: Only stores unique content
*
* Storage structure (v6.3.0):
* Storage structure:
* Version content is stored using system metadata keys:
* __system_version_{entityId}_{contentHash}
*
@ -160,7 +160,7 @@ export class VersionStorage {
* @returns Storage key for version content
*/
private getVersionPath(entityId: string, contentHash: string): string {
// v6.3.0: Use system-prefixed key for BaseStorage.saveMetadata/getMetadata
// Use system-prefixed key for BaseStorage.saveMetadata/getMetadata
// BaseStorage recognizes __system_ prefix and routes to _system/ directory
return `__system_version_${entityId}_${contentHash}`
}
@ -173,7 +173,7 @@ export class VersionStorage {
*/
private async contentExists(key: string): Promise<boolean> {
try {
// v6.3.0: Use getMetadata to check existence
// Use getMetadata to check existence
const adapter = this.brain.storageAdapter
if (!adapter) return false
@ -200,7 +200,7 @@ export class VersionStorage {
throw new Error('Storage adapter not available')
}
// v6.3.0: Use saveMetadata for storing version content
// Use saveMetadata for storing version content
// The key is system-prefixed so it routes to _system/ directory
await adapter.saveMetadata(key, entity)
}
@ -218,7 +218,7 @@ export class VersionStorage {
throw new Error('Storage adapter not available')
}
// v6.3.0: Use getMetadata for reading version content
// Use getMetadata for reading version content
const entity = await adapter.getMetadata(key)
if (!entity) {
throw new Error(`Version data not found: ${key}`)
@ -234,14 +234,14 @@ export class VersionStorage {
* Deleting the version index entry (via VersionIndex.removeVersion) is sufficient.
* The content may be shared with other versions (same contentHash).
*
* v6.3.0: We don't actually delete version content to avoid breaking
* We don't actually delete version content to avoid breaking
* other versions that may reference the same content hash.
* A separate garbage collection process could clean up unreferenced content.
*
* @param key Storage key (unused - kept for API compatibility)
*/
private async deleteVersionData(_key: string): Promise<void> {
// v6.3.0: Version content is content-addressed and may be shared.
// Version content is content-addressed and may be shared.
// We don't delete it here to prevent breaking other versions.
// The version INDEX is deleted via VersionIndex.removeVersion().
// A GC process could clean up unreferenced content in the future.

2
src/versioning/VersioningAPI.ts
View file

@ -1,5 +1,5 @@
/**
* VersioningAPI - Public API for Entity Versioning (v5.3.0)
* VersioningAPI - Public API for Entity Versioning
*
* User-friendly wrapper around VersionManager with:
* - Clean, simple API

2
src/vfs/MimeTypeDetector.ts
View file

@ -1,5 +1,5 @@
/**
* MIME Type Detection Service (v5.2.0)
* MIME Type Detection Service
*
* Provides comprehensive MIME type detection using:
* 1. Industry-standard `mime` library (2000+ IANA types)

26
src/vfs/PathResolver.ts
View file

@ -71,7 +71,7 @@ export class PathResolver {
/**
* Resolve a path to an entity ID
* v6.1.0: Uses 3-tier caching + MetadataIndexManager for optimal performance
* Uses 3-tier caching + MetadataIndexManager for optimal performance
* Works for ALL storage adapters (FileSystem, GCS, S3, Azure, R2, OPFS)
*/
async resolve(path: string, options?: {
@ -255,14 +255,14 @@ export class PathResolver {
// Still need to verify it exists
}
// v4.7.0: Use proper graph traversal to find children
// Use proper graph traversal to find children
// VFS relationships are now part of the knowledge graph
const relations = await this.brain.getRelations({
from: parentId,
type: VerbType.Contains
})
// v6.0.2: PERFORMANCE FIX - Batch fetch all children (eliminates N+1 pattern)
// PERFORMANCE FIX - Batch fetch all children (eliminates N+1 pattern)
// Before: N sequential get() calls (10 children = 10 × 300ms = 3000ms on GCS)
// After: 1 batch call (10 children = 1 × 300ms = 300ms on GCS)
// 10x improvement for cloud storage (GCS, S3, Azure)
@ -292,7 +292,7 @@ export class PathResolver {
* Uses proper graph relationships to traverse the tree
*/
async getChildren(dirId: string): Promise<VFSEntity[]> {
// v4.7.0: Use O(1) graph relationships (VFS creates these in mkdir/writeFile)
// Use O(1) graph relationships (VFS creates these in mkdir/writeFile)
// VFS relationships are now part of the knowledge graph (no special filtering needed)
const relations = await this.brain.getRelations({
from: dirId,
@ -302,12 +302,12 @@ export class PathResolver {
const validChildren: VFSEntity[] = []
const childNames = new Set<string>()
// v5.12.0: Batch fetch all child entities (eliminates N+1 query pattern)
// Batch fetch all child entities (eliminates N+1 query pattern)
// This is WIRED UP AND USED - no longer a stub!
const childIds = relations.map(r => r.to)
const childrenMap = await this.brain.batchGet(childIds)
// v7.4.1: Deduplicate by entity ID to handle duplicate relationship records
// Deduplicate by entity ID to handle duplicate relationship records
// This can occur when multiple Brainy instances create relationships concurrently
// for the same storage path (each instance has its own in-memory GraphAdjacencyIndex).
// The Set lookup is O(1), adding negligible overhead.
@ -356,7 +356,7 @@ export class PathResolver {
}
/**
* Invalidate ALL caches (v6.3.0)
* Invalidate ALL caches
* Call this when switching branches (checkout), clearing data (clear), or forking
* This ensures no stale data from previous branch/state remains in cache
*/
@ -381,13 +381,13 @@ export class PathResolver {
/**
* Invalidate cache entries for a path and its children
* v6.2.9 FIX: Also invalidates UnifiedCache to prevent stale entity IDs
* FIX: Also invalidates UnifiedCache to prevent stale entity IDs
* This fixes the "Source entity not found" bug after delete+recreate operations
*/
invalidatePath(path: string, recursive = false): void {
const normalizedPath = this.normalizePath(path)
// v6.2.9 FIX: Clear parent cache BEFORE deleting from pathCache
// FIX: Clear parent cache BEFORE deleting from pathCache
// (we need the entityId from the cache entry)
const cached = this.pathCache.get(normalizedPath)
if (cached) {
@ -398,7 +398,7 @@ export class PathResolver {
this.pathCache.delete(normalizedPath)
this.hotPaths.delete(normalizedPath)
// v6.2.9 CRITICAL FIX: Also invalidate UnifiedCache (global LRU cache)
// CRITICAL FIX: Also invalidate UnifiedCache (global LRU cache)
// This was missing before, causing stale entity IDs to be returned after delete
const cacheKey = `vfs:path:${normalizedPath}`
getGlobalCache().delete(cacheKey)
@ -411,12 +411,12 @@ export class PathResolver {
if (cachedPath.startsWith(prefix)) {
this.pathCache.delete(cachedPath)
this.hotPaths.delete(cachedPath)
// v6.2.9: Also clear parent cache for this entry
// Also clear parent cache for this entry
this.parentCache.delete(entry.entityId)
}
}
// v6.2.9 CRITICAL FIX: Also invalidate UnifiedCache entries with this prefix
// CRITICAL FIX: Also invalidate UnifiedCache entries with this prefix
const globalCachePrefix = `vfs:path:${prefix}`
getGlobalCache().deleteByPrefix(globalCachePrefix)
}
@ -561,7 +561,7 @@ export class PathResolver {
/**
* Get cache statistics
* v6.1.0: Added MetadataIndexManager metrics
* Added MetadataIndexManager metrics
*/
getStats(): {
cacheSize: number

88
src/vfs/VirtualFileSystem.ts
View file

@ -81,10 +81,10 @@ export class VirtualFileSystem implements IVirtualFileSystem {
// Mutex for preventing race conditions in directory creation
private mkdirLocks: Map<string, Promise<void>> = new Map()
// v5.8.0: Singleton promise for root initialization (prevents duplicate roots)
// Singleton promise for root initialization (prevents duplicate roots)
private rootInitPromise: Promise<string> | null = null
// v5.10.0: Fixed VFS root ID (prevents duplicates across instances)
// Fixed VFS root ID (prevents duplicates across instances)
// Uses deterministic UUID format for storage compatibility
private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
@ -99,7 +99,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}
/**
* v5.2.0: Access to BlobStorage for unified file storage
* Access to BlobStorage for unified file storage
*/
private get blobStorage() {
// TypeScript doesn't know about blobStorage on storage, use type assertion
@ -119,14 +119,14 @@ export class VirtualFileSystem implements IVirtualFileSystem {
// Merge config with defaults
this.config = { ...this.getDefaultConfig(), ...config }
// v5.0.1: VFS is now auto-initialized during brain.init()
// VFS is now auto-initialized during brain.init()
// Brain is guaranteed to be initialized when this is called
// Removed brain.init() check to prevent infinite recursion
// Create or find root entity
this.rootEntityId = await this.initializeRoot()
// v5.10.0: Clean up old UUID-based roots from v5.9.0 (one-time migration)
// Clean up old UUID-based roots (one-time migration)
await this.cleanupOldRoots()
// Initialize projection registry with auto-discovery of built-in projections
@ -180,7 +180,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}
/**
* v5.8.0: CRITICAL FIX - Prevent duplicate root creation
* CRITICAL FIX - Prevent duplicate root creation
* Uses singleton promise pattern to ensure only ONE root initialization
* happens even with concurrent init() calls
*/
@ -206,7 +206,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}
/**
* v5.10.0: Atomic root initialization with fixed ID
* Atomic root initialization with fixed ID
* Uses deterministic ID to prevent duplicates across all VFS instances
*
* ARCHITECTURAL FIX: Instead of query-then-create (race condition),
@ -266,7 +266,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}
/**
* v5.10.0: Get standard root metadata
* Get standard root metadata
* Centralized to ensure consistency
*/
private getRootMetadata(): VFSMetadata {
@ -286,7 +286,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}
/**
* v5.10.0: Cleanup old UUID-based VFS roots (migration from v5.9.0)
* Cleanup old UUID-based VFS roots
* Called during init to remove duplicate roots created before fixed-ID fix
*
* This is a one-time migration helper that can be removed in future versions.
@ -308,7 +308,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
const duplicates = oldRoots.filter(r => r.id !== VirtualFileSystem.VFS_ROOT_ID)
if (duplicates.length > 0) {
console.log(`VFS: Found ${duplicates.length} old UUID-based root(s) from v5.9.0, cleaning up...`)
console.log(`VFS: Found ${duplicates.length} old UUID-based root(s), cleaning up...`)
for (const duplicate of duplicates) {
try {
@ -352,23 +352,23 @@ export class VirtualFileSystem implements IVirtualFileSystem {
throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'readFile')
}
// v5.2.0: Unified blob storage - ONE path only
// Unified blob storage - ONE path only
if (!entity.metadata.storage?.type || entity.metadata.storage.type !== 'blob') {
throw new VFSError(
VFSErrorCode.EIO,
`File has no blob storage: ${path}. Requires v5.2.0+ storage format.`,
`File has no blob storage: ${path}. Requires blob storage format.`,
path,
'readFile'
)
}
// v5.8.0: CRITICAL FIX - Isolate blob errors from VFS tree corruption
// CRITICAL FIX - Isolate blob errors from VFS tree corruption
// Blob read errors MUST NOT cascade to VFS tree structure
try {
// Read from BlobStorage (handles decompression automatically)
const content = await this.blobStorage.read(entity.metadata.storage.hash)
// v6.2.0: REMOVED updateAccessTime() for performance
// REMOVED updateAccessTime() for performance
// Access time updates caused 50-100ms GCS write on EVERY file read
// Modern file systems use 'noatime' for same reason (performance)
// Field 'accessed' still exists in metadata for backward compat but won't update
@ -436,7 +436,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
existingId = null
}
// v5.2.0: Unified blob storage for ALL files (no size-based branching)
// Unified blob storage for ALL files (no size-based branching)
// Store in BlobStorage (content-addressable, auto-deduplication, streaming)
const blobHash = await this.blobStorage.write(buffer)
@ -450,7 +450,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
compressed: blobMetadata?.compressed
}
// Detect MIME type (v5.2.0: using comprehensive MimeTypeDetector)
// Detect MIME type (using comprehensive MimeTypeDetector)
const mimeType = mimeDetector.detectMimeType(name, buffer)
// Create metadata
@ -459,8 +459,8 @@ export class VirtualFileSystem implements IVirtualFileSystem {
name,
parent: parentId,
vfsType: 'file',
isVFS: true, // v4.3.3: Mark as VFS entity (internal)
isVFSEntity: true, // v5.3.0: Explicit flag for developer filtering
isVFS: true, // Mark as VFS entity (internal)
isVFSEntity: true, // Explicit flag for developer filtering
size: buffer.length,
mimeType,
extension: this.getExtension(name),
@ -470,7 +470,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
accessed: Date.now(),
modified: Date.now(),
storage: storageStrategy
// v5.2.0: No rawData - content is in BlobStorage
// No rawData - content is in BlobStorage
// Backward compatibility: readFile() checks for rawData for legacy files
}
@ -481,7 +481,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
if (existingId) {
// Update existing file
// v5.2.0: No entity.data - content is in BlobStorage
// No entity.data - content is in BlobStorage
await this.brain.update({
id: existingId,
metadata
@ -500,7 +500,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
from: parentId,
to: existingId,
type: VerbType.Contains,
metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
metadata: { isVFS: true } // Mark as VFS relationship
})
}
} else {
@ -519,7 +519,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
from: parentId,
to: entity,
type: VerbType.Contains,
metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
metadata: { isVFS: true } // Mark as VFS relationship
})
// Update path resolver cache
@ -574,7 +574,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'unlink')
}
// v5.2.0: Delete blob from BlobStorage (decrements ref count)
// Delete blob from BlobStorage (decrements ref count)
if (entity.metadata.storage?.type === 'blob') {
await this.blobStorage.delete(entity.metadata.storage.hash)
}
@ -617,7 +617,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}
/**
* v6.2.0: Gather descendants using graph traversal + bulk fetch
* Gather descendants using graph traversal + bulk fetch
*
* ARCHITECTURE:
* 1. Traverse graph to collect entity IDs (in-memory, fast)
@ -694,7 +694,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
/**
* Get a properly structured tree for the given path
*
* v6.2.0: Graph traversal + ONE batch fetch (55x faster on cloud storage)
* Graph traversal + ONE batch fetch (55x faster on cloud storage)
*
* Architecture:
* 1. Resolve path to entity ID
@ -733,7 +733,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
/**
* Get all descendants of a directory (flat list)
*
* v6.2.0: Same optimization as getTreeStructure
* Same optimization as getTreeStructure
*/
async getDescendants(path: string, options?: {
includeAncestor?: boolean
@ -876,8 +876,8 @@ export class VirtualFileSystem implements IVirtualFileSystem {
name,
parent: parentId,
vfsType: 'directory',
isVFS: true, // v4.3.3: Mark as VFS entity (internal)
isVFSEntity: true, // v5.3.0: Explicit flag for developer filtering
isVFS: true, // Mark as VFS entity (internal)
isVFSEntity: true, // Explicit flag for developer filtering
size: 0,
permissions: options?.mode || this.config.permissions?.defaultDirectory || 0o755,
owner: 'user',
@ -900,8 +900,8 @@ export class VirtualFileSystem implements IVirtualFileSystem {
to: entity,
type: VerbType.Contains,
metadata: {
isVFS: true, // v4.5.1: Mark as VFS relationship
relationshipType: 'vfs' // v4.9.0: Standardized relationship type metadata
isVFS: true, // Mark as VFS relationship
relationshipType: 'vfs' // Standardized relationship type metadata
}
})
}
@ -921,7 +921,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
/**
* Remove a directory
*
* v6.4.0: Optimized for cloud storage using batch operations
* Optimized for cloud storage using batch operations
* - Uses gatherDescendants() for efficient graph traversal + batch fetch
* - Uses deleteMany() for chunked transactional deletion
* - Parallel blob cleanup with chunking
@ -950,7 +950,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
throw new VFSError(VFSErrorCode.ENOTEMPTY, `Directory not empty: ${path}`, path, 'rmdir')
}
// v6.4.0: OPTIMIZED batch deletion for recursive case
// OPTIMIZED batch deletion for recursive case
if (options?.recursive && children.length > 0) {
// Phase 1: Gather all descendants in ONE batch fetch
const descendants = await this.gatherDescendants(entityId, Infinity)
@ -1020,7 +1020,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
children = children.slice(0, options.limit)
}
// v6.2.0: REMOVED updateAccessTime() for performance
// REMOVED updateAccessTime() for performance
// Directory access time updates caused 50-100ms GCS write on EVERY readdir
// await this.updateAccessTime(entityId) // ← REMOVED
@ -1117,7 +1117,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
offset: options?.offset,
explain: options?.explain,
where: {
vfsType: 'file' // v4.7.0: Search VFS files
vfsType: 'file' // Search VFS files
}
}
@ -1167,7 +1167,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
threshold: options?.threshold || 0.7,
type: [NounType.File, NounType.Document, NounType.Media],
where: {
vfsType: 'file' // v4.7.0: Find similar VFS files
vfsType: 'file' // Find similar VFS files
}
})
@ -1294,7 +1294,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
return filename.substring(lastDot + 1).toLowerCase()
}
// v5.2.0: MIME detection moved to MimeTypeDetector service
// MIME detection moved to MimeTypeDetector service
// Removed detectMimeType() and isTextFile() - now using mimeDetector singleton
private getFileNounType(mimeType: string): NounType {
@ -1307,7 +1307,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
return NounType.File
}
// v5.2.0: Removed compression methods (shouldCompress, compress, decompress)
// Removed compression methods (shouldCompress, compress, decompress)
// BlobStorage handles all compression automatically with zstd
private async generateEmbedding(buffer: Buffer, mimeType: string): Promise<number[] | undefined> {
@ -1371,7 +1371,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
return metadata
}
// v6.2.0: REMOVED updateAccessTime() method entirely
// REMOVED updateAccessTime() method entirely
// Access time updates caused 50-100ms GCS write on EVERY file/dir read
// Modern file systems use 'noatime' for same reason
// Field 'accessed' still exists in metadata for backward compat but won't update
@ -1674,7 +1674,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
from: newParentId,
to: entityId,
type: VerbType.Contains,
metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
metadata: { isVFS: true } // Mark as VFS relationship
})
}
}
@ -1752,7 +1752,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
from: parentId,
to: newEntity,
type: VerbType.Contains,
metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
metadata: { isVFS: true } // Mark as VFS relationship
})
}
@ -1774,7 +1774,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
/**
* Copy a directory recursively
*
* v6.4.0: Optimized for cloud storage using batch operations
* Optimized for cloud storage using batch operations
* - Uses gatherDescendants() for efficient graph traversal + batch fetch
* - Uses addMany() for batch entity creation
* - Uses relateMany() for batch relationship creation
@ -1941,7 +1941,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
from: parentId,
to: entity,
type: VerbType.Contains,
metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
metadata: { isVFS: true } // Mark as VFS relationship
})
// Update path resolver cache
@ -2151,7 +2151,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
from: fromEntityId,
to: toEntityId,
type: type as any, // Convert string to VerbType
metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship
metadata: { isVFS: true } // Mark as VFS relationship
})
// Invalidate caches for both paths
@ -2378,7 +2378,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
/**
* Bulk write operations for performance
*
* v6.5.0: Prevents race condition by processing mkdir operations
* Prevents race condition by processing mkdir operations
* sequentially before parallel batch processing of other operations.
*/
async bulkWrite(operations: Array<{

10
src/vfs/importers/DirectoryImporter.ts
View file

@ -27,7 +27,7 @@ export interface ImportOptions {
showProgress?: boolean // Log progress (default: false)
filter?: (path: string) => boolean // Custom filter function
// v4.10.0: Import tracking
// Import tracking
importId?: string // Unique import identifier (auto-generated if not provided)
projectId?: string // Project identifier grouping related imports
customMetadata?: Record<string, any> // Custom metadata to attach
@ -66,7 +66,7 @@ export class DirectoryImporter {
async import(sourcePath: string, options: ImportOptions = {}): Promise<ImportResult> {
const startTime = Date.now()
// v4.10.0: Generate tracking metadata
// Generate tracking metadata
const importId = options.importId || uuidv4()
const projectId = options.projectId || this.deriveProjectId(options.targetPath || '/')
const trackingMetadata = {
@ -225,7 +225,7 @@ export class DirectoryImporter {
try {
await this.vfs.mkdir(dirPath, {
recursive: true,
metadata: trackingMetadata // v4.10.0: Add tracking metadata
metadata: trackingMetadata // Add tracking metadata
})
result.directoriesCreated++
} catch (error: any) {
@ -332,7 +332,7 @@ export class DirectoryImporter {
originalPath: filePath,
originalSize: stats.size,
originalModified: stats.mtime.getTime(),
...trackingMetadata // v4.10.0: Add tracking metadata
...trackingMetadata // Add tracking metadata
}
})
@ -382,6 +382,6 @@ export class DirectoryImporter {
return false
}
// v5.2.0: MIME detection moved to MimeTypeDetector service
// MIME detection moved to MimeTypeDetector service
// Removed detectMimeType() - now using mimeDetector singleton
}

2
src/vfs/index.ts
View file

@ -10,7 +10,7 @@ export { VirtualFileSystem } from './VirtualFileSystem.js'
export { PathResolver } from './PathResolver.js'
export * from './types.js'
// MIME Type Detection (v5.2.0)
// MIME Type Detection
export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js'
// fs compatibility layer

2
src/vfs/semantic/SemanticPathResolver.ts
View file

@ -317,7 +317,7 @@ export class SemanticPathResolver {
}
/**
* Invalidate ALL caches (v6.3.0)
* Invalidate ALL caches
* Clears both traditional path cache AND semantic cache
* Call this when switching branches, clearing data, or forking
*/

2
src/vfs/semantic/projections/AuthorProjection.ts
View file

@ -52,7 +52,7 @@ export class AuthorProjection extends BaseProjectionStrategy {
* Resolve author to entity IDs using REAL Brainy.find()
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, authorName: string): Promise<string[]> {
// v4.7.0: VFS entities are part of the knowledge graph
// VFS entities are part of the knowledge graph
const results = await brain.find({
where: {
vfsType: 'file',

2
src/vfs/semantic/projections/TagProjection.ts
View file

@ -52,7 +52,7 @@ export class TagProjection extends BaseProjectionStrategy {
* Resolve tag to entity IDs using REAL Brainy.find()
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, tagName: string): Promise<string[]> {
// v4.7.0: VFS entities are part of the knowledge graph
// VFS entities are part of the knowledge graph
const results = await brain.find({
where: {
vfsType: 'file',

2
src/vfs/semantic/projections/TemporalProjection.ts
View file

@ -69,7 +69,7 @@ export class TemporalProjection extends BaseProjectionStrategy {
const endOfDay = new Date(date)
endOfDay.setHours(23, 59, 59, 999)
// v4.7.0: VFS entities are part of the knowledge graph
// VFS entities are part of the knowledge graph
const results = await brain.find({
where: {
vfsType: 'file',

6
src/vfs/types.ts
View file

@ -33,8 +33,8 @@ export interface VFSMetadata {
name: string // Filename or directory name
parent?: string // Parent directory entity ID
vfsType: 'file' | 'directory' | 'symlink'
isVFS?: boolean // v4.3.3: Mark as VFS entity (internal, separates from knowledge graph)
isVFSEntity?: boolean // v5.3.0: Explicit developer-facing flag for filtering VFS entities
isVFS?: boolean // Mark as VFS entity (internal, separates from knowledge graph)
isVFSEntity?: boolean // Explicit developer-facing flag for filtering VFS entities
// File attributes
size: number // Size in bytes (0 for directories)
@ -50,7 +50,7 @@ export interface VFSMetadata {
accessed: number // Last access timestamp (ms)
modified: number // Last modification timestamp (ms)
// Content storage strategy (v5.2.0: unified blob storage)
// Content storage strategy (unified blob storage)
storage?: {
type: 'blob' // All files now use BlobStorage (was 'inline'|'reference'|'chunked')
hash: string // SHA-256 content hash from BlobStorage