fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -90,8 +90,8 @@ Adds a new entity to the brain.
|
||||||
- `id` - Custom ID (auto-generated if not provided)
|
- `id` - Custom ID (auto-generated if not provided)
|
||||||
- `vector` - Pre-computed embedding vector
|
- `vector` - Pre-computed embedding vector
|
||||||
- `service` - Service name for multi-tenancy
|
- `service` - Service name for multi-tenancy
|
||||||
- `confidence` - Type classification confidence (0-1) ✨ *New in v4.3.0*
|
- `confidence` - Type classification confidence (0-1) ✨
|
||||||
- `weight` - Entity importance/salience (0-1) ✨ *New in v4.3.0*
|
- `weight` - Entity importance/salience (0-1) ✨
|
||||||
- `writeOnly` - Skip validation for high-speed ingestion
|
- `writeOnly` - Skip validation for high-speed ingestion
|
||||||
|
|
||||||
**Returns:** Entity ID
|
**Returns:** Entity ID
|
||||||
|
|
@ -116,7 +116,7 @@ const id = await brain.add({
|
||||||
### `async get(id: string, options?: GetOptions): Promise<Entity | null>`
|
### `async get(id: string, options?: GetOptions): Promise<Entity | null>`
|
||||||
Retrieves an entity by ID.
|
Retrieves an entity by ID.
|
||||||
|
|
||||||
✨ **v5.11.1 Performance Optimization**:
|
✨ **Performance Optimization**:
|
||||||
- **Default (metadata-only)**: 76-81% faster, 95% less bandwidth - perfect for VFS, existence checks, metadata access
|
- **Default (metadata-only)**: 76-81% faster, 95% less bandwidth - perfect for VFS, existence checks, metadata access
|
||||||
- **Opt-in vectors**: Use `{ includeVectors: true }` when computing similarity
|
- **Opt-in vectors**: Use `{ includeVectors: true }` when computing similarity
|
||||||
|
|
||||||
|
|
@ -127,7 +127,7 @@ Retrieves an entity by ID.
|
||||||
|
|
||||||
**Returns:** Entity object or null if not found
|
**Returns:** Entity object or null if not found
|
||||||
|
|
||||||
**Entity Properties:** ✨ *Updated in v4.3.0, v5.11.1*
|
**Entity Properties:**
|
||||||
- `id` - Unique identifier
|
- `id` - Unique identifier
|
||||||
- `type` - NounType classification
|
- `type` - NounType classification
|
||||||
- `data` - Original content
|
- `data` - Original content
|
||||||
|
|
@ -172,8 +172,8 @@ Updates an existing entity.
|
||||||
- `metadata` - New or partial metadata
|
- `metadata` - New or partial metadata
|
||||||
- `merge` - Merge metadata (true) or replace (false), default: true
|
- `merge` - Merge metadata (true) or replace (false), default: true
|
||||||
- `vector` - New embedding vector
|
- `vector` - New embedding vector
|
||||||
- `confidence` - Update type classification confidence ✨ *New in v4.3.0*
|
- `confidence` - Update type classification confidence ✨
|
||||||
- `weight` - Update entity importance/salience ✨ *New in v4.3.0*
|
- `weight` - Update entity importance/salience ✨
|
||||||
|
|
||||||
**Example:**
|
**Example:**
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -406,7 +406,7 @@ Universal search with Triple Intelligence fusion.
|
||||||
|
|
||||||
**Returns:** Array of Result objects with scores
|
**Returns:** Array of Result objects with scores
|
||||||
|
|
||||||
**Result Properties:** ✨ *Enhanced in v4.3.0*
|
**Result Properties:** ✨
|
||||||
- `id` - Entity ID
|
- `id` - Entity ID
|
||||||
- `score` - Relevance score (0-1)
|
- `score` - Relevance score (0-1)
|
||||||
- `type` - Entity type (flattened for convenience) *Enhanced*
|
- `type` - Entity type (flattened for convenience) *Enhanced*
|
||||||
|
|
@ -422,7 +422,7 @@ Universal search with Triple Intelligence fusion.
|
||||||
// Natural language search
|
// Natural language search
|
||||||
const results = await brain.find('recent product launches')
|
const results = await brain.find('recent product launches')
|
||||||
|
|
||||||
// NEW in v4.3.0: Direct access to flattened fields
|
// Direct access to flattened fields
|
||||||
console.log(results[0].metadata) // Direct access (convenient!)
|
console.log(results[0].metadata) // Direct access (convenient!)
|
||||||
console.log(results[0].confidence) // Type confidence
|
console.log(results[0].confidence) // Type confidence
|
||||||
console.log(results[0].weight) // Entity importance
|
console.log(results[0].weight) // Entity importance
|
||||||
|
|
@ -469,7 +469,7 @@ Finds similar entities using vector similarity.
|
||||||
- `type` - Filter by type(s)
|
- `type` - Filter by type(s)
|
||||||
- `where` - Metadata filters
|
- `where` - Metadata filters
|
||||||
|
|
||||||
**Returns:** Array of Result objects (same structure as `find()`) ✨ *Enhanced in v4.3.0*
|
**Returns:** Array of Result objects (same structure as `find()`) ✨
|
||||||
|
|
||||||
**Example:**
|
**Example:**
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -480,7 +480,7 @@ const similar = await brain.similar({
|
||||||
type: NounType.Document
|
type: NounType.Document
|
||||||
})
|
})
|
||||||
|
|
||||||
// NEW in v4.3.0: Access flattened fields directly
|
// Access flattened fields directly
|
||||||
for (const result of similar) {
|
for (const result of similar) {
|
||||||
console.log(`Similarity: ${result.score}`)
|
console.log(`Similarity: ${result.score}`)
|
||||||
console.log(`Type: ${result.type}`) // Flattened
|
console.log(`Type: ${result.type}`) // Flattened
|
||||||
|
|
@ -491,7 +491,7 @@ for (const result of similar) {
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async embed(data: any): Promise<Vector>` ✨ *v7.1.0*
|
### `async embed(data: any): Promise<Vector>`
|
||||||
Generates an embedding vector from data.
|
Generates an embedding vector from data.
|
||||||
|
|
||||||
**Parameters:**
|
**Parameters:**
|
||||||
|
|
@ -507,7 +507,7 @@ console.log(vector.length) // 384
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async embedBatch(texts: string[]): Promise<number[][]>` ✨ *v7.1.0, Optimized v7.9.0*
|
### `async embedBatch(texts: string[]): Promise<number[][]>`
|
||||||
Batch embed multiple texts using native WASM batch API (single forward pass).
|
Batch embed multiple texts using native WASM batch API (single forward pass).
|
||||||
|
|
||||||
**Parameters:**
|
**Parameters:**
|
||||||
|
|
@ -515,7 +515,7 @@ Batch embed multiple texts using native WASM batch API (single forward pass).
|
||||||
|
|
||||||
**Returns:** Array of 384-dimensional vectors
|
**Returns:** Array of 384-dimensional vectors
|
||||||
|
|
||||||
> **v7.9.0**: Uses the engine's native `embed_batch()` for a single model forward pass instead of N individual `embed()` calls.
|
> Uses the engine's native `embed_batch()` for a single model forward pass instead of N individual `embed()` calls.
|
||||||
|
|
||||||
**Example:**
|
**Example:**
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -530,7 +530,7 @@ console.log(embeddings[0].length) // 384
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async similarity(textA: string, textB: string): Promise<number>` ✨ *v7.1.0*
|
### `async similarity(textA: string, textB: string): Promise<number>`
|
||||||
Calculate semantic similarity between two texts.
|
Calculate semantic similarity between two texts.
|
||||||
|
|
||||||
**Parameters:**
|
**Parameters:**
|
||||||
|
|
@ -550,7 +550,7 @@ console.log(score) // ~0.85 (high semantic similarity)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async neighbors(entityId: string, options?): Promise<string[]>` ✨ *v7.1.0*
|
### `async neighbors(entityId: string, options?): Promise<string[]>`
|
||||||
Get graph neighbors of an entity.
|
Get graph neighbors of an entity.
|
||||||
|
|
||||||
**Parameters:**
|
**Parameters:**
|
||||||
|
|
@ -582,7 +582,7 @@ const extended = await brain.neighbors(entityId, {
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async findDuplicates(options?): Promise<DuplicateResult[]>` ✨ *v7.1.0*
|
### `async findDuplicates(options?): Promise<DuplicateResult[]>`
|
||||||
Find semantic duplicates in the database.
|
Find semantic duplicates in the database.
|
||||||
|
|
||||||
**Parameters:**
|
**Parameters:**
|
||||||
|
|
@ -614,7 +614,7 @@ const personDupes = await brain.findDuplicates({
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async indexStats(): Promise<IndexStats>` ✨ *v7.1.0*
|
### `async indexStats(): Promise<IndexStats>`
|
||||||
Get comprehensive index statistics.
|
Get comprehensive index statistics.
|
||||||
|
|
||||||
**Returns:**
|
**Returns:**
|
||||||
|
|
@ -639,7 +639,7 @@ console.log(`Fields: ${stats.metadataFields.join(', ')}`)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `async cluster(options?): Promise<ClusterResult[]>` ✨ *v7.1.0*
|
### `async cluster(options?): Promise<ClusterResult[]>`
|
||||||
Cluster entities by semantic similarity.
|
Cluster entities by semantic similarity.
|
||||||
|
|
||||||
Groups entities into clusters based on their embedding similarity using
|
Groups entities into clusters based on their embedding similarity using
|
||||||
|
|
@ -1665,7 +1665,7 @@ Executes the pipeline.
|
||||||
|
|
||||||
### Core Interfaces
|
### Core Interfaces
|
||||||
|
|
||||||
✨ *Updated in v4.3.0 - Added confidence/weight to Entity, flattened Result fields*
|
✨ *Updated in Added confidence/weight to Entity, flattened Result fields*
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
interface Entity<T = any> {
|
interface Entity<T = any> {
|
||||||
|
|
@ -1714,7 +1714,7 @@ interface Result<T = any> {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Key Changes in v4.3.0:**
|
**Key Changes:**
|
||||||
- ✅ `Entity` now exposes `confidence` and `weight`
|
- ✅ `Entity` now exposes `confidence` and `weight`
|
||||||
- ✅ `Result` flattens commonly-used entity fields to top level
|
- ✅ `Result` flattens commonly-used entity fields to top level
|
||||||
- ✅ Direct access: `result.metadata` instead of `result.entity.metadata`
|
- ✅ Direct access: `result.metadata` instead of `result.entity.metadata`
|
||||||
|
|
|
||||||
|
|
@ -1,23 +1,22 @@
|
||||||
# Batch Operations API v5.12.0
|
# Batch Operations API
|
||||||
|
|
||||||
> **Enterprise Production-Ready** | Zero N+1 Query Patterns | 90%+ Performance Improvement
|
> **Enterprise Production-Ready** | Zero N+1 Query Patterns | 90%+ Performance Improvement
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Brainy v5.12.0 introduces comprehensive batch operations at the storage layer, eliminating N+1 query patterns and dramatically improving performance for VFS operations, relationship queries, and entity retrieval on cloud storage.
|
Brainy introduces comprehensive batch operations at the storage layer, eliminating N+1 query patterns and dramatically improving performance for VFS operations, relationship queries, and entity retrieval on cloud storage.
|
||||||
|
|
||||||
### Problem Solved
|
### Problem Solved
|
||||||
|
|
||||||
**Before v5.12.0:**
|
**Before optimization:**
|
||||||
- VFS `getTreeStructure()` on cloud storage: **12.7 seconds** for directory with 12 files
|
- VFS `getTreeStructure()` on cloud storage: **12.7 seconds** for directory with 12 files
|
||||||
- N+1 query pattern: 1 directory query + N individual file queries (22 sequential calls × 580ms latency)
|
- N+1 query pattern: 1 directory query + N individual file queries (22 sequential calls × 580ms latency)
|
||||||
|
|
||||||
**After v5.12.0:**
|
**After optimization:**
|
||||||
- VFS `getTreeStructure()`: **<1 second** for 12 files (90%+ improvement)
|
- VFS `getTreeStructure()`: **<1 second** for 12 files (90%+ improvement)
|
||||||
- 2-3 batched calls instead of 22 sequential calls
|
- 2-3 batched calls instead of 22 sequential calls
|
||||||
- Native cloud storage batch APIs for maximum throughput
|
- Native cloud storage batch APIs for maximum throughput
|
||||||
|
|
||||||
**IMPORTANT:** The v5.12.0 batch optimizations apply **ONLY to `getTreeStructure()`**, not to `readFile()` or individual `get()` operations. See v6.0.0 changes for comprehensive storage path optimizations.
|
**IMPORTANT:** The batch optimizations apply **ONLY to `getTreeStructure()`**, not to `readFile()` or individual `get()` operations. See changes for comprehensive storage path optimizations.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -56,7 +55,7 @@ results.size // → 3 (number of found entities)
|
||||||
|
|
||||||
### 2. `storage.getNounMetadataBatch(ids)`
|
### 2. `storage.getNounMetadataBatch(ids)`
|
||||||
|
|
||||||
Batch metadata retrieval with direct O(1) path construction (v6.0.0+).
|
Batch metadata retrieval with direct O(1) path construction.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const storage = brain.storage as BaseStorage
|
const storage = brain.storage as BaseStorage
|
||||||
|
|
@ -254,7 +253,7 @@ VFS operations automatically use batch APIs for maximum performance.
|
||||||
// OLD: Sequential N+1 pattern (12.7 seconds for 12 files)
|
// OLD: Sequential N+1 pattern (12.7 seconds for 12 files)
|
||||||
const tree = await brain.vfs.getTreeStructure('/my-dir')
|
const tree = await brain.vfs.getTreeStructure('/my-dir')
|
||||||
|
|
||||||
// NEW v5.12.0: Parallel breadth-first with batching (<1 second)
|
// NEW Parallel breadth-first with batching (<1 second)
|
||||||
// ✅ PathResolver.getChildren() uses brain.batchGet() internally
|
// ✅ PathResolver.getChildren() uses brain.batchGet() internally
|
||||||
// ✅ Parallel traversal of directories at same tree level
|
// ✅ Parallel traversal of directories at same tree level
|
||||||
// ✅ 2-3 batched calls instead of 22 sequential calls
|
// ✅ 2-3 batched calls instead of 22 sequential calls
|
||||||
|
|
@ -285,11 +284,11 @@ VFS.getTreeStructure()
|
||||||
|
|
||||||
## Advanced Features Compatibility
|
## Advanced Features Compatibility
|
||||||
|
|
||||||
### ✅ ID-First Storage Architecture (v6.0.0+)
|
### ✅ ID-First Storage Architecture
|
||||||
|
|
||||||
All batch operations use direct ID-first paths - no type lookup needed!
|
All batch operations use direct ID-first paths - no type lookup needed!
|
||||||
|
|
||||||
**NEW v6.0.0 Path Structure:**
|
**ID-First Path Structure:**
|
||||||
```
|
```
|
||||||
entities/nouns/{SHARD}/{ID}/metadata.json
|
entities/nouns/{SHARD}/{ID}/metadata.json
|
||||||
entities/verbs/{SHARD}/{ID}/metadata.json
|
entities/verbs/{SHARD}/{ID}/metadata.json
|
||||||
|
|
@ -391,7 +390,7 @@ Historical queries use `HistoricalStorageAdapter` which wraps batch operations t
|
||||||
|
|
||||||
### VFS Operations (12 Files)
|
### VFS Operations (12 Files)
|
||||||
|
|
||||||
| Storage | Before v5.12.0 | After v5.12.0 | Improvement |
|
| Storage | Before optimization | After optimization | Improvement |
|
||||||
|---------|---------------|---------------|-------------|
|
|---------|---------------|---------------|-------------|
|
||||||
| **GCS** | 12.7s | <1s | **92% faster** |
|
| **GCS** | 12.7s | <1s | **92% faster** |
|
||||||
| **S3** | 13.2s | <1s | **92% faster** |
|
| **S3** | 13.2s | <1s | **92% faster** |
|
||||||
|
|
@ -658,7 +657,7 @@ if (typeof selfWithBatch.readBatch === 'function') {
|
||||||
- Zero N+1 query patterns
|
- Zero N+1 query patterns
|
||||||
|
|
||||||
**Compatibility:**
|
**Compatibility:**
|
||||||
- ✅ ID-first storage (v6.0.0+)
|
- ✅ ID-first storage
|
||||||
- ✅ Sharding (256 shards)
|
- ✅ Sharding (256 shards)
|
||||||
- ✅ COW (branch isolation, inheritance)
|
- ✅ COW (branch isolation, inheritance)
|
||||||
- ✅ fork() and checkout()
|
- ✅ fork() and checkout()
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Creating Augmentations for Brainy
|
# Creating Augmentations for Brainy
|
||||||
|
|
||||||
> **Updated for v4.0.0** - Includes metadata structure changes and type system improvements
|
> **Updated** - Includes metadata structure changes and type system improvements
|
||||||
|
|
||||||
## The BrainyAugmentation Interface
|
## The BrainyAugmentation Interface
|
||||||
|
|
||||||
|
|
@ -23,13 +23,13 @@ interface BrainyAugmentation {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## v4.0.0 Breaking Changes for Augmentation Developers
|
## Breaking Changes for Augmentation Developers
|
||||||
|
|
||||||
### 1. Metadata Structure Separation
|
### 1. Metadata Structure Separation
|
||||||
v4.0.0 introduces strict metadata/vector separation for billion-scale performance:
|
Brainy introduces strict metadata/vector separation for billion-scale performance:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// ✅ v4.0.0: Metadata has required type field
|
// ✅ Metadata has required type field
|
||||||
interface NounMetadata {
|
interface NounMetadata {
|
||||||
noun: NounType // Required! Must be a valid noun type
|
noun: NounType // Required! Must be a valid noun type
|
||||||
[key: string]: any // Your custom metadata
|
[key: string]: any // Your custom metadata
|
||||||
|
|
@ -61,7 +61,7 @@ The verb relationship field changed from `type` to `verb`:
|
||||||
// ❌ v3.x
|
// ❌ v3.x
|
||||||
verb.type === 'relatedTo'
|
verb.type === 'relatedTo'
|
||||||
|
|
||||||
// ✅ v4.0.0
|
// ✅ Current
|
||||||
verb.verb === 'relatedTo'
|
verb.verb === 'relatedTo'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -69,7 +69,7 @@ verb.verb === 'relatedTo'
|
||||||
|
|
||||||
Storage augmentations are special - they provide the storage backend for Brainy.
|
Storage augmentations are special - they provide the storage backend for Brainy.
|
||||||
|
|
||||||
### Important: v4.0.0 Storage Requirements
|
### Important: Storage Requirements
|
||||||
|
|
||||||
Your storage adapter MUST:
|
Your storage adapter MUST:
|
||||||
1. **Wrap metadata** with required `noun`/`verb` fields
|
1. **Wrap metadata** with required `noun`/`verb` fields
|
||||||
|
|
@ -96,7 +96,7 @@ export class MyCustomStorage extends BaseStorageAdapter {
|
||||||
const noun = await this._getNoun(id)
|
const noun = await this._getNoun(id)
|
||||||
if (!noun) return null
|
if (!noun) return null
|
||||||
|
|
||||||
// Fetch metadata separately (v4.0.0 pattern)
|
// Fetch metadata separately
|
||||||
const metadata = await this.getNounMetadata(id)
|
const metadata = await this.getNounMetadata(id)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
@ -109,14 +109,14 @@ export class MyCustomStorage extends BaseStorageAdapter {
|
||||||
async saveNoun(noun: HNSWNoun, metadata?: NounMetadata): Promise<void> {
|
async saveNoun(noun: HNSWNoun, metadata?: NounMetadata): Promise<void> {
|
||||||
// Validate metadata has required 'noun' field
|
// Validate metadata has required 'noun' field
|
||||||
if (!metadata?.noun) {
|
if (!metadata?.noun) {
|
||||||
throw new Error('v4.0.0: NounMetadata requires "noun" field')
|
throw new Error('NounMetadata requires "noun" field')
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.database.save({
|
await this.database.save({
|
||||||
id: noun.id,
|
id: noun.id,
|
||||||
vector: noun.vector,
|
vector: noun.vector,
|
||||||
nounType: noun.nounType,
|
nounType: noun.nounType,
|
||||||
metadata: metadata // Stored separately in v4.0.0
|
metadata: metadata // Stored separately
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -356,7 +356,7 @@ Future capability for premium augmentations:
|
||||||
6. **Log appropriately** - Use context.log() for consistent output
|
6. **Log appropriately** - Use context.log() for consistent output
|
||||||
7. **Document your augmentation** - Include examples
|
7. **Document your augmentation** - Include examples
|
||||||
|
|
||||||
### v4.0.0 Specific Best Practices
|
### Specific Best Practices
|
||||||
|
|
||||||
8. **Always include `noun` field** when creating/modifying NounMetadata:
|
8. **Always include `noun` field** when creating/modifying NounMetadata:
|
||||||
```typescript
|
```typescript
|
||||||
|
|
|
||||||
|
|
@ -613,7 +613,7 @@ vfsFiles.forEach(f => {
|
||||||
})
|
})
|
||||||
|
|
||||||
// 3. VFS FILTERING IN KNOWLEDGE QUERIES
|
// 3. VFS FILTERING IN KNOWLEDGE QUERIES
|
||||||
console.log('\n🔍 Understanding VFS filtering (v4.4.0)...\n')
|
console.log('\n🔍 Understanding VFS filtering...\n')
|
||||||
|
|
||||||
// Create some knowledge entities
|
// Create some knowledge entities
|
||||||
const conceptId = await brain.add({
|
const conceptId = await brain.add({
|
||||||
|
|
@ -758,7 +758,7 @@ await brain.close()
|
||||||
|
|
||||||
### Key Concepts
|
### Key Concepts
|
||||||
|
|
||||||
#### 1. **VFS Filtering Architecture (v4.4.0)**
|
#### 1. **VFS Filtering Architecture**
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// 🎯 DEFAULT BEHAVIOR: Clean Separation
|
// 🎯 DEFAULT BEHAVIOR: Clean Separation
|
||||||
|
|
|
||||||
|
|
@ -13,8 +13,7 @@ Brainy's `find()` method is the most advanced query system in any vector databas
|
||||||
- **Data Structure**: Multi-layer graph with 16 connections per node
|
- **Data Structure**: Multi-layer graph with 16 connections per node
|
||||||
- **Use Cases**: "Find similar documents", "Content like this"
|
- **Use Cases**: "Find similar documents", "Content like this"
|
||||||
|
|
||||||
### 2. Text Intelligence (Word Index) - v7.7.0
|
### 2. Text Intelligence (Word Index) - **Purpose**: Keyword/exact text matching
|
||||||
- **Purpose**: Keyword/exact text matching
|
|
||||||
- **Algorithm**: Inverted word index with FNV-1a hashing
|
- **Algorithm**: Inverted word index with FNV-1a hashing
|
||||||
- **Performance**: O(log C) where C = chunks (~50 values each)
|
- **Performance**: O(log C) where C = chunks (~50 values each)
|
||||||
- **Data Structure**: `__words__ → hash → Roaring Bitmap of entity IDs`
|
- **Data Structure**: `__words__ → hash → Roaring Bitmap of entity IDs`
|
||||||
|
|
@ -92,7 +91,7 @@ await brain.find({
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. Hybrid Search (v7.7.0+)
|
### 4. Hybrid Search
|
||||||
```typescript
|
```typescript
|
||||||
// Zero-config hybrid: automatically combines text + semantic search
|
// Zero-config hybrid: automatically combines text + semantic search
|
||||||
await brain.find({ query: "David Smith" })
|
await brain.find({ query: "David Smith" })
|
||||||
|
|
@ -113,7 +112,7 @@ await brain.find({ query: "search term", hybridAlpha: 0.3 })
|
||||||
- Medium queries (3-4 words): alpha = 0.5 (balanced)
|
- Medium queries (3-4 words): alpha = 0.5 (balanced)
|
||||||
- Long queries (5+ words): alpha = 0.7 (favor semantic matching)
|
- Long queries (5+ words): alpha = 0.7 (favor semantic matching)
|
||||||
|
|
||||||
### 5. Match Visibility (v7.8.0)
|
### 5. Match Visibility
|
||||||
|
|
||||||
Search results include match details showing what matched:
|
Search results include match details showing what matched:
|
||||||
|
|
||||||
|
|
@ -132,7 +131,7 @@ Use this for:
|
||||||
- **Explaining** why a result was found (matchSource)
|
- **Explaining** why a result was found (matchSource)
|
||||||
- **Debugging** search behavior (separate scores)
|
- **Debugging** search behavior (separate scores)
|
||||||
|
|
||||||
### 6. Semantic Highlighting (v7.8.0)
|
### 6. Semantic Highlighting
|
||||||
|
|
||||||
Highlight which concepts/words in text matched your query:
|
Highlight which concepts/words in text matched your query:
|
||||||
|
|
||||||
|
|
@ -446,7 +445,7 @@ await brain.find({
|
||||||
// Total performance: ~1.2ms for 100K entities
|
// Total performance: ~1.2ms for 100K entities
|
||||||
```
|
```
|
||||||
|
|
||||||
## Filter Syntax Reference (v5.8.0+)
|
## Filter Syntax Reference
|
||||||
|
|
||||||
### Where Clause: Complete Operator Guide
|
### Where Clause: Complete Operator Guide
|
||||||
|
|
||||||
|
|
@ -771,7 +770,7 @@ await brain.find({
|
||||||
// Returns: Verified people with high reputation who work at Stanford AI Lab
|
// Returns: Verified people with high reputation who work at Stanford AI Lab
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Pagination with Graph Queries (v5.8.0+)
|
#### Pagination with Graph Queries
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Page through high-degree nodes efficiently
|
// Page through high-degree nodes efficiently
|
||||||
|
|
@ -792,7 +791,7 @@ const verbIds = await brain.graphIndex.getVerbIdsBySource('source-id', {
|
||||||
|
|
||||||
**Note**: See `src/graph/graphAdjacencyIndex.ts` for low-level graph operations.
|
**Note**: See `src/graph/graphAdjacencyIndex.ts` for low-level graph operations.
|
||||||
|
|
||||||
### Sorting Results (v4.5.4+)
|
### Sorting Results
|
||||||
|
|
||||||
Sort query results by any field, including timestamps:
|
Sort query results by any field, including timestamps:
|
||||||
|
|
||||||
|
|
@ -895,7 +894,7 @@ const page1 = await getPaginatedResults(0) // First 20
|
||||||
const page2 = await getPaginatedResults(1) // Next 20
|
const page2 = await getPaginatedResults(1) // Next 20
|
||||||
```
|
```
|
||||||
|
|
||||||
**Graph pagination** (v5.8.0+):
|
**Graph pagination**:
|
||||||
```typescript
|
```typescript
|
||||||
// Paginate through high-degree node relationships
|
// Paginate through high-degree node relationships
|
||||||
async function getNeighborPage(entityId: string, page: number, pageSize: number = 50) {
|
async function getNeighborPage(entityId: string, page: number, pageSize: number = 50) {
|
||||||
|
|
@ -1208,7 +1207,7 @@ await brain.find({ type: NounType.Document })
|
||||||
where: { age: { greaterThan: 18 } } // Old API
|
where: { age: { greaterThan: 18 } } // Old API
|
||||||
|
|
||||||
// ✅ Correct: Use canonical operators
|
// ✅ Correct: Use canonical operators
|
||||||
where: { age: { gt: 18 } } // v5.0.0+
|
where: { age: { gt: 18 } }
|
||||||
```
|
```
|
||||||
|
|
||||||
### Graph Traversal Issues
|
### Graph Traversal Issues
|
||||||
|
|
@ -1291,7 +1290,7 @@ console.log(`Results: ${results.length}, Unique: ${uniqueIds.size}`)
|
||||||
// Brainy should never return duplicates - report if found
|
// Brainy should never return duplicates - report if found
|
||||||
```
|
```
|
||||||
|
|
||||||
## VFS (Virtual File System) Visibility (v4.7.0+)
|
## VFS (Virtual File System) Visibility
|
||||||
|
|
||||||
### Default Behavior
|
### Default Behavior
|
||||||
|
|
||||||
|
|
@ -1342,7 +1341,7 @@ await brain.find({
|
||||||
|
|
||||||
### Migration from v4.6.x
|
### Migration from v4.6.x
|
||||||
|
|
||||||
**BREAKING CHANGE** (v4.7.0): The `includeVFS` parameter has been removed:
|
**BREAKING CHANGE**: The `includeVFS` parameter has been removed:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// ❌ Old (v4.6.x and earlier)
|
// ❌ Old (v4.6.x and earlier)
|
||||||
|
|
@ -1351,7 +1350,7 @@ await brain.find({
|
||||||
includeVFS: true // No longer needed!
|
includeVFS: true // No longer needed!
|
||||||
})
|
})
|
||||||
|
|
||||||
// ✅ New (v4.7.0+)
|
// ✅ New
|
||||||
await brain.find({
|
await brain.find({
|
||||||
query: 'docs' // VFS included by default
|
query: 'docs' // VFS included by default
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -24,11 +24,11 @@ Where:
|
||||||
- `f` = number of fields for entity type
|
- `f` = number of fields for entity type
|
||||||
- `t` = number of types (42 nouns, 127 verbs)
|
- `t` = number of types (42 nouns, 127 verbs)
|
||||||
|
|
||||||
### v5.11.1: brain.get() Metadata-Only Optimization
|
### brain.get() Metadata-Only Optimization
|
||||||
|
|
||||||
✨ **Massive Performance Improvement**: `brain.get()` is now **76-81% faster** by default!
|
✨ **Massive Performance Improvement**: `brain.get()` is now **76-81% faster** by default!
|
||||||
|
|
||||||
| Operation | Before (v5.11.0) | After (v5.11.1) | Speedup | Use Case |
|
| Operation | Before | After | Speedup | Use Case |
|
||||||
|-----------|------------------|-----------------|---------|----------|
|
|-----------|------------------|-----------------|---------|----------|
|
||||||
| **brain.get() (metadata-only)** | 43ms, 6KB | **10ms, 300 bytes** | **76-81%** | VFS, existence checks, metadata |
|
| **brain.get() (metadata-only)** | 43ms, 6KB | **10ms, 300 bytes** | **76-81%** | VFS, existence checks, metadata |
|
||||||
| **brain.get({ includeVectors: true })** | 43ms, 6KB | 43ms, 6KB | 0% | Similarity calculations |
|
| **brain.get({ includeVectors: true })** | 43ms, 6KB | 43ms, 6KB | 0% | Similarity calculations |
|
||||||
|
|
@ -308,7 +308,7 @@ const results = await Promise.all(searchPromises)
|
||||||
- ✅ **Horizontally Scalable**: Stateless operations support clustering
|
- ✅ **Horizontally Scalable**: Stateless operations support clustering
|
||||||
- ✅ **Zero Stubs**: Every line of code is production-ready
|
- ✅ **Zero Stubs**: Every line of code is production-ready
|
||||||
|
|
||||||
## Lazy Loading Performance (v5.7.7+)
|
## Lazy Loading Performance
|
||||||
|
|
||||||
Brainy supports two initialization modes for optimal performance across different use cases:
|
Brainy supports two initialization modes for optimal performance across different use cases:
|
||||||
|
|
||||||
|
|
@ -324,7 +324,7 @@ await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities)
|
||||||
- First query: Instant (indexes already loaded)
|
- First query: Instant (indexes already loaded)
|
||||||
- Use case: Traditional applications, long-running servers
|
- Use case: Traditional applications, long-running servers
|
||||||
|
|
||||||
### Mode 2: Lazy Loading (v5.7.7+)
|
### Mode 2: Lazy Loading
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const brain = new Brainy({ disableAutoRebuild: true })
|
const brain = new Brainy({ disableAutoRebuild: true })
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
# Brainy Documentation (v6.5.0)
|
# Brainy Documentation
|
||||||
|
|
||||||
Welcome to the comprehensive documentation for Brainy, the multi-dimensional AI database with Triple Intelligence Engine.
|
Welcome to the comprehensive documentation for Brainy, the multi-dimensional AI database with Triple Intelligence Engine.
|
||||||
|
|
||||||
## 🆕 What's New in v4.0.0
|
## 🆕 What's New in
|
||||||
|
|
||||||
**Production-Ready Cost Optimization:**
|
**Production-Ready Cost Optimization:**
|
||||||
- **Lifecycle Management**: Automatic tier transitions for S3, GCS, Azure (96% cost savings!)
|
- **Lifecycle Management**: Automatic tier transitions for S3, GCS, Azure (96% cost savings!)
|
||||||
|
|
@ -14,7 +14,7 @@ Welcome to the comprehensive documentation for Brainy, the multi-dimensional AI
|
||||||
|
|
||||||
**Cost Impact Example (500TB dataset):**
|
**Cost Impact Example (500TB dataset):**
|
||||||
- Before: $138,000/year
|
- Before: $138,000/year
|
||||||
- After v4.0.0: $5,940/year
|
- After $5,940/year
|
||||||
- **Savings: $132,060/year (96%)**
|
- **Savings: $132,060/year (96%)**
|
||||||
|
|
||||||
## 📊 Implementation Status
|
## 📊 Implementation Status
|
||||||
|
|
@ -117,7 +117,7 @@ const results = await brain.find("highly rated technology articles by researcher
|
||||||
| [API Reference](./api/README.md) | **START HERE** - Complete API documentation with examples |
|
| [API Reference](./api/README.md) | **START HERE** - Complete API documentation with examples |
|
||||||
| [VFS Quick Start](./vfs/QUICK_START.md) | Virtual filesystem in 30 seconds |
|
| [VFS Quick Start](./vfs/QUICK_START.md) | Virtual filesystem in 30 seconds |
|
||||||
|
|
||||||
### 🆕 v4.0.0 Migration & Optimization
|
### 🆕 Migration & Optimization
|
||||||
|
|
||||||
| Document | Description |
|
| Document | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
|
|
@ -135,15 +135,15 @@ const results = await brain.find("highly rated technology articles by researcher
|
||||||
| [Noun-Verb Taxonomy](./architecture/noun-verb-taxonomy.md) | Revolutionary data model - entities and relationships |
|
| [Noun-Verb Taxonomy](./architecture/noun-verb-taxonomy.md) | Revolutionary data model - entities and relationships |
|
||||||
| [Triple Intelligence](./architecture/triple-intelligence.md) | Vector + Graph + Field unified query system |
|
| [Triple Intelligence](./architecture/triple-intelligence.md) | Vector + Graph + Field unified query system |
|
||||||
| [Zero Configuration](./architecture/zero-config.md) | Auto-adapts to any environment |
|
| [Zero Configuration](./architecture/zero-config.md) | Auto-adapts to any environment |
|
||||||
| [Storage Architecture](./architecture/storage-architecture.md) | **v4.0.0** - Storage adapters and optimization |
|
| [Storage Architecture](./architecture/storage-architecture.md) | Storage adapters and optimization |
|
||||||
| [Data Storage Architecture](./architecture/data-storage-architecture.md) | **v4.0.0** - Metadata/vector separation, sharding |
|
| [Data Storage Architecture](./architecture/data-storage-architecture.md) | Metadata/vector separation, sharding |
|
||||||
| [Index Architecture](./architecture/index-architecture.md) | HNSW, Graph, and Metadata indexing |
|
| [Index Architecture](./architecture/index-architecture.md) | HNSW, Graph, and Metadata indexing |
|
||||||
|
|
||||||
### 💾 Storage & Deployment
|
### 💾 Storage & Deployment
|
||||||
|
|
||||||
| Document | Description |
|
| Document | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| [Cloud Deployment Guide](./deployment/CLOUD_DEPLOYMENT_GUIDE.md) | **v4.0.0** - Deploy on AWS, GCP, Azure, Cloudflare |
|
| [Cloud Deployment Guide](./deployment/CLOUD_DEPLOYMENT_GUIDE.md) | Deploy on AWS, GCP, Azure, Cloudflare |
|
||||||
| [AWS Deployment](./deployment/aws-deployment.md) | AWS-specific deployment patterns |
|
| [AWS Deployment](./deployment/aws-deployment.md) | AWS-specific deployment patterns |
|
||||||
| [GCP Deployment](./deployment/gcp-deployment.md) | Google Cloud deployment |
|
| [GCP Deployment](./deployment/gcp-deployment.md) | Google Cloud deployment |
|
||||||
| [Kubernetes Deployment](./deployment/kubernetes-deployment.md) | K8s deployment configurations |
|
| [Kubernetes Deployment](./deployment/kubernetes-deployment.md) | K8s deployment configurations |
|
||||||
|
|
@ -258,7 +258,7 @@ const results = await brain.find("highly rated technology articles by researcher
|
||||||
```
|
```
|
||||||
docs/
|
docs/
|
||||||
├── README.md (this file) # Complete documentation index
|
├── README.md (this file) # Complete documentation index
|
||||||
├── MIGRATION-V3-TO-V4.md # v4.0.0 migration guide
|
├── MIGRATION-V3-TO-V4.md # Migration guide
|
||||||
│
|
│
|
||||||
├── guides/ # User guides
|
├── guides/ # User guides
|
||||||
│ ├── natural-language.md
|
│ ├── natural-language.md
|
||||||
|
|
@ -276,9 +276,7 @@ docs/
|
||||||
│ ├── noun-verb-taxonomy.md
|
│ ├── noun-verb-taxonomy.md
|
||||||
│ ├── triple-intelligence.md
|
│ ├── triple-intelligence.md
|
||||||
│ ├── zero-config.md
|
│ ├── zero-config.md
|
||||||
│ ├── storage-architecture.md # v4.0.0
|
│ ├── storage-architecture.md│ ├── data-storage-architecture.md│ ├── index-architecture.md
|
||||||
│ ├── data-storage-architecture.md # v4.0.0
|
|
||||||
│ ├── index-architecture.md
|
|
||||||
│ ├── distributed-storage.md
|
│ ├── distributed-storage.md
|
||||||
│ ├── augmentations.md
|
│ ├── augmentations.md
|
||||||
│ ├── augmentation-system-audit.md
|
│ ├── augmentation-system-audit.md
|
||||||
|
|
@ -287,15 +285,10 @@ docs/
|
||||||
│ └── ...
|
│ └── ...
|
||||||
│
|
│
|
||||||
├── operations/ # Operations guides
|
├── operations/ # Operations guides
|
||||||
│ ├── cost-optimization-aws-s3.md # v4.0.0
|
│ ├── cost-optimization-aws-s3.md│ ├── cost-optimization-gcs.md│ ├── cost-optimization-azure.md│ ├── cost-optimization-cloudflare-r2.md│ └── capacity-planning.md
|
||||||
│ ├── cost-optimization-gcs.md # v4.0.0
|
|
||||||
│ ├── cost-optimization-azure.md # v4.0.0
|
|
||||||
│ ├── cost-optimization-cloudflare-r2.md # v4.0.0
|
|
||||||
│ └── capacity-planning.md
|
|
||||||
│
|
│
|
||||||
├── deployment/ # Deployment guides
|
├── deployment/ # Deployment guides
|
||||||
│ ├── CLOUD_DEPLOYMENT_GUIDE.md # v4.0.0
|
│ ├── CLOUD_DEPLOYMENT_GUIDE.md│ ├── aws-deployment.md
|
||||||
│ ├── aws-deployment.md
|
|
||||||
│ ├── gcp-deployment.md
|
│ ├── gcp-deployment.md
|
||||||
│ └── kubernetes-deployment.md
|
│ └── kubernetes-deployment.md
|
||||||
│
|
│
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ git commit -m "chore: remove unused dependency"
|
||||||
# DON'T use BREAKING CHANGE for internal changes
|
# DON'T use BREAKING CHANGE for internal changes
|
||||||
git commit -m "feat: improve model delivery
|
git commit -m "feat: improve model delivery
|
||||||
|
|
||||||
BREAKING CHANGE: removed tar-stream dependency" # WRONG! This triggers v3.0.0
|
BREAKING CHANGE: removed tar-stream dependency" # WRONG! This triggers
|
||||||
```
|
```
|
||||||
|
|
||||||
## Release Workflow Checklist
|
## Release Workflow Checklist
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
# Brainy Stage 3: Canonical Taxonomy (v6.0.0)
|
# Brainy Stage 3: Canonical Taxonomy
|
||||||
|
|
||||||
**Status:** FINAL - This is the definitive, timeless taxonomy
|
**Status:** FINAL - This is the definitive, timeless taxonomy
|
||||||
**Total Types:** 169 (42 nouns + 127 verbs)
|
**Total Types:** 169 (42 nouns + 127 verbs)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Brainy Complete Public API Reference
|
# Brainy Complete Public API Reference
|
||||||
|
|
||||||
> **Accurate API documentation for Brainy v6.5.0+**
|
> **Accurate API documentation for Brainy**
|
||||||
|
|
||||||
## Initialization
|
## Initialization
|
||||||
|
|
||||||
|
|
@ -20,7 +20,7 @@ const brain = new Brainy({
|
||||||
await brain.init()
|
await brain.init()
|
||||||
```
|
```
|
||||||
|
|
||||||
## Readiness API (v7.3.0+)
|
## Readiness API
|
||||||
|
|
||||||
For reliable initialization detection, especially in cloud environments with progressive initialization:
|
For reliable initialization detection, especially in cloud environments with progressive initialization:
|
||||||
|
|
||||||
|
|
@ -355,7 +355,7 @@ await vfs.mkdir('/project/src', { recursive: true })
|
||||||
const files = await vfs.readdir('/project')
|
const files = await vfs.readdir('/project')
|
||||||
await vfs.rmdir('/project', { recursive: true })
|
await vfs.rmdir('/project', { recursive: true })
|
||||||
|
|
||||||
// Bulk operations (v6.5.0+)
|
// Bulk operations
|
||||||
const result = await vfs.bulkWrite([
|
const result = await vfs.bulkWrite([
|
||||||
{ type: 'mkdir', path: '/data' },
|
{ type: 'mkdir', path: '/data' },
|
||||||
{ type: 'write', path: '/data/config.json', data: '{}' },
|
{ type: 'write', path: '/data/config.json', data: '{}' },
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
# Brainy Data Storage Architecture (v5.11.0)
|
# Brainy Data Storage Architecture
|
||||||
|
|
||||||
**Complete file structure reference for all storage backends**
|
**Complete file structure reference for all storage backends**
|
||||||
|
|
||||||
|
|
@ -13,7 +13,7 @@ This document explains how Brainy stores, indexes, and scales data across all st
|
||||||
3. [The 4 Indexes](#3-the-4-indexes)
|
3. [The 4 Indexes](#3-the-4-indexes)
|
||||||
4. [Sharding Strategy](#4-sharding-strategy)
|
4. [Sharding Strategy](#4-sharding-strategy)
|
||||||
5. [COW (Copy-on-Write) Architecture](#5-cow-copy-on-write-architecture)
|
5. [COW (Copy-on-Write) Architecture](#5-cow-copy-on-write-architecture)
|
||||||
6. [ID-First Storage Architecture (v6.0.0+)](#6-id-first-storage-architecture-v600)
|
6. [ID-First Storage Architecture](#6-id-first-storage-architecture-v600)
|
||||||
7. [VFS (Virtual File System)](#7-vfs-virtual-file-system)
|
7. [VFS (Virtual File System)](#7-vfs-virtual-file-system)
|
||||||
8. [Storage Backend Mapping](#8-storage-backend-mapping)
|
8. [Storage Backend Mapping](#8-storage-backend-mapping)
|
||||||
9. [Performance Characteristics](#9-performance-characteristics)
|
9. [Performance Characteristics](#9-performance-characteristics)
|
||||||
|
|
@ -438,7 +438,7 @@ Unlike entities and relationships, system metadata consists of **index files** t
|
||||||
|
|
||||||
Understanding how Brainy constructs storage paths is critical for debugging and optimization.
|
Understanding how Brainy constructs storage paths is critical for debugging and optimization.
|
||||||
|
|
||||||
### Path Construction Steps (v6.0.0+)
|
### Path Construction Steps
|
||||||
|
|
||||||
**For an entity (noun)**:
|
**For an entity (noun)**:
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -498,7 +498,7 @@ const fieldIndexPath = `_system/metadata_indexes/__metadata_field_index__${field
|
||||||
const hnswNodePath = `_system/hnsw/nodes/${shard}/${entityId}.json`
|
const hnswNodePath = `_system/hnsw/nodes/${shard}/${entityId}.json`
|
||||||
```
|
```
|
||||||
|
|
||||||
### Path Patterns Summary (v6.0.0+)
|
### Path Patterns Summary
|
||||||
|
|
||||||
| Data Type | Path Pattern | Sharded? | Branched? |
|
| Data Type | Path Pattern | Sharded? | Branched? |
|
||||||
|-----------|--------------|----------|-----------|
|
|-----------|--------------|----------|-----------|
|
||||||
|
|
@ -516,7 +516,7 @@ const hnswNodePath = `_system/hnsw/nodes/${shard}/${entityId}.json`
|
||||||
| **HNSW node** | `_system/hnsw/nodes/{shard}/{uuid}.json` | ✅ Yes (UUID) | ❌ No |
|
| **HNSW node** | `_system/hnsw/nodes/{shard}/{uuid}.json` | ✅ Yes (UUID) | ❌ No |
|
||||||
| **Field index** | `_system/metadata_indexes/__metadata_field_index__{field}.json` | ❌ No | ❌ No |
|
| **Field index** | `_system/metadata_indexes/__metadata_field_index__{field}.json` | ❌ No | ❌ No |
|
||||||
|
|
||||||
### Key Principles (v6.0.0+)
|
### Key Principles
|
||||||
|
|
||||||
1. **Shard Extraction**: Always use first 2 hex characters of UUID/SHA-256
|
1. **Shard Extraction**: Always use first 2 hex characters of UUID/SHA-256
|
||||||
2. **ID-First**: Shard + ID come BEFORE type (type is in metadata)
|
2. **ID-First**: Shard + ID come BEFORE type (type is in metadata)
|
||||||
|
|
@ -549,7 +549,7 @@ Brainy uses four complementary index systems for different query patterns.
|
||||||
- Memory (standard): ~200MB per 100K entities
|
- Memory (standard): ~200MB per 100K entities
|
||||||
- Memory (lazy): ~15-33MB per 100K entities (5-10x less!)
|
- Memory (lazy): ~15-33MB per 100K entities (5-10x less!)
|
||||||
|
|
||||||
**Automatic Lazy Mode** (v3.36.0+): Enables automatically when vectors don't fit in UnifiedCache
|
**Automatic Lazy Mode**: Enables automatically when vectors don't fit in UnifiedCache
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -559,7 +559,7 @@ Brainy uses four complementary index systems for different query patterns.
|
||||||
**Location**: MetadataIndexManager index on `noun` field
|
**Location**: MetadataIndexManager index on `noun` field
|
||||||
**Data Structure**: RoaringBitmap32 per type value
|
**Data Structure**: RoaringBitmap32 per type value
|
||||||
|
|
||||||
**How It Works** (v6.0.0+):
|
**How It Works**:
|
||||||
```typescript
|
```typescript
|
||||||
// Find all Person entities
|
// Find all Person entities
|
||||||
const people = await brain.getNouns({ type: 'person' })
|
const people = await brain.getNouns({ type: 'person' })
|
||||||
|
|
@ -721,7 +721,7 @@ COW is Brainy's **git-like versioning system** that enables:
|
||||||
- ✅ **Deduplication** (identical data stored only once)
|
- ✅ **Deduplication** (identical data stored only once)
|
||||||
- ✅ **Version history** (full audit trail of all changes)
|
- ✅ **Version history** (full audit trail of all changes)
|
||||||
|
|
||||||
**Status**: ALWAYS ENABLED (v5.11.0+) - cannot be disabled
|
**Status**: ALWAYS ENABLED - cannot be disabled
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -782,19 +782,19 @@ _cow/blobs/ab/abc123...sha256.bin (used by both entities)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. ID-First Storage Architecture (v6.0.0+)
|
## 6. ID-First Storage Architecture
|
||||||
|
|
||||||
### 6.1 What is ID-First?
|
### 6.1 What is ID-First?
|
||||||
|
|
||||||
**ID-first storage** organizes entities by **ID shard only** - no type directories! This eliminates 42-type sequential searches that caused 20-21 second delays on cloud storage.
|
**ID-first storage** organizes entities by **ID shard only** - no type directories! This eliminates 42-type sequential searches that caused 20-21 second delays on cloud storage.
|
||||||
|
|
||||||
**Old type-first structure** (v5.4.0-v5.12.0):
|
**Old type-first structure** (v5.12.0):
|
||||||
```
|
```
|
||||||
branches/main/entities/nouns/{TYPE}/metadata/00/001234...uuid.json
|
branches/main/entities/nouns/{TYPE}/metadata/00/001234...uuid.json
|
||||||
# Problem: Requires knowing type OR searching 42 type directories!
|
# Problem: Requires knowing type OR searching 42 type directories!
|
||||||
```
|
```
|
||||||
|
|
||||||
**NEW ID-first structure** (v6.0.0+):
|
**NEW ID-first structure**:
|
||||||
```
|
```
|
||||||
branches/main/entities/nouns/00/001234...uuid/metadata.json
|
branches/main/entities/nouns/00/001234...uuid/metadata.json
|
||||||
# Direct O(1) lookup - no type needed!
|
# Direct O(1) lookup - no type needed!
|
||||||
|
|
@ -809,7 +809,7 @@ branches/main/entities/nouns/00/001234...uuid/metadata.json
|
||||||
// v5.x: Had to search 42 types if type unknown
|
// v5.x: Had to search 42 types if type unknown
|
||||||
// Result: 21 seconds on GCS (42 types × 500ms)
|
// Result: 21 seconds on GCS (42 types × 500ms)
|
||||||
|
|
||||||
// v6.0.0: Direct path from ID
|
// Direct path from ID
|
||||||
const id = '001234...'
|
const id = '001234...'
|
||||||
const shard = id.substring(0, 2) // '00'
|
const shard = id.substring(0, 2) // '00'
|
||||||
const path = `branches/main/entities/nouns/${shard}/${id}/metadata.json`
|
const path = `branches/main/entities/nouns/${shard}/${id}/metadata.json`
|
||||||
|
|
@ -1165,7 +1165,7 @@ opfs://root/brainy/
|
||||||
|
|
||||||
### 10.2 clear() Operation
|
### 10.2 clear() Operation
|
||||||
|
|
||||||
**What clear() deletes** (v5.11.0+):
|
**What clear() deletes**:
|
||||||
|
|
||||||
✅ Deletes:
|
✅ Deletes:
|
||||||
- `branches/` → ALL entity data (all types, all shards, all branches, all forks)
|
- `branches/` → ALL entity data (all types, all shards, all branches, all forks)
|
||||||
|
|
@ -1184,7 +1184,7 @@ opfs://root/brainy/
|
||||||
|
|
||||||
**Example**:
|
**Example**:
|
||||||
```typescript
|
```typescript
|
||||||
await brain.storage.clear() // ✅ Deletes ALL data correctly (v5.11.0+)
|
await brain.storage.clear() // ✅ Deletes ALL data correctly
|
||||||
await brain.add({ data: 'Alice', type: 'person' }) // ✅ COW reinitializes automatically
|
await brain.add({ data: 'Alice', type: 'person' }) // ✅ COW reinitializes automatically
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -1353,7 +1353,7 @@ const historicalData = await yesterday.getNouns({ type: 'Character' })
|
||||||
await brain.storage.clear()
|
await brain.storage.clear()
|
||||||
```
|
```
|
||||||
|
|
||||||
**What happens in storage** (v5.11.0+):
|
**What happens in storage**:
|
||||||
```
|
```
|
||||||
1. Delete all entity data:
|
1. Delete all entity data:
|
||||||
→ Remove: branches/ (entire directory)
|
→ Remove: branches/ (entire directory)
|
||||||
|
|
@ -1452,7 +1452,7 @@ await brain.addBatch([
|
||||||
|
|
||||||
## 11. Summary
|
## 11. Summary
|
||||||
|
|
||||||
**Complete Storage Structure (v6.0.0)**:
|
**Complete Storage Structure**:
|
||||||
- **3 storage layers**: branches/ (data), _cow/ (versions), _system/ (indexes)
|
- **3 storage layers**: branches/ (data), _cow/ (versions), _system/ (indexes)
|
||||||
- **2 files per entity**: metadata.json + vector.json (optimized I/O)
|
- **2 files per entity**: metadata.json + vector.json (optimized I/O)
|
||||||
- **4 indexes**: HNSW (semantic), Type Index (metadata-based), Graph (relationships), Metadata (fields)
|
- **4 indexes**: HNSW (semantic), Type Index (metadata-based), Graph (relationships), Metadata (fields)
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ Brainy has **3 main indexes** at the top level, each with multiple sub-indexes m
|
||||||
- **FieldTypeInference** - DuckDB-inspired value-based field type detection
|
- **FieldTypeInference** - DuckDB-inspired value-based field type detection
|
||||||
- **Field Sparse Indexes** - Per-field sparse indexes with roaring bitmaps (dynamic count)
|
- **Field Sparse Indexes** - Per-field sparse indexes with roaring bitmaps (dynamic count)
|
||||||
- **Sorted Indexes** - Support orderBy queries (automatically maintained)
|
- **Sorted Indexes** - Support orderBy queries (automatically maintained)
|
||||||
- **Word Index (`__words__`)** - Text search via FNV-1a word hashes (v7.7.0)
|
- **Word Index (`__words__`)** - Text search via FNV-1a word hashes
|
||||||
|
|
||||||
**GraphAdjacencyIndex contains:**
|
**GraphAdjacencyIndex contains:**
|
||||||
- **lsmTreeSource** - Source → Targets (outgoing edges)
|
- **lsmTreeSource** - Source → Targets (outgoing edges)
|
||||||
|
|
@ -39,7 +39,7 @@ All indexes share a **UnifiedCache** for coordinated memory management, ensuring
|
||||||
|
|
||||||
**Purpose**: Enable O(1) field-value lookups and O(log n) range queries on metadata fields using adaptive chunked sparse indexing.
|
**Purpose**: Enable O(1) field-value lookups and O(log n) range queries on metadata fields using adaptive chunked sparse indexing.
|
||||||
|
|
||||||
### Internal Architecture (v3.42.0)
|
### Internal Architecture
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
class MetadataIndexManager {
|
class MetadataIndexManager {
|
||||||
|
|
@ -64,7 +64,7 @@ class MetadataIndexManager {
|
||||||
|
|
||||||
### Key Data Structures
|
### Key Data Structures
|
||||||
|
|
||||||
#### Chunked Sparse Index (NEW in v3.42.0)
|
#### Chunked Sparse Index
|
||||||
```typescript
|
```typescript
|
||||||
// SparseIndex: Directory of chunks for a field
|
// SparseIndex: Directory of chunks for a field
|
||||||
// Example: field="status"
|
// Example: field="status"
|
||||||
|
|
@ -87,7 +87,7 @@ interface ChunkDescriptor {
|
||||||
class ChunkData {
|
class ChunkData {
|
||||||
chunkId: number
|
chunkId: number
|
||||||
field: string
|
field: string
|
||||||
entries: Map<value, RoaringBitmap32> // ~50 values per chunk (v3.43.0: roaring bitmaps!)
|
entries: Map<value, RoaringBitmap32> // ~50 values per chunk (roaring bitmaps!)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -96,7 +96,7 @@ class ChunkData {
|
||||||
- O(log n) range queries with zone maps
|
- O(log n) range queries with zone maps
|
||||||
- 630x file reduction (560k flat files → 89 chunk files)
|
- 630x file reduction (560k flat files → 89 chunk files)
|
||||||
|
|
||||||
#### Roaring Bitmap Optimization (NEW in v3.43.0)
|
#### Roaring Bitmap Optimization
|
||||||
|
|
||||||
**Problem Solved**: JavaScript `Set<string>` for storing entity IDs was inefficient:
|
**Problem Solved**: JavaScript `Set<string>` for storing entity IDs was inefficient:
|
||||||
- Memory overhead: ~40 bytes per UUID string (36 chars + overhead)
|
- Memory overhead: ~40 bytes per UUID string (36 chars + overhead)
|
||||||
|
|
@ -150,7 +150,7 @@ class ChunkData {
|
||||||
|
|
||||||
**Multi-Field Intersection (THE BIG WIN!)**:
|
**Multi-Field Intersection (THE BIG WIN!)**:
|
||||||
```typescript
|
```typescript
|
||||||
// Before (v3.42.0): JavaScript array filtering
|
// Before: JavaScript array filtering
|
||||||
async getIdsForFilter(filter: {status: 'active', role: 'admin'}): Promise<string[]> {
|
async getIdsForFilter(filter: {status: 'active', role: 'admin'}): Promise<string[]> {
|
||||||
// 1. Fetch UUID arrays for each field
|
// 1. Fetch UUID arrays for each field
|
||||||
const statusIds = await this.getIds('status', 'active') // ["uuid1", "uuid2", ...]
|
const statusIds = await this.getIds('status', 'active') // ["uuid1", "uuid2", ...]
|
||||||
|
|
@ -160,7 +160,7 @@ async getIdsForFilter(filter: {status: 'active', role: 'admin'}): Promise<string
|
||||||
return statusIds.filter(id => roleIds.includes(id)) // O(n*m) array filtering
|
return statusIds.filter(id => roleIds.includes(id)) // O(n*m) array filtering
|
||||||
}
|
}
|
||||||
|
|
||||||
// After (v3.43.0): Roaring bitmap intersection
|
// After: Roaring bitmap intersection
|
||||||
async getIdsForMultipleFields(pairs: [{field, value}, ...]): Promise<string[]> {
|
async getIdsForMultipleFields(pairs: [{field, value}, ...]): Promise<string[]> {
|
||||||
// 1. Fetch roaring bitmaps (integers, not UUIDs)
|
// 1. Fetch roaring bitmaps (integers, not UUIDs)
|
||||||
const bitmaps: RoaringBitmap32[] = []
|
const bitmaps: RoaringBitmap32[] = []
|
||||||
|
|
@ -229,7 +229,7 @@ interface ZoneMap {
|
||||||
|
|
||||||
**Use case**: Enables NLP to understand "find characters named John" → knows 'name' is a character field
|
**Use case**: Enables NLP to understand "find characters named John" → knows 'name' is a character field
|
||||||
|
|
||||||
#### Word Index (`__words__`) - v7.7.0
|
#### Word Index (`__words__`) -
|
||||||
```typescript
|
```typescript
|
||||||
// Special field for text/keyword search
|
// Special field for text/keyword search
|
||||||
// Entity text content is tokenized and indexed as word hashes
|
// Entity text content is tokenized and indexed as word hashes
|
||||||
|
|
@ -253,14 +253,14 @@ interface ZoneMap {
|
||||||
- **Lowercase normalization**: Case-insensitive matching
|
- **Lowercase normalization**: Case-insensitive matching
|
||||||
- **Automatic integration**: Words extracted via `extractIndexableFields()`
|
- **Automatic integration**: Words extracted via `extractIndexableFields()`
|
||||||
|
|
||||||
**Hybrid Search** (v7.7.0): Text results combined with vector results using Reciprocal Rank Fusion (RRF):
|
**Hybrid Search**: Text results combined with vector results using Reciprocal Rank Fusion (RRF):
|
||||||
```typescript
|
```typescript
|
||||||
// RRF formula: score(d) = sum(1 / (k + rank(d)))
|
// RRF formula: score(d) = sum(1 / (k + rank(d)))
|
||||||
// where k = 60 (standard constant)
|
// where k = 60 (standard constant)
|
||||||
// alpha = weight for semantic (0 = text only, 1 = semantic only)
|
// alpha = weight for semantic (0 = text only, 1 = semantic only)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Query Algorithm (v3.42.0)
|
### Query Algorithm
|
||||||
|
|
||||||
**Exact Match Query**:
|
**Exact Match Query**:
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -317,7 +317,7 @@ async getIdsForRange(field: string, min: any, max: any): Promise<string[]> {
|
||||||
- Adaptive chunking: ~50 values per chunk optimizes I/O
|
- Adaptive chunking: ~50 values per chunk optimizes I/O
|
||||||
- Immediate flushing: No need for dirty tracking or batch writes
|
- Immediate flushing: No need for dirty tracking or batch writes
|
||||||
|
|
||||||
### Temporal Bucketing (v3.41.0)
|
### Temporal Bucketing
|
||||||
|
|
||||||
**Problem Solved**: High-cardinality timestamp fields created massive file pollution.
|
**Problem Solved**: High-cardinality timestamp fields created massive file pollution.
|
||||||
- Example: 575 entities with unique timestamps → 358,407 index files (98.7% pollution!)
|
- Example: 575 entities with unique timestamps → 358,407 index files (98.7% pollution!)
|
||||||
|
|
@ -396,7 +396,7 @@ const DEFAULT_EXCLUDE_FIELDS = [
|
||||||
]
|
]
|
||||||
```
|
```
|
||||||
|
|
||||||
**Note**: Timestamp fields like `modified`, `accessed`, `created` are NO LONGER excluded as of v3.41.0 - they are indexed with automatic bucketing.
|
**Note**: Timestamp fields like `modified`, `accessed`, `created` are NO LONGER excluded as of they are indexed with automatic bucketing.
|
||||||
|
|
||||||
## 2. HNSWIndex - Vector Similarity Search
|
## 2. HNSWIndex - Vector Similarity Search
|
||||||
|
|
||||||
|
|
@ -735,7 +735,7 @@ async stats(): Promise<Statistics> {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5. Index Rebuilding (v5.7.7: Lazy Loading Support)
|
### 5. Index Rebuilding (Lazy Loading Support)
|
||||||
|
|
||||||
**Two modes of index loading:**
|
**Two modes of index loading:**
|
||||||
|
|
||||||
|
|
@ -762,7 +762,7 @@ async init(): Promise<void> {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Mode 2: Lazy Loading on First Query (v5.7.7+)
|
#### Mode 2: Lazy Loading on First Query
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// When disableAutoRebuild: true
|
// When disableAutoRebuild: true
|
||||||
|
|
@ -918,7 +918,7 @@ All indexes scale gracefully:
|
||||||
- [Performance Guide](../PERFORMANCE.md) - Performance tuning
|
- [Performance Guide](../PERFORMANCE.md) - Performance tuning
|
||||||
- [Overview](./overview.md) - High-level architecture
|
- [Overview](./overview.md) - High-level architecture
|
||||||
|
|
||||||
## Summary: Index Hierarchy (v5.7.7)
|
## Summary: Index Hierarchy
|
||||||
|
|
||||||
### Level 1: Main Indexes (3)
|
### Level 1: Main Indexes (3)
|
||||||
All have rebuild() methods and are covered by lazy loading:
|
All have rebuild() methods and are covered by lazy loading:
|
||||||
|
|
@ -933,7 +933,7 @@ Automatically managed by parent rebuild():
|
||||||
- **4 LSM-trees** (lsmTreeSource, lsmTreeTarget, lsmTreeVerbsBySource, lsmTreeVerbsByTarget)
|
- **4 LSM-trees** (lsmTreeSource, lsmTreeTarget, lsmTreeVerbsBySource, lsmTreeVerbsByTarget)
|
||||||
- **In-memory graph structures** (sourceIndex, targetIndex, verbIndex)
|
- **In-memory graph structures** (sourceIndex, targetIndex, verbIndex)
|
||||||
|
|
||||||
### Lazy Loading (v5.7.7)
|
### Lazy Loading
|
||||||
- **Mode 1**: Auto-rebuild on init() (default)
|
- **Mode 1**: Auto-rebuild on init() (default)
|
||||||
- **Mode 2**: Lazy rebuild on first query (when `disableAutoRebuild: true`)
|
- **Mode 2**: Lazy rebuild on first query (when `disableAutoRebuild: true`)
|
||||||
- **Concurrency-safe**: Mutex prevents duplicate rebuilds
|
- **Concurrency-safe**: Mutex prevents duplicate rebuilds
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ This document explains how Brainy's four indexes (MetadataIndex, HNSWIndex, Grap
|
||||||
| **GraphAdjacencyIndex** | Relationships via LSM-tree SSTables | LSM-tree auto-persistence | v3.44.0 |
|
| **GraphAdjacencyIndex** | Relationships via LSM-tree SSTables | LSM-tree auto-persistence | v3.44.0 |
|
||||||
| **DeletedItemsIndex** | Set of deleted IDs | `storage.saveDeletedItems()` | v3.0.0 |
|
| **DeletedItemsIndex** | Set of deleted IDs | `storage.saveDeletedItems()` | v3.0.0 |
|
||||||
|
|
||||||
#### MetadataIndex Persistence Details (v4.2.1+)
|
#### MetadataIndex Persistence Details
|
||||||
|
|
||||||
The MetadataIndex now persists two components:
|
The MetadataIndex now persists two components:
|
||||||
|
|
||||||
|
|
@ -129,7 +129,7 @@ async init(): Promise<void> {
|
||||||
- 100-2000ms: One-time rebuild to create indices
|
- 100-2000ms: One-time rebuild to create indices
|
||||||
- Total: ~1-3 seconds (one time only)
|
- Total: ~1-3 seconds (one time only)
|
||||||
|
|
||||||
#### Mode 2: Lazy Loading on First Query (v5.7.7+)
|
#### Mode 2: Lazy Loading on First Query
|
||||||
|
|
||||||
When `disableAutoRebuild: true`, indexes remain empty after init() and rebuild on first query:
|
When `disableAutoRebuild: true`, indexes remain empty after init() and rebuild on first query:
|
||||||
|
|
||||||
|
|
@ -350,7 +350,7 @@ public async rebuild(options?: {
|
||||||
|
|
||||||
**Performance Impact**: 200-600x speedup (5 minutes → 500ms for 10K entities)
|
**Performance Impact**: 200-600x speedup (5 minutes → 500ms for 10K entities)
|
||||||
|
|
||||||
**Correct Pattern** (v3.45.0):
|
**Correct Pattern**:
|
||||||
```typescript
|
```typescript
|
||||||
// Load ALL nouns ONCE (not 31 times!)
|
// Load ALL nouns ONCE (not 31 times!)
|
||||||
while (hasMore) {
|
while (hasMore) {
|
||||||
|
|
@ -392,7 +392,7 @@ while (hasMore) {
|
||||||
```typescript
|
```typescript
|
||||||
// src/utils/metadataIndex.ts (lines 202-216)
|
// src/utils/metadataIndex.ts (lines 202-216)
|
||||||
async init(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
// STEP 1: Load field registry to discover persisted indices (v4.2.1)
|
// STEP 1: Load field registry to discover persisted indices
|
||||||
// This is THE KEY FIX - O(1) discovery of existing indices
|
// This is THE KEY FIX - O(1) discovery of existing indices
|
||||||
await this.loadFieldRegistry()
|
await this.loadFieldRegistry()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
# Storage Architecture (v4.0.0)
|
# Storage Architecture
|
||||||
|
|
||||||
> **Updated for v4.0.0**: Metadata/vector separation, UUID-based sharding, lifecycle management
|
> **Updated**: Metadata/vector separation, UUID-based sharding, lifecycle management
|
||||||
|
|
||||||
## Storage Structure
|
## Storage Structure
|
||||||
|
|
||||||
### v4.0.0 Architecture: Metadata/Vector Separation
|
### Architecture: Metadata/Vector Separation
|
||||||
|
|
||||||
In v4.0.0, entities and relationships are split into **2 separate files** for optimal performance at billion-entity scale:
|
entities and relationships are split into **2 separate files** for optimal performance at billion-entity scale:
|
||||||
|
|
||||||
```
|
```
|
||||||
brainy-data/
|
brainy-data/
|
||||||
|
|
@ -64,7 +64,7 @@ const shard = uuid.substring(0, 2) // "3f"
|
||||||
|
|
||||||
## Storage Adapters
|
## Storage Adapters
|
||||||
|
|
||||||
Brainy provides multiple storage adapters with identical APIs and v4.0.0 production features:
|
Brainy provides multiple storage adapters with identical APIs and production features:
|
||||||
|
|
||||||
### FileSystem Storage (Node.js)
|
### FileSystem Storage (Node.js)
|
||||||
```typescript
|
```typescript
|
||||||
|
|
@ -72,14 +72,14 @@ const brain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'filesystem',
|
type: 'filesystem',
|
||||||
path: './data',
|
path: './data',
|
||||||
compression: true // v4.0.0: Gzip compression (60-80% space savings)
|
compression: true // Gzip compression (60-80% space savings)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
- **Use case**: Server applications, CLI tools
|
- **Use case**: Server applications, CLI tools
|
||||||
- **Performance**: Direct file I/O with optional compression
|
- **Performance**: Direct file I/O with optional compression
|
||||||
- **Persistence**: Permanent on disk
|
- **Persistence**: Permanent on disk
|
||||||
- **v4.0.0 Features**:
|
- **Features**:
|
||||||
- **Gzip Compression**: 60-80% storage savings with minimal CPU overhead
|
- **Gzip Compression**: 60-80% storage savings with minimal CPU overhead
|
||||||
- **Batch Delete**: Efficient bulk deletion with retries
|
- **Batch Delete**: Efficient bulk deletion with retries
|
||||||
- **UUID Sharding**: Automatic 256-shard distribution
|
- **UUID Sharding**: Automatic 256-shard distribution
|
||||||
|
|
@ -101,7 +101,7 @@ const brain = new Brainy({
|
||||||
- **Use case**: Distributed applications, cloud deployments
|
- **Use case**: Distributed applications, cloud deployments
|
||||||
- **Performance**: Network dependent, with intelligent caching
|
- **Performance**: Network dependent, with intelligent caching
|
||||||
- **Persistence**: Cloud storage durability (99.999999999%)
|
- **Persistence**: Cloud storage durability (99.999999999%)
|
||||||
- **v4.0.0 Features**:
|
- **Features**:
|
||||||
- **Lifecycle Policies**: Automatic tier transitions (Standard → IA → Glacier → Deep Archive)
|
- **Lifecycle Policies**: Automatic tier transitions (Standard → IA → Glacier → Deep Archive)
|
||||||
- **Intelligent-Tiering**: Automatic optimization based on access patterns (up to 95% savings)
|
- **Intelligent-Tiering**: Automatic optimization based on access patterns (up to 95% savings)
|
||||||
- **Batch Delete**: Efficient bulk deletion (1000 objects per request)
|
- **Batch Delete**: Efficient bulk deletion (1000 objects per request)
|
||||||
|
|
@ -120,7 +120,7 @@ const brain = new Brainy({
|
||||||
- **Use case**: Google Cloud deployments
|
- **Use case**: Google Cloud deployments
|
||||||
- **Performance**: Global CDN with edge caching
|
- **Performance**: Global CDN with edge caching
|
||||||
- **Persistence**: 99.999999999% durability
|
- **Persistence**: 99.999999999% durability
|
||||||
- **v4.0.0 Features**:
|
- **Features**:
|
||||||
- **Lifecycle Policies**: Automatic tier transitions (Standard → Nearline → Coldline → Archive)
|
- **Lifecycle Policies**: Automatic tier transitions (Standard → Nearline → Coldline → Archive)
|
||||||
- **Autoclass**: Intelligent automatic tier optimization
|
- **Autoclass**: Intelligent automatic tier optimization
|
||||||
- **Batch Delete**: Efficient bulk operations
|
- **Batch Delete**: Efficient bulk operations
|
||||||
|
|
@ -139,7 +139,7 @@ const brain = new Brainy({
|
||||||
- **Use case**: Azure cloud deployments
|
- **Use case**: Azure cloud deployments
|
||||||
- **Performance**: Global replication with CDN
|
- **Performance**: Global replication with CDN
|
||||||
- **Persistence**: LRS, ZRS, GRS, RA-GRS options
|
- **Persistence**: LRS, ZRS, GRS, RA-GRS options
|
||||||
- **v4.0.0 Features**:
|
- **Features**:
|
||||||
- **Blob Tier Management**: Hot/Cool/Archive tiers (99% cost savings)
|
- **Blob Tier Management**: Hot/Cool/Archive tiers (99% cost savings)
|
||||||
- **Lifecycle Policies**: Automatic tier transitions and deletions
|
- **Lifecycle Policies**: Automatic tier transitions and deletions
|
||||||
- **Batch Delete**: BlobBatchClient for efficient bulk operations
|
- **Batch Delete**: BlobBatchClient for efficient bulk operations
|
||||||
|
|
@ -157,7 +157,7 @@ const brain = new Brainy({
|
||||||
- **Use case**: Browser applications, PWAs
|
- **Use case**: Browser applications, PWAs
|
||||||
- **Performance**: Near-native file system speed
|
- **Performance**: Near-native file system speed
|
||||||
- **Persistence**: Permanent in browser (with quota limits)
|
- **Persistence**: Permanent in browser (with quota limits)
|
||||||
- **v4.0.0 Features**:
|
- **Features**:
|
||||||
- **Quota Monitoring**: Real-time quota tracking and warnings
|
- **Quota Monitoring**: Real-time quota tracking and warnings
|
||||||
- **Batch Delete**: Efficient bulk deletion
|
- **Batch Delete**: Efficient bulk deletion
|
||||||
- **Storage Status**: Detailed usage/available reporting
|
- **Storage Status**: Detailed usage/available reporting
|
||||||
|
|
@ -244,7 +244,7 @@ Ensures durability and enables recovery:
|
||||||
2. Replay operations from last checkpoint
|
2. Replay operations from last checkpoint
|
||||||
3. Verify checksums for integrity
|
3. Verify checksums for integrity
|
||||||
|
|
||||||
## Storage Optimization (v4.0.0)
|
## Storage Optimization
|
||||||
|
|
||||||
### 1. Lifecycle Policies (Cloud Storage)
|
### 1. Lifecycle Policies (Cloud Storage)
|
||||||
|
|
||||||
|
|
@ -359,7 +359,7 @@ const brain = new Brainy({
|
||||||
### 5. Batch Operations
|
### 5. Batch Operations
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// v4.0.0: Efficient batch delete
|
// Efficient batch delete
|
||||||
await storage.batchDelete([
|
await storage.batchDelete([
|
||||||
'entities/nouns/vectors/00/00123456-....json',
|
'entities/nouns/vectors/00/00123456-....json',
|
||||||
'entities/nouns/metadata/00/00123456-....json',
|
'entities/nouns/metadata/00/00123456-....json',
|
||||||
|
|
@ -522,7 +522,7 @@ console.log(stats)
|
||||||
// }
|
// }
|
||||||
```
|
```
|
||||||
|
|
||||||
## Best Practices (v4.0.0)
|
## Best Practices
|
||||||
|
|
||||||
### Choose the Right Adapter
|
### Choose the Right Adapter
|
||||||
1. **Development**: FileSystem with compression (local persistence, small storage footprint)
|
1. **Development**: FileSystem with compression (local persistence, small storage footprint)
|
||||||
|
|
@ -537,7 +537,7 @@ console.log(stats)
|
||||||
4. **Archival**: Cloud storage with lifecycle policies (96% cost savings!)
|
4. **Archival**: Cloud storage with lifecycle policies (96% cost savings!)
|
||||||
5. **Large-scale**: Metadata/vector separation + UUID sharding + lifecycle policies
|
5. **Large-scale**: Metadata/vector separation + UUID sharding + lifecycle policies
|
||||||
|
|
||||||
### v4.0.0 Cost Optimization
|
### Cost Optimization
|
||||||
1. **Enable lifecycle policies** for cloud storage (automated cost reduction)
|
1. **Enable lifecycle policies** for cloud storage (automated cost reduction)
|
||||||
2. **Use Intelligent-Tiering (S3)** or Autoclass (GCS) for automatic optimization
|
2. **Use Intelligent-Tiering (S3)** or Autoclass (GCS) for automatic optimization
|
||||||
3. **Enable compression** for FileSystem storage (60-80% space savings)
|
3. **Enable compression** for FileSystem storage (60-80% space savings)
|
||||||
|
|
@ -547,7 +547,7 @@ console.log(stats)
|
||||||
|
|
||||||
**Example Cost Savings (500TB dataset):**
|
**Example Cost Savings (500TB dataset):**
|
||||||
- Without lifecycle policies: **$138,000/year**
|
- Without lifecycle policies: **$138,000/year**
|
||||||
- With v4.0.0 lifecycle policies: **$5,940/year**
|
- With lifecycle policies: **$5,940/year**
|
||||||
- **Savings: $132,060/year (96%)**
|
- **Savings: $132,060/year (96%)**
|
||||||
|
|
||||||
### Monitor and Maintain
|
### Monitor and Maintain
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
# 🔌 Brainy v4.0.0 Augmentations Complete Reference
|
# 🔌 Brainy Augmentations Complete Reference
|
||||||
|
|
||||||
> **All augmentations that power Brainy's extensibility - with locations, usage, and examples**
|
> **All augmentations that power Brainy's extensibility - with locations, usage, and examples**
|
||||||
>
|
>
|
||||||
> **⚠️ v4.0.0 Update**: Updated for metadata structure changes and billion-scale optimizations
|
> **⚠️ Update**: Updated for metadata structure changes and billion-scale optimizations
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
|
|
@ -19,7 +19,7 @@ const brain = new Brainy({
|
||||||
await brain.init() // Augmentations initialize automatically
|
await brain.init() // Augmentations initialize automatically
|
||||||
```
|
```
|
||||||
|
|
||||||
## v4.0.0 Augmentation Architecture
|
## Augmentation Architecture
|
||||||
|
|
||||||
### Key Improvements for Billion-Scale Performance
|
### Key Improvements for Billion-Scale Performance
|
||||||
|
|
||||||
|
|
@ -40,7 +40,7 @@ await brain.init() // Augmentations initialize automatically
|
||||||
|
|
||||||
### What This Means for Augmentation Users
|
### What This Means for Augmentation Users
|
||||||
|
|
||||||
**✅ If you use built-in augmentations**: No changes needed! They're all updated for v4.0.0.
|
**✅ If you use built-in augmentations**: No changes needed! They're all updated.
|
||||||
|
|
||||||
**⚠️ If you created custom storage augmentations**: Update your storage adapter to:
|
**⚠️ If you created custom storage augmentations**: Update your storage adapter to:
|
||||||
- Wrap metadata with required `noun`/`verb` fields
|
- Wrap metadata with required `noun`/`verb` fields
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
# 🛠️ Brainy Augmentation Developer Guide
|
# 🛠️ Brainy Augmentation Developer Guide
|
||||||
|
|
||||||
> **How to create, test, and use augmentations in Brainy v4.0.0**
|
> **How to create, test, and use augmentations in Brainy**
|
||||||
>
|
>
|
||||||
> **⚠️ v4.0.0 Update**: This guide has been updated with breaking changes for metadata structure and type system improvements.
|
> **⚠️ Update**: This guide has been updated with breaking changes for metadata structure and type system improvements.
|
||||||
|
|
||||||
## v4.0.0 Migration Guide
|
## Migration Guide
|
||||||
|
|
||||||
### What Changed?
|
### What Changed?
|
||||||
|
|
||||||
|
|
@ -31,7 +31,7 @@ const verb = {
|
||||||
}
|
}
|
||||||
if (verb.type === 'relatedTo') { ... }
|
if (verb.type === 'relatedTo') { ... }
|
||||||
|
|
||||||
// ✅ v4.0.0
|
// ✅ Current
|
||||||
const verb = {
|
const verb = {
|
||||||
verb: 'relatedTo',
|
verb: 'relatedTo',
|
||||||
sourceId: 'a',
|
sourceId: 'a',
|
||||||
|
|
@ -70,7 +70,7 @@ export class MyFirstAugmentation extends BaseAugmentation {
|
||||||
if (operation === 'add') {
|
if (operation === 'add') {
|
||||||
console.log('Noun added:', params.noun)
|
console.log('Noun added:', params.noun)
|
||||||
|
|
||||||
// v4.0.0: Access metadata correctly
|
// Access metadata correctly
|
||||||
if (params.noun?.metadata) {
|
if (params.noun?.metadata) {
|
||||||
console.log('Noun type:', params.noun.metadata.noun) // Required field
|
console.log('Noun type:', params.noun.metadata.noun) // Required field
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
# Brainy v4.0.0 Cloud Deployment Guide
|
# Brainy Cloud Deployment Guide
|
||||||
|
|
||||||
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy v4.0.0 codebase.
|
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy codebase.
|
||||||
|
|
||||||
## 🆕 v4.0.0 Production Features
|
## 🆕 Production Features
|
||||||
|
|
||||||
**Cost Optimization at Scale:**
|
**Cost Optimization at Scale:**
|
||||||
- **Lifecycle Policies**: Automatic tier transitions for 96% cost savings
|
- **Lifecycle Policies**: Automatic tier transitions for 96% cost savings
|
||||||
|
|
@ -12,7 +12,7 @@ This guide provides production-ready deployment configurations for Brainy using
|
||||||
|
|
||||||
**Example Impact (500TB dataset):**
|
**Example Impact (500TB dataset):**
|
||||||
- Before: $138,000/year
|
- Before: $138,000/year
|
||||||
- With v4.0.0 lifecycle policies: $5,940/year
|
- With lifecycle policies: $5,940/year
|
||||||
- **Savings: $132,060/year (96%)**
|
- **Savings: $132,060/year (96%)**
|
||||||
|
|
||||||
See the [Cost Optimization](#cost-optimization-v40) section below for implementation details.
|
See the [Cost Optimization](#cost-optimization-v40) section below for implementation details.
|
||||||
|
|
@ -727,7 +727,7 @@ S3CompatibleStorage constructor parameters (verified from source):
|
||||||
4. **Implement rate limiting** to prevent abuse
|
4. **Implement rate limiting** to prevent abuse
|
||||||
5. **Configure CORS** appropriately for your use case
|
5. **Configure CORS** appropriately for your use case
|
||||||
|
|
||||||
## Cost Optimization (v4.0.0)
|
## Cost Optimization
|
||||||
|
|
||||||
### Enable Lifecycle Policies
|
### Enable Lifecycle Policies
|
||||||
|
|
||||||
|
|
@ -765,7 +765,7 @@ await storage.enableIntelligentTiering('entities/', 'auto-optimize')
|
||||||
|
|
||||||
**Efficient bulk deletions:**
|
**Efficient bulk deletions:**
|
||||||
```javascript
|
```javascript
|
||||||
// v4.0.0: Batch delete (1000 objects per request)
|
// Batch delete (1000 objects per request)
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
const idsToDelete = [/* array of entity IDs */]
|
||||||
const paths = idsToDelete.flatMap(id => [
|
const paths = idsToDelete.flatMap(id => [
|
||||||
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
||||||
|
|
@ -793,8 +793,8 @@ const brain = new Brainy({ storage })
|
||||||
2. **Use S3CompatibleStorage** for cloud deployments (better scalability)
|
2. **Use S3CompatibleStorage** for cloud deployments (better scalability)
|
||||||
3. **Enable the cache augmentation** for frequently accessed data
|
3. **Enable the cache augmentation** for frequently accessed data
|
||||||
4. **Configure appropriate memory limits** for your runtime
|
4. **Configure appropriate memory limits** for your runtime
|
||||||
5. **v4.0.0**: Enable lifecycle policies to reduce storage costs by 96%
|
5. Enable lifecycle policies to reduce storage costs by 96%
|
||||||
6. **v4.0.0**: Use batch operations for cleanup tasks
|
6. Use batch operations for cleanup tasks
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -387,7 +387,7 @@ jobs:
|
||||||
|
|
||||||
3. **Cold Starts (Lambda)**
|
3. **Cold Starts (Lambda)**
|
||||||
|
|
||||||
**v7.3.0+ Progressive Initialization (Zero-Config)**
|
**Progressive Initialization (Zero-Config)**
|
||||||
|
|
||||||
Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME)
|
Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME)
|
||||||
and uses progressive initialization for <200ms cold starts:
|
and uses progressive initialization for <200ms cold starts:
|
||||||
|
|
@ -441,7 +441,7 @@ jobs:
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Readiness Detection (v7.3.0+)**
|
**Readiness Detection**
|
||||||
|
|
||||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -507,7 +507,7 @@ const brain = new Brainy({
|
||||||
|
|
||||||
2. **Cold Starts**
|
2. **Cold Starts**
|
||||||
|
|
||||||
**v7.3.0+ Progressive Initialization (Zero-Config)**
|
**Progressive Initialization (Zero-Config)**
|
||||||
|
|
||||||
Brainy automatically detects Cloud Run and Cloud Functions environments
|
Brainy automatically detects Cloud Run and Cloud Functions environments
|
||||||
and uses progressive initialization for <200ms cold starts:
|
and uses progressive initialization for <200ms cold starts:
|
||||||
|
|
@ -557,7 +557,7 @@ const brain = new Brainy({
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
**Readiness Detection (v7.3.0+)**
|
**Readiness Detection**
|
||||||
|
|
||||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ if (success) {
|
||||||
|
|
||||||
Brainy uses **Snowflake-style Copy-on-Write (COW)** for instant forking:
|
Brainy uses **Snowflake-style Copy-on-Write (COW)** for instant forking:
|
||||||
|
|
||||||
### Architecture (v5.0.0)
|
### Architecture
|
||||||
|
|
||||||
1. **HNSW Index COW** (The Performance Bottleneck):
|
1. **HNSW Index COW** (The Performance Bottleneck):
|
||||||
- **Shallow Copy**: O(1) Map reference copying (~10ms for 1M+ nodes)
|
- **Shallow Copy**: O(1) Map reference copying (~10ms for 1M+ nodes)
|
||||||
|
|
@ -79,7 +79,7 @@ Brainy uses **Snowflake-style Copy-on-Write (COW)** for instant forking:
|
||||||
- **BlobStorage**: Content-addressable with deduplication
|
- **BlobStorage**: Content-addressable with deduplication
|
||||||
- **All Adapters**: Memory, FileSystem, S3, R2, GCS, Azure, OPFS
|
- **All Adapters**: Memory, FileSystem, S3, R2, GCS, Azure, OPFS
|
||||||
|
|
||||||
**Performance (v5.0.0)**:
|
**Performance**:
|
||||||
- **Fork time**: < 100ms @ 10K entities (MEASURED in tests)
|
- **Fork time**: < 100ms @ 10K entities (MEASURED in tests)
|
||||||
- **Storage overhead**: 10-20% (shared blobs, only changed data duplicated)
|
- **Storage overhead**: 10-20% (shared blobs, only changed data duplicated)
|
||||||
- **Memory overhead**: 10-20% (shared HNSW nodes, only modified nodes duplicated)
|
- **Memory overhead**: 10-20% (shared HNSW nodes, only modified nodes duplicated)
|
||||||
|
|
@ -316,7 +316,7 @@ console.log(originalData[0].data.version) // 1 (original state!)
|
||||||
// Clean up
|
// Clean up
|
||||||
await snapshot.destroy()
|
await snapshot.destroy()
|
||||||
|
|
||||||
// Note: Time-travel queries (asOf) planned for v5.1.0
|
// Note: Time-travel queries (asOf) Planned
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
@ -339,9 +339,9 @@ const fork3 = await brain.fork('test', {
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Branch Management (v5.0.0)
|
### Branch Management
|
||||||
|
|
||||||
**NEW in v5.0.0:** Full branch management now available!
|
Full branch management now available!
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// List all branches
|
// List all branches
|
||||||
|
|
@ -359,9 +359,9 @@ await brain.checkout('experiment')
|
||||||
await brain.deleteBranch('old-experiment')
|
await brain.deleteBranch('old-experiment')
|
||||||
```
|
```
|
||||||
|
|
||||||
### Commit Tracking (v5.0.0)
|
### Commit Tracking
|
||||||
|
|
||||||
**NEW in v5.0.0:** Git-style commit tracking!
|
Git-style commit tracking!
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Create a commit (snapshot of current state)
|
// Create a commit (snapshot of current state)
|
||||||
|
|
@ -375,9 +375,9 @@ const commitHash = await brain.commit({
|
||||||
console.log(commitHash) // 'a3f2c1b9...'
|
console.log(commitHash) // 'a3f2c1b9...'
|
||||||
```
|
```
|
||||||
|
|
||||||
### Commit History (v5.0.0)
|
### Commit History
|
||||||
|
|
||||||
**NEW in v5.0.0:** View commit history!
|
View commit history!
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// Get commit history for current branch
|
// Get commit history for current branch
|
||||||
|
|
@ -425,7 +425,7 @@ Savings: 40% less memory
|
||||||
|
|
||||||
## Zero Configuration
|
## Zero Configuration
|
||||||
|
|
||||||
**Fork is enabled by default in v5.0.0+. No setup required.**
|
**Fork is enabled by default. No setup required.**
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
// This is all you need:
|
// This is all you need:
|
||||||
|
|
@ -612,8 +612,7 @@ const fork = await brain.fork() // Done!
|
||||||
|
|
||||||
## What's Implemented vs. What's Next
|
## What's Implemented vs. What's Next
|
||||||
|
|
||||||
### ✅ Available in v5.0.0:
|
### ✅ Available in - ✅ `fork()` - Instant clone in <100ms
|
||||||
- ✅ `fork()` - Instant clone in <100ms
|
|
||||||
- ✅ `listBranches()` - List all forks
|
- ✅ `listBranches()` - List all forks
|
||||||
- ✅ `getCurrentBranch()` - Get active branch
|
- ✅ `getCurrentBranch()` - Get active branch
|
||||||
- ✅ `checkout()` - Switch between branches
|
- ✅ `checkout()` - Switch between branches
|
||||||
|
|
@ -621,14 +620,14 @@ const fork = await brain.fork() // Done!
|
||||||
- ✅ `commit()` - Create state snapshots
|
- ✅ `commit()` - Create state snapshots
|
||||||
- ✅ `getHistory()` - View commit history
|
- ✅ `getHistory()` - View commit history
|
||||||
|
|
||||||
### 🔮 Planned for v5.1.0+:
|
### 🔮 Planned for:
|
||||||
|
|
||||||
**Temporal Features:**
|
**Temporal Features:**
|
||||||
- `asOf(timestamp)` - Query data at specific time (✅ v5.0.0+)
|
- `asOf(timestamp)` - Query data at specific time (✅ available)
|
||||||
- `rollback(commitHash)` - Restore to previous state
|
- `rollback(commitHash)` - Restore to previous state
|
||||||
- Full audit trail for all changes
|
- Full audit trail for all changes
|
||||||
|
|
||||||
These features require additional temporal infrastructure and are being carefully designed for v5.1.0+
|
These features require additional temporal infrastructure and are being carefully designed
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -691,4 +690,4 @@ console.log('Fork created in < 2 seconds! 🚀')
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Brainy v5.0.0+** | [GitHub](https://github.com/soulcraftlabs/brainy) | [npm](https://npmjs.com/package/@soulcraft/brainy)
|
**Brainy** | [GitHub](https://github.com/soulcraftlabs/brainy) | [npm](https://npmjs.com/package/@soulcraft/brainy)
|
||||||
|
|
|
||||||
|
|
@ -92,7 +92,7 @@ await brain.import(yaml, { format: 'yaml' })
|
||||||
// ✨ Hierarchical data becomes a connected graph!
|
// ✨ Hierarchical data becomes a connected graph!
|
||||||
```
|
```
|
||||||
|
|
||||||
### 📄 Import Word Documents (DOCX) - v4.2.0
|
### 📄 Import Word Documents (DOCX) -
|
||||||
```javascript
|
```javascript
|
||||||
// From file path
|
// From file path
|
||||||
await brain.import('research-paper.docx')
|
await brain.import('research-paper.docx')
|
||||||
|
|
@ -121,7 +121,7 @@ await brain.import('https://api.example.com/data.json')
|
||||||
await brain.import('https://data.gov/census.csv')
|
await brain.import('https://data.gov/census.csv')
|
||||||
// ✨ Fetches CSV from web, parses, imports!
|
// ✨ Fetches CSV from web, parses, imports!
|
||||||
|
|
||||||
// With authentication (v4.2.0)
|
// With authentication
|
||||||
await brain.import({
|
await brain.import({
|
||||||
type: 'url',
|
type: 'url',
|
||||||
data: 'https://api.example.com/private/data.xlsx',
|
data: 'https://api.example.com/private/data.xlsx',
|
||||||
|
|
@ -132,7 +132,7 @@ await brain.import({
|
||||||
})
|
})
|
||||||
// ✨ Supports basic authentication for protected resources!
|
// ✨ Supports basic authentication for protected resources!
|
||||||
|
|
||||||
// With custom headers (v4.2.0)
|
// With custom headers
|
||||||
await brain.import({
|
await brain.import({
|
||||||
type: 'url',
|
type: 'url',
|
||||||
data: 'https://api.example.com/data.json',
|
data: 'https://api.example.com/data.json',
|
||||||
|
|
@ -163,9 +163,9 @@ When you import data, Brainy:
|
||||||
2. **Intelligent parsing** - CSV (encoding/delimiter), Excel (multi-sheet), PDF (text/tables), DOCX (headings/paragraphs)
|
2. **Intelligent parsing** - CSV (encoding/delimiter), Excel (multi-sheet), PDF (text/tables), DOCX (headings/paragraphs)
|
||||||
3. **Identifies entity types** - Uses AI to classify as Person, Document, Product, etc. (31 types!)
|
3. **Identifies entity types** - Uses AI to classify as Person, Document, Product, etc. (31 types!)
|
||||||
4. **Finds relationships** - Detects connections like "belongsTo", "createdBy", "references" (40 types!)
|
4. **Finds relationships** - Detects connections like "belongsTo", "createdBy", "references" (40 types!)
|
||||||
5. **Scores confidence & weight** - Every entity and relationship gets quality metrics (v4.2.0)
|
5. **Scores confidence & weight** - Every entity and relationship gets quality metrics
|
||||||
6. **Creates embeddings** - Makes everything semantically searchable
|
6. **Creates embeddings** - Makes everything semantically searchable
|
||||||
7. **Indexes metadata** - Enables lightning-fast filtering with range queries (v4.2.0)
|
7. **Indexes metadata** - Enables lightning-fast filtering with range queries
|
||||||
|
|
||||||
## Intelligent Type Detection
|
## Intelligent Type Detection
|
||||||
|
|
||||||
|
|
@ -204,8 +204,7 @@ await brain.import(data)
|
||||||
// - Bob "memberOf" Engineering
|
// - Bob "memberOf" Engineering
|
||||||
```
|
```
|
||||||
|
|
||||||
## Confidence & Weight Scoring - v4.2.0
|
## Confidence & Weight Scoring -
|
||||||
|
|
||||||
Every entity and relationship gets confidence and weight scores:
|
Every entity and relationship gets confidence and weight scores:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
|
|
@ -236,8 +235,7 @@ const mediumConfidence = await brain.find({
|
||||||
|
|
||||||
**Weights** indicate importance/relevance within the document context.
|
**Weights** indicate importance/relevance within the document context.
|
||||||
|
|
||||||
## Per-Sheet Excel Extraction - v4.2.0
|
## Per-Sheet Excel Extraction -
|
||||||
|
|
||||||
Excel files with multiple sheets can be organized by sheet:
|
Excel files with multiple sheets can be organized by sheet:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ Each phase adds intelligence and structure to your raw data, transforming it int
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 🌊 Always-On Streaming Architecture (v4.2.0+)
|
## 🌊 Always-On Streaming Architecture
|
||||||
|
|
||||||
All imports use streaming with **progressive flush intervals**:
|
All imports use streaming with **progressive flush intervals**:
|
||||||
|
|
||||||
|
|
@ -1062,7 +1062,7 @@ if (!fromEntity || !toEntity) {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 5.3b: Check for Duplicates (v3.43.2 Critical Fix)
|
#### 5.3b: Check for Duplicates (Critical Fix)
|
||||||
|
|
||||||
**The Bug**: Without duplicate checking, re-importing would create:
|
**The Bug**: Without duplicate checking, re-importing would create:
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
**How to Use Progress Tracking in Your Applications**
|
**How to Use Progress Tracking in Your Applications**
|
||||||
|
|
||||||
Brainy v4.5.0+ provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX).
|
Brainy provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX).
|
||||||
|
|
||||||
> **⚠️ KEY FEATURE:** The progress API is **100% standardized**. Write your progress handler ONCE and it works for ALL formats with zero format-specific code! See [Standard Import Progress API](./standard-import-progress.md) for the complete interface documentation.
|
> **⚠️ KEY FEATURE:** The progress API is **100% standardized**. Write your progress handler ONCE and it works for ALL formats with zero format-specific code! See [Standard Import Progress API](./standard-import-progress.md) for the complete interface documentation.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -74,7 +74,7 @@ interface ImporterProgressCallback {
|
||||||
|
|
||||||
## 🎯 Overview
|
## 🎯 Overview
|
||||||
|
|
||||||
As of v4.5.0, Brainy supports comprehensive, multi-dimensional progress tracking for imports:
|
Brainy supports comprehensive, multi-dimensional progress tracking for imports:
|
||||||
- **Bytes processed** (always available, most deterministic)
|
- **Bytes processed** (always available, most deterministic)
|
||||||
- **Entities extracted** (AI extraction phase)
|
- **Entities extracted** (AI extraction phase)
|
||||||
- **Stage-specific metrics** (parsing: MB/s, extraction: entities/s)
|
- **Stage-specific metrics** (parsing: MB/s, extraction: entities/s)
|
||||||
|
|
@ -120,13 +120,13 @@ export interface FormatHandlerOptions {
|
||||||
// ... existing options ...
|
// ... existing options ...
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Progress hooks (v4.5.0)
|
* Progress hooks
|
||||||
* Handlers call these to report progress during processing
|
* Handlers call these to report progress during processing
|
||||||
*/
|
*/
|
||||||
progressHooks?: FormatHandlerProgressHooks
|
progressHooks?: FormatHandlerProgressHooks
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Total file size in bytes (v4.5.0)
|
* Total file size in bytes
|
||||||
* Used for progress percentage calculation
|
* Used for progress percentage calculation
|
||||||
*/
|
*/
|
||||||
totalBytes?: number
|
totalBytes?: number
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ await brain.import(file, {
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Import Tracking (v4.10.0+)
|
### Import Tracking
|
||||||
|
|
||||||
Track and organize imports by project:
|
Track and organize imports by project:
|
||||||
|
|
||||||
|
|
@ -149,7 +149,7 @@ await brain.import(file, {
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
### Always-On Streaming (v4.2.0+)
|
### Always-On Streaming
|
||||||
|
|
||||||
All imports use streaming with adaptive flush intervals. Query data as it's imported:
|
All imports use streaming with adaptive flush intervals. Query data as it's imported:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@ interface ImportProgress {
|
||||||
/** Estimated time remaining (milliseconds) */
|
/** Estimated time remaining (milliseconds) */
|
||||||
eta?: number
|
eta?: number
|
||||||
|
|
||||||
/** Whether data is queryable at this point (v4.2.0+) */
|
/** Whether data is queryable at this point */
|
||||||
queryable?: boolean
|
queryable?: boolean
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -300,7 +300,7 @@ interface ImportProgress {
|
||||||
relationships?: number // Relationships inferred so far
|
relationships?: number // Relationships inferred so far
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether data is queryable (v4.2.0+)
|
* Whether data is queryable
|
||||||
*
|
*
|
||||||
* true = Indexes flushed, queries will be fast and complete
|
* true = Indexes flushed, queries will be fast and complete
|
||||||
* false/undefined = Data in storage but indexes not flushed yet
|
* false/undefined = Data in storage but indexes not flushed yet
|
||||||
|
|
@ -376,7 +376,7 @@ No changes required! Streaming is now always enabled with optimal defaults:
|
||||||
// Before (v3.x, v4.0, v4.1): Works the same
|
// Before (v3.x, v4.0, v4.1): Works the same
|
||||||
await brain.import(file)
|
await brain.import(file)
|
||||||
|
|
||||||
// After (v4.2.0+): Streaming always on, zero config
|
// After: Streaming always on, zero config
|
||||||
await brain.import(file)
|
await brain.import(file)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Capacity Planning & Operations Guide
|
# Capacity Planning & Operations Guide
|
||||||
|
|
||||||
**Brainy v3.36.0+ Enterprise Operations**
|
**Brainy Enterprise Operations**
|
||||||
|
|
||||||
This guide provides production-ready capacity planning formulas, deployment strategies, and operational guidelines for scaling Brainy from development (2GB) to enterprise (128GB+) deployments.
|
This guide provides production-ready capacity planning formulas, deployment strategies, and operational guidelines for scaling Brainy from development (2GB) to enterprise (128GB+) deployments.
|
||||||
|
|
||||||
|
|
@ -704,8 +704,7 @@ if (stats.fairness.fairnessViolation) {
|
||||||
|
|
||||||
## 📚 Additional Resources
|
## 📚 Additional Resources
|
||||||
|
|
||||||
- **[Migration Guide](../guides/migration-3.36.0.md)** - Upgrading to v3.36.0
|
- **[Migration Guide](../guides/migration-3.36.0.md)** - Upgrading to **[Architecture Overview](../architecture/data-storage-architecture.md)** - Deep dive into storage and caching
|
||||||
- **[Architecture Overview](../architecture/data-storage-architecture.md)** - Deep dive into storage and caching
|
|
||||||
- **[GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)** - Report problems or ask questions
|
- **[GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)** - Report problems or ask questions
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
# AWS S3 Cost Optimization Guide for Brainy v4.0.0
|
# AWS S3 Cost Optimization Guide for Brainy
|
||||||
|
|
||||||
> **Cost Impact**: Reduce storage costs from $138k/year to $5.9k/year at 500TB scale (**96% savings**)
|
> **Cost Impact**: Reduce storage costs from $138k/year to $5.9k/year at 500TB scale (**96% savings**)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Brainy v4.0.0 provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance.
|
Brainy provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance.
|
||||||
|
|
||||||
## Cost Breakdown (Before Optimization)
|
## Cost Breakdown (Before Optimization)
|
||||||
|
|
||||||
|
|
@ -250,7 +249,7 @@ Note: Retrieval costs may be significant if archived data is accessed frequently
|
||||||
### Efficient Cleanup
|
### Efficient Cleanup
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// v4.0.0: Batch delete (1000 objects per request)
|
// Batch delete (1000 objects per request)
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
const idsToDelete = [/* array of entity IDs */]
|
||||||
|
|
||||||
// Generate paths for both vector and metadata files
|
// Generate paths for both vector and metadata files
|
||||||
|
|
@ -396,6 +395,5 @@ await storage.enableIntelligentTiering('entities/', 'new-config')
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Version**: v4.0.0
|
|
||||||
**Last Updated**: 2025-10-17
|
**Last Updated**: 2025-10-17
|
||||||
**Cloud Provider**: AWS S3
|
**Cloud Provider**: AWS S3
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
# Azure Blob Storage Cost Optimization Guide for Brainy v4.0.0
|
# Azure Blob Storage Cost Optimization Guide for Brainy
|
||||||
|
|
||||||
> **Cost Impact**: Reduce storage costs from $107k/year to $5k/year at 500TB scale (**95% savings**)
|
> **Cost Impact**: Reduce storage costs from $107k/year to $5k/year at 500TB scale (**95% savings**)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Brainy v4.0.0 provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations.
|
Brainy provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations.
|
||||||
|
|
||||||
## Cost Breakdown (Before Optimization)
|
## Cost Breakdown (Before Optimization)
|
||||||
|
|
||||||
|
|
@ -333,7 +332,7 @@ Examples:
|
||||||
### Efficient Bulk Deletions
|
### Efficient Bulk Deletions
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// v4.0.0: Batch delete (256 blobs per request)
|
// Batch delete (256 blobs per request)
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
const idsToDelete = [/* array of entity IDs */]
|
||||||
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
const paths = idsToDelete.flatMap(id => {
|
||||||
|
|
@ -549,6 +548,5 @@ const storage = new AzureBlobStorage({
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Version**: v4.0.0
|
|
||||||
**Last Updated**: 2025-10-17
|
**Last Updated**: 2025-10-17
|
||||||
**Cloud Provider**: Azure Blob Storage
|
**Cloud Provider**: Azure Blob Storage
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
# Cloudflare R2 Cost Optimization Guide for Brainy v4.0.0
|
# Cloudflare R2 Cost Optimization Guide for Brainy
|
||||||
|
|
||||||
> **Cost Impact**: $0 egress fees + 96% storage savings = Lowest cloud storage costs
|
> **Cost Impact**: $0 egress fees + 96% storage savings = Lowest cloud storage costs
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy v4.0.0 fully supports R2 with lifecycle policies and batch operations.
|
Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy fully supports R2 with lifecycle policies and batch operations.
|
||||||
|
|
||||||
## Cost Breakdown
|
## Cost Breakdown
|
||||||
|
|
||||||
|
|
@ -49,7 +48,7 @@ Savings vs AWS: $156,000/year (62%)
|
||||||
- Cloudflare plans to add infrequent access tiers
|
- Cloudflare plans to add infrequent access tiers
|
||||||
|
|
||||||
**When lifecycle features arrive:**
|
**When lifecycle features arrive:**
|
||||||
- Brainy v4.0.0 is already prepared with `setLifecyclePolicy()` support
|
- Brainy is already prepared with `setLifecyclePolicy()` support
|
||||||
- Will work seamlessly once Cloudflare enables lifecycle management
|
- Will work seamlessly once Cloudflare enables lifecycle management
|
||||||
|
|
||||||
## Strategy 1: Use R2 Standard (Current Best Practice)
|
## Strategy 1: Use R2 Standard (Current Best Practice)
|
||||||
|
|
@ -187,7 +186,7 @@ Savings: $193,624/year (77%)
|
||||||
### Efficient Bulk Deletions
|
### Efficient Bulk Deletions
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// v4.0.0: R2 supports S3 batch delete API
|
// R2 supports S3 batch delete API
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
const idsToDelete = [/* array of entity IDs */]
|
||||||
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
const paths = idsToDelete.flatMap(id => {
|
||||||
|
|
@ -361,7 +360,7 @@ ROI: 3.4 months
|
||||||
### Prepared for Future Features
|
### Prepared for Future Features
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Brainy v4.0.0 is ready for R2 lifecycle features
|
// Brainy is ready for R2 lifecycle features
|
||||||
await storage.setLifecyclePolicy({
|
await storage.setLifecyclePolicy({
|
||||||
rules: [{
|
rules: [{
|
||||||
id: 'archive-old-data',
|
id: 'archive-old-data',
|
||||||
|
|
@ -446,7 +445,6 @@ const storage = new S3CompatibleStorage({
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Version**: v4.0.0
|
|
||||||
**Last Updated**: 2025-10-17
|
**Last Updated**: 2025-10-17
|
||||||
**Cloud Provider**: Cloudflare R2
|
**Cloud Provider**: Cloudflare R2
|
||||||
**Key Advantage**: **$0 egress fees forever**
|
**Key Advantage**: **$0 egress fees forever**
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
# Google Cloud Storage Cost Optimization Guide for Brainy v4.0.0
|
# Google Cloud Storage Cost Optimization Guide for Brainy
|
||||||
|
|
||||||
> **Cost Impact**: Reduce storage costs from $138k/year to $8.3k/year at 500TB scale (**94% savings**)
|
> **Cost Impact**: Reduce storage costs from $138k/year to $8.3k/year at 500TB scale (**94% savings**)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Brainy v4.0.0 provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management.
|
Brainy provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management.
|
||||||
|
|
||||||
## Cost Breakdown (Before Optimization)
|
## Cost Breakdown (Before Optimization)
|
||||||
|
|
||||||
|
|
@ -241,7 +240,7 @@ Warning: High retrieval costs if archived data is accessed frequently
|
||||||
### Efficient Cleanup
|
### Efficient Cleanup
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// v4.0.0: Batch delete (100 objects per request for GCS)
|
// Batch delete (100 objects per request for GCS)
|
||||||
const idsToDelete = [/* array of entity IDs */]
|
const idsToDelete = [/* array of entity IDs */]
|
||||||
|
|
||||||
const paths = idsToDelete.flatMap(id => {
|
const paths = idsToDelete.flatMap(id => {
|
||||||
|
|
@ -417,6 +416,5 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Version**: v4.0.0
|
|
||||||
**Last Updated**: 2025-10-17
|
**Last Updated**: 2025-10-17
|
||||||
**Cloud Provider**: Google Cloud Storage
|
**Cloud Provider**: Google Cloud Storage
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Transaction System
|
# Transaction System
|
||||||
|
|
||||||
**Status:** ✅ Production Ready (v5.8.0+)
|
**Status:** ✅ Production Ready
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
|
|
@ -123,7 +123,7 @@ await brain.relate({ from: id1, to: id2, type: VerbType.RelatesTo })
|
||||||
- Transaction operations don't need to know about shards
|
- Transaction operations don't need to know about shards
|
||||||
- Rollback works across all shards involved
|
- Rollback works across all shards involved
|
||||||
|
|
||||||
### ID-First Storage (v6.0.0+)
|
### ID-First Storage
|
||||||
|
|
||||||
✅ **Fully Compatible**
|
✅ **Fully Compatible**
|
||||||
|
|
||||||
|
|
@ -152,7 +152,7 @@ await brain.update({
|
||||||
**How It Works:**
|
**How It Works:**
|
||||||
- Type information stored in metadata.noun field
|
- Type information stored in metadata.noun field
|
||||||
- Storage layer uses O(1) ID-first path construction
|
- Storage layer uses O(1) ID-first path construction
|
||||||
- No type cache needed (removed in v6.0.0)
|
- No type cache needed (removed in a previous version)
|
||||||
- Type counters adjusted on commit/rollback
|
- Type counters adjusted on commit/rollback
|
||||||
- 40x faster on cloud storage (eliminates 42-type search)
|
- 40x faster on cloud storage (eliminates 42-type search)
|
||||||
|
|
||||||
|
|
@ -563,7 +563,7 @@ interface StorageAdapter {
|
||||||
|
|
||||||
## Version History
|
## Version History
|
||||||
|
|
||||||
- **v5.8.0**: Initial transaction system release
|
- Initial transaction system release
|
||||||
- Atomic operations with rollback
|
- Atomic operations with rollback
|
||||||
- Compatible with COW, sharding, type-aware, distributed
|
- Compatible with COW, sharding, type-aware, distributed
|
||||||
- 36/36 unit tests passing
|
- 36/36 unit tests passing
|
||||||
|
|
|
||||||
|
|
@ -294,7 +294,7 @@ npm install @soulcraft/brainy@latest # Update if needed
|
||||||
|
|
||||||
### "VFS not initialized" errors
|
### "VFS not initialized" errors
|
||||||
```typescript
|
```typescript
|
||||||
// v5.1.0+: Just await brain.init() - VFS is auto-initialized!
|
// Just await brain.init() - VFS is auto-initialized!
|
||||||
await brain.init()
|
await brain.init()
|
||||||
// VFS ready - use brain.vfs directly
|
// VFS ready - use brain.vfs directly
|
||||||
await brain.vfs.writeFile('/test.txt', 'data')
|
await brain.vfs.writeFile('/test.txt', 'data')
|
||||||
|
|
|
||||||
|
|
@ -48,11 +48,11 @@ const results = await vfs.search('files about authentication')
|
||||||
await vfs.addRelationship('/docs/spec.md', '/projects/my-app/', 'implements')
|
await vfs.addRelationship('/docs/spec.md', '/projects/my-app/', 'implements')
|
||||||
```
|
```
|
||||||
|
|
||||||
## ⚡ Performance (v5.11.1)
|
## ⚡ Performance
|
||||||
|
|
||||||
**75% Faster File Operations!** VFS now automatically benefits from brain.get() metadata-only optimization:
|
**75% Faster File Operations!** VFS now automatically benefits from brain.get() metadata-only optimization:
|
||||||
|
|
||||||
| Operation | Before (v5.11.0) | After (v5.11.1) | Speedup |
|
| Operation | Before | After | Speedup |
|
||||||
|-----------|------------------|-----------------|---------|
|
|-----------|------------------|-----------------|---------|
|
||||||
| `readFile()` | 53ms | **~13ms** | **75%** |
|
| `readFile()` | 53ms | **~13ms** | **75%** |
|
||||||
| `stat()` | 53ms | **~13ms** | **75%** |
|
| `stat()` | 53ms | **~13ms** | **75%** |
|
||||||
|
|
@ -60,7 +60,7 @@ await vfs.addRelationship('/docs/spec.md', '/projects/my-app/', 'implements')
|
||||||
|
|
||||||
**Zero configuration** - automatic optimization for all VFS operations!
|
**Zero configuration** - automatic optimization for all VFS operations!
|
||||||
|
|
||||||
VFS operations only need metadata (path, size, timestamps), not 384-dimensional vector embeddings. The v5.11.1 optimization automatically uses metadata-only reads, saving 95% bandwidth and 76-81% time.
|
VFS operations only need metadata (path, size, timestamps), not 384-dimensional vector embeddings. The optimization automatically uses metadata-only reads, saving 95% bandwidth and 76-81% time.
|
||||||
|
|
||||||
## Core Features
|
## Core Features
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,8 @@
|
||||||
**Root Cause:**
|
**Root Cause:**
|
||||||
The root directory entity exists but doesn't have the proper metadata structure.
|
The root directory entity exists but doesn't have the proper metadata structure.
|
||||||
|
|
||||||
**Solution (v3.15.0+):**
|
**Solution:**
|
||||||
This issue has been fixed in v3.15.0. The VFS now:
|
This issue has been fixed. The VFS now:
|
||||||
1. Ensures root directory has `vfsType: 'directory'` metadata
|
1. Ensures root directory has `vfsType: 'directory'` metadata
|
||||||
2. Adds compatibility layer for entities with malformed metadata
|
2. Adds compatibility layer for entities with malformed metadata
|
||||||
3. Automatically repairs metadata on entity retrieval
|
3. Automatically repairs metadata on entity retrieval
|
||||||
|
|
@ -47,7 +47,7 @@ await brain.update({
|
||||||
**Root Cause:**
|
**Root Cause:**
|
||||||
Contains relationships are missing between parent directories and files.
|
Contains relationships are missing between parent directories and files.
|
||||||
|
|
||||||
**Solution (v3.15.0+):**
|
**Solution:**
|
||||||
This issue has been fixed. The VFS now:
|
This issue has been fixed. The VFS now:
|
||||||
1. Creates Contains relationships when writing new files
|
1. Creates Contains relationships when writing new files
|
||||||
2. Ensures Contains relationships exist when updating files
|
2. Ensures Contains relationships exist when updating files
|
||||||
|
|
|
||||||
|
|
@ -418,7 +418,7 @@ await vfs.importDirectory()// Import directory from filesystem
|
||||||
|
|
||||||
## Bulk Write Operations
|
## Bulk Write Operations
|
||||||
|
|
||||||
**v6.5.0**: `bulkWrite` efficiently processes multiple VFS operations with automatic ordering to prevent race conditions.
|
`bulkWrite` efficiently processes multiple VFS operations with automatic ordering to prevent race conditions.
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const result = await vfs.bulkWrite([
|
const result = await vfs.bulkWrite([
|
||||||
|
|
@ -439,7 +439,7 @@ console.log(`Successful: ${result.successful}, Failed: ${result.failed.length}`)
|
||||||
- `delete` - Delete file
|
- `delete` - Delete file
|
||||||
- `update` - Update file metadata only
|
- `update` - Update file metadata only
|
||||||
|
|
||||||
**Operation ordering (v6.5.0):**
|
**Operation ordering:**
|
||||||
1. `mkdir` operations run **first**, sequentially, sorted by path depth (shallowest first)
|
1. `mkdir` operations run **first**, sequentially, sorted by path depth (shallowest first)
|
||||||
2. Other operations (`write`, `delete`, `update`) run **after** in parallel batches of 10
|
2. Other operations (`write`, `delete`, `update`) run **after** in parallel batches of 10
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
# VFS Initialization Guide (v5.1.0+)
|
# VFS Initialization Guide
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
|
|
@ -30,7 +30,7 @@ await vfs.init() // ❌ Separate initialization
|
||||||
await vfs.writeFile(...)
|
await vfs.writeFile(...)
|
||||||
```
|
```
|
||||||
|
|
||||||
### After (v5.1.0+):
|
### After:
|
||||||
```javascript
|
```javascript
|
||||||
const brain = new Brainy(...)
|
const brain = new Brainy(...)
|
||||||
await brain.init() // VFS auto-initialized!
|
await brain.init() // VFS auto-initialized!
|
||||||
|
|
@ -53,7 +53,7 @@ await vfs.init()
|
||||||
await vfs.writeFile('/test.txt', 'data')
|
await vfs.writeFile('/test.txt', 'data')
|
||||||
```
|
```
|
||||||
|
|
||||||
### New Pattern (v5.1.0+):
|
### New Pattern:
|
||||||
```javascript
|
```javascript
|
||||||
// Just remove the () and init() call
|
// Just remove the () and init() call
|
||||||
await brain.vfs.writeFile('/test.txt', 'data')
|
await brain.vfs.writeFile('/test.txt', 'data')
|
||||||
|
|
@ -142,7 +142,7 @@ await brain.init() // Required!
|
||||||
await brain.vfs.writeFile(...) // Now this works
|
await brain.vfs.writeFile(...) // Now this works
|
||||||
```
|
```
|
||||||
|
|
||||||
## Fork Support (v5.0.0+)
|
## Fork Support
|
||||||
|
|
||||||
VFS works seamlessly with Brainy's Copy-on-Write (COW) fork feature:
|
VFS works seamlessly with Brainy's Copy-on-Write (COW) fork feature:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ export class ConfigAPI {
|
||||||
// Store in cache
|
// Store in cache
|
||||||
this.configCache.set(key, entry)
|
this.configCache.set(key, entry)
|
||||||
|
|
||||||
// v4.0.0: Persist to storage as NounMetadata
|
// Persist to storage as NounMetadata
|
||||||
const configId = this.CONFIG_NOUN_PREFIX + key
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
||||||
const nounMetadata = {
|
const nounMetadata = {
|
||||||
noun: 'config' as any,
|
noun: 'config' as any,
|
||||||
|
|
@ -94,7 +94,7 @@ export class ConfigAPI {
|
||||||
return defaultValue
|
return defaultValue
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Extract ConfigEntry from NounMetadata
|
// Extract ConfigEntry from NounMetadata
|
||||||
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
|
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
|
||||||
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
|
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
|
||||||
: ((metadata.createdAt as unknown as number) || Date.now())
|
: ((metadata.createdAt as unknown as number) || Date.now())
|
||||||
|
|
@ -140,7 +140,7 @@ export class ConfigAPI {
|
||||||
// Remove from cache
|
// Remove from cache
|
||||||
this.configCache.delete(key)
|
this.configCache.delete(key)
|
||||||
|
|
||||||
// v4.0.0: Remove from storage
|
// Remove from storage
|
||||||
const configId = this.CONFIG_NOUN_PREFIX + key
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
||||||
await this.storage.deleteNounMetadata(configId)
|
await this.storage.deleteNounMetadata(configId)
|
||||||
}
|
}
|
||||||
|
|
@ -149,7 +149,7 @@ export class ConfigAPI {
|
||||||
* List all configuration keys
|
* List all configuration keys
|
||||||
*/
|
*/
|
||||||
async list(): Promise<string[]> {
|
async list(): Promise<string[]> {
|
||||||
// v4.0.0: Get all nouns and filter for config entries
|
// Get all nouns and filter for config entries
|
||||||
const result = await this.storage.getNouns({ pagination: { limit: 10000 } })
|
const result = await this.storage.getNouns({ pagination: { limit: 10000 } })
|
||||||
|
|
||||||
const configKeys: string[] = []
|
const configKeys: string[] = []
|
||||||
|
|
@ -213,7 +213,7 @@ export class ConfigAPI {
|
||||||
for (const [key, entry] of Object.entries(config)) {
|
for (const [key, entry] of Object.entries(config)) {
|
||||||
this.configCache.set(key, entry)
|
this.configCache.set(key, entry)
|
||||||
const configId = this.CONFIG_NOUN_PREFIX + key
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
||||||
// v4.0.0: Convert ConfigEntry to NounMetadata
|
// Convert ConfigEntry to NounMetadata
|
||||||
const nounMetadata = {
|
const nounMetadata = {
|
||||||
noun: 'config' as any,
|
noun: 'config' as any,
|
||||||
...entry,
|
...entry,
|
||||||
|
|
@ -240,7 +240,7 @@ export class ConfigAPI {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Extract ConfigEntry from NounMetadata
|
// Extract ConfigEntry from NounMetadata
|
||||||
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
|
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
|
||||||
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
|
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
|
||||||
: ((metadata.createdAt as unknown as number) || Date.now())
|
: ((metadata.createdAt as unknown as number) || Date.now())
|
||||||
|
|
|
||||||
|
|
@ -210,7 +210,7 @@ export class DataAPI {
|
||||||
this.validateImportItem(mapped)
|
this.validateImportItem(mapped)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Save entity - separate vector and metadata
|
// Save entity - separate vector and metadata
|
||||||
const id = mapped.id || this.generateId()
|
const id = mapped.id || this.generateId()
|
||||||
const noun: HNSWNoun = {
|
const noun: HNSWNoun = {
|
||||||
id,
|
id,
|
||||||
|
|
|
||||||
|
|
@ -375,7 +375,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get CommitLog for temporal features (v5.0.0+)
|
* Get CommitLog for temporal features
|
||||||
*
|
*
|
||||||
* Provides access to commit history for time-travel queries, audit trails,
|
* Provides access to commit history for time-travel queries, audit trails,
|
||||||
* and branch management. Available after initialize() is called.
|
* and branch management. Available after initialize() is called.
|
||||||
|
|
@ -414,7 +414,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get BlobStorage for content-addressable storage (v5.0.0+)
|
* Get BlobStorage for content-addressable storage
|
||||||
*
|
*
|
||||||
* Provides access to the underlying blob storage system for storing
|
* Provides access to the underlying blob storage system for storing
|
||||||
* and retrieving content-addressed data. Available after initialize() is called.
|
* and retrieving content-addressed data. Available after initialize() is called.
|
||||||
|
|
@ -454,7 +454,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get RefManager for branch/ref management (v5.0.0+)
|
* Get RefManager for branch/ref management
|
||||||
*
|
*
|
||||||
* Provides access to the reference manager for creating, updating,
|
* Provides access to the reference manager for creating, updating,
|
||||||
* and managing Git-style branches and refs. Available after initialize() is called.
|
* and managing Git-style branches and refs. Available after initialize() is called.
|
||||||
|
|
@ -496,7 +496,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get current branch name (v5.0.0+)
|
* Get current branch name
|
||||||
*
|
*
|
||||||
* Convenience helper for getting the current branch from the Brainy instance.
|
* Convenience helper for getting the current branch from the Brainy instance.
|
||||||
* Available after initialize() is called.
|
* Available after initialize() is called.
|
||||||
|
|
@ -525,7 +525,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation {
|
||||||
if (typeof brain.getCurrentBranch !== 'function') {
|
if (typeof brain.getCurrentBranch !== 'function') {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`${this.name}: getCurrentBranch() not available on Brainy instance. ` +
|
`${this.name}: getCurrentBranch() not available on Brainy instance. ` +
|
||||||
`This method requires Brainy v5.0.0+.`
|
`This method requires Brainy with VFS support.`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* Format Handler Registry (v5.2.0)
|
* Format Handler Registry
|
||||||
*
|
*
|
||||||
* Central registry for format handlers with:
|
* Central registry for format handlers with:
|
||||||
* - MIME type-based routing
|
* - MIME type-based routing
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ export class IntelligentImportAugmentation extends BaseAugmentation {
|
||||||
enableCSV: true,
|
enableCSV: true,
|
||||||
enableExcel: true,
|
enableExcel: true,
|
||||||
enablePDF: true,
|
enablePDF: true,
|
||||||
enableImage: true, // v5.2.0: Image handler enabled by default
|
enableImage: true, // Image handler enabled by default
|
||||||
maxFileSize: 100 * 1024 * 1024, // 100MB default
|
maxFileSize: 100 * 1024 * 1024, // 100MB default
|
||||||
enableCache: true,
|
enableCache: true,
|
||||||
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
|
cacheTTL: 24 * 60 * 60 * 1000, // 24 hours
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ export class CSVHandler extends BaseFormatHandler {
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
|
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
|
||||||
const totalBytes = buffer.length
|
const totalBytes = buffer.length
|
||||||
|
|
||||||
// v4.5.0: Report total bytes for progress tracking
|
// Report total bytes for progress tracking
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(0)
|
progressHooks.onBytesProcessed(0)
|
||||||
}
|
}
|
||||||
|
|
@ -55,7 +55,7 @@ export class CSVHandler extends BaseFormatHandler {
|
||||||
// Detect delimiter if not specified
|
// Detect delimiter if not specified
|
||||||
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
|
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
|
||||||
|
|
||||||
// v4.5.0: Report progress - parsing started
|
// Report progress - parsing started
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
|
progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
|
||||||
}
|
}
|
||||||
|
|
@ -75,7 +75,7 @@ export class CSVHandler extends BaseFormatHandler {
|
||||||
cast: false // We'll do type inference ourselves
|
cast: false // We'll do type inference ourselves
|
||||||
})
|
})
|
||||||
|
|
||||||
// v4.5.0: Report bytes processed (entire file parsed)
|
// Report bytes processed (entire file parsed)
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(totalBytes)
|
progressHooks.onBytesProcessed(totalBytes)
|
||||||
}
|
}
|
||||||
|
|
@ -83,7 +83,7 @@ export class CSVHandler extends BaseFormatHandler {
|
||||||
// Convert to array of objects
|
// Convert to array of objects
|
||||||
const data = Array.isArray(records) ? records : [records]
|
const data = Array.isArray(records) ? records : [records]
|
||||||
|
|
||||||
// v4.5.0: Report data extraction progress
|
// Report data extraction progress
|
||||||
if (progressHooks?.onDataExtracted) {
|
if (progressHooks?.onDataExtracted) {
|
||||||
progressHooks.onDataExtracted(data.length, data.length)
|
progressHooks.onDataExtracted(data.length, data.length)
|
||||||
}
|
}
|
||||||
|
|
@ -101,7 +101,7 @@ export class CSVHandler extends BaseFormatHandler {
|
||||||
converted[key] = this.convertValue(value, types[key] || 'string')
|
converted[key] = this.convertValue(value, types[key] || 'string')
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report progress every 1000 rows
|
// Report progress every 1000 rows
|
||||||
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
||||||
progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
|
progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
|
||||||
}
|
}
|
||||||
|
|
@ -111,7 +111,7 @@ export class CSVHandler extends BaseFormatHandler {
|
||||||
|
|
||||||
const processingTime = Date.now() - startTime
|
const processingTime = Date.now() - startTime
|
||||||
|
|
||||||
// v4.5.0: Final progress update
|
// Final progress update
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
|
progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ export class ExcelHandler extends BaseFormatHandler {
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
||||||
const totalBytes = buffer.length
|
const totalBytes = buffer.length
|
||||||
|
|
||||||
// v4.5.0: Report start
|
// Report start
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(0)
|
progressHooks.onBytesProcessed(0)
|
||||||
}
|
}
|
||||||
|
|
@ -47,7 +47,7 @@ export class ExcelHandler extends BaseFormatHandler {
|
||||||
// Determine which sheets to process
|
// Determine which sheets to process
|
||||||
const sheetsToProcess = this.getSheetsToProcess(workbook, options)
|
const sheetsToProcess = this.getSheetsToProcess(workbook, options)
|
||||||
|
|
||||||
// v4.5.0: Report workbook loaded
|
// Report workbook loaded
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`)
|
progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`)
|
||||||
}
|
}
|
||||||
|
|
@ -59,7 +59,7 @@ export class ExcelHandler extends BaseFormatHandler {
|
||||||
for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) {
|
for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) {
|
||||||
const sheetName = sheetsToProcess[sheetIndex]
|
const sheetName = sheetsToProcess[sheetIndex]
|
||||||
|
|
||||||
// v4.5.0: Report current sheet
|
// Report current sheet
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(
|
progressHooks.onCurrentItem(
|
||||||
`Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})`
|
`Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})`
|
||||||
|
|
@ -117,19 +117,19 @@ export class ExcelHandler extends BaseFormatHandler {
|
||||||
headers
|
headers
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Estimate bytes processed (sheets are sequential)
|
// Estimate bytes processed (sheets are sequential)
|
||||||
const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes)
|
const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes)
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(bytesProcessed)
|
progressHooks.onBytesProcessed(bytesProcessed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report extraction progress
|
// Report extraction progress
|
||||||
if (progressHooks?.onDataExtracted) {
|
if (progressHooks?.onDataExtracted) {
|
||||||
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report data extraction complete
|
// Report data extraction complete
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`)
|
progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`)
|
||||||
}
|
}
|
||||||
|
|
@ -152,7 +152,7 @@ export class ExcelHandler extends BaseFormatHandler {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report progress every 1000 rows (avoid spam)
|
// Report progress every 1000 rows (avoid spam)
|
||||||
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
||||||
progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`)
|
progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`)
|
||||||
}
|
}
|
||||||
|
|
@ -160,14 +160,14 @@ export class ExcelHandler extends BaseFormatHandler {
|
||||||
return converted
|
return converted
|
||||||
})
|
})
|
||||||
|
|
||||||
// v4.5.0: Final progress - all bytes processed
|
// Final progress - all bytes processed
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(totalBytes)
|
progressHooks.onBytesProcessed(totalBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
const processingTime = Date.now() - startTime
|
const processingTime = Date.now() - startTime
|
||||||
|
|
||||||
// v4.5.0: Report completion
|
// Report completion
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(
|
progressHooks.onCurrentItem(
|
||||||
`Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows`
|
`Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows`
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* Image Import Handler (v5.8.0 - Pure JavaScript)
|
* Image Import Handler
|
||||||
*
|
*
|
||||||
* Handles image files with:
|
* Handles image files with:
|
||||||
* - EXIF metadata extraction (camera, GPS, timestamps) via exifr
|
* - EXIF metadata extraction (camera, GPS, timestamps) via exifr
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG
|
* - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG
|
||||||
*
|
*
|
||||||
* NO NATIVE DEPENDENCIES - Pure JavaScript implementation
|
* NO NATIVE DEPENDENCIES - Pure JavaScript implementation
|
||||||
* Replaces Sharp (v5.7.x) with lightweight pure-JS alternatives
|
* Replaces Sharp with lightweight pure-JS alternatives
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { BaseFormatHandler } from './base.js'
|
import { BaseFormatHandler } from './base.js'
|
||||||
|
|
@ -96,7 +96,7 @@ export interface ImageHandlerOptions extends FormatHandlerOptions {
|
||||||
* Enables developers to import images into the knowledge graph with
|
* Enables developers to import images into the knowledge graph with
|
||||||
* full metadata extraction.
|
* full metadata extraction.
|
||||||
*
|
*
|
||||||
* v5.8.0: Pure JavaScript implementation (no native dependencies)
|
* Pure JavaScript implementation (no native dependencies)
|
||||||
*/
|
*/
|
||||||
export class ImageHandler extends BaseFormatHandler {
|
export class ImageHandler extends BaseFormatHandler {
|
||||||
readonly format = 'image'
|
readonly format = 'image'
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,7 @@ export class PDFHandler extends BaseFormatHandler {
|
||||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
||||||
const totalBytes = buffer.length
|
const totalBytes = buffer.length
|
||||||
|
|
||||||
// v4.5.0: Report start
|
// Report start
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(0)
|
progressHooks.onBytesProcessed(0)
|
||||||
}
|
}
|
||||||
|
|
@ -74,7 +74,7 @@ export class PDFHandler extends BaseFormatHandler {
|
||||||
const metadata = await pdfDoc.getMetadata()
|
const metadata = await pdfDoc.getMetadata()
|
||||||
const numPages = pdfDoc.numPages
|
const numPages = pdfDoc.numPages
|
||||||
|
|
||||||
// v4.5.0: Report document loaded
|
// Report document loaded
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
|
progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
|
||||||
}
|
}
|
||||||
|
|
@ -85,7 +85,7 @@ export class PDFHandler extends BaseFormatHandler {
|
||||||
let detectedTables = 0
|
let detectedTables = 0
|
||||||
|
|
||||||
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
|
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
|
||||||
// v4.5.0: Report current page
|
// Report current page
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
|
progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
|
||||||
}
|
}
|
||||||
|
|
@ -131,19 +131,19 @@ export class PDFHandler extends BaseFormatHandler {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Estimate bytes processed (pages are sequential)
|
// Estimate bytes processed (pages are sequential)
|
||||||
const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
|
const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(bytesProcessed)
|
progressHooks.onBytesProcessed(bytesProcessed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report extraction progress
|
// Report extraction progress
|
||||||
if (progressHooks?.onDataExtracted) {
|
if (progressHooks?.onDataExtracted) {
|
||||||
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Final progress - all bytes processed
|
// Final progress - all bytes processed
|
||||||
if (progressHooks?.onBytesProcessed) {
|
if (progressHooks?.onBytesProcessed) {
|
||||||
progressHooks.onBytesProcessed(totalBytes)
|
progressHooks.onBytesProcessed(totalBytes)
|
||||||
}
|
}
|
||||||
|
|
@ -153,7 +153,7 @@ export class PDFHandler extends BaseFormatHandler {
|
||||||
|
|
||||||
const processingTime = Date.now() - startTime
|
const processingTime = Date.now() - startTime
|
||||||
|
|
||||||
// v4.5.0: Report completion
|
// Report completion
|
||||||
if (progressHooks?.onCurrentItem) {
|
if (progressHooks?.onCurrentItem) {
|
||||||
progressHooks.onCurrentItem(
|
progressHooks.onCurrentItem(
|
||||||
`PDF complete: ${numPages} pages, ${allData.length} items extracted`
|
`PDF complete: ${numPages} pages, ${allData.length} items extracted`
|
||||||
|
|
|
||||||
|
|
@ -18,14 +18,14 @@ export { ExcelHandler } from './handlers/excelHandler.js'
|
||||||
export { PDFHandler } from './handlers/pdfHandler.js'
|
export { PDFHandler } from './handlers/pdfHandler.js'
|
||||||
export { ImageHandler } from './handlers/imageHandler.js'
|
export { ImageHandler } from './handlers/imageHandler.js'
|
||||||
|
|
||||||
// Format Handler Registry (v5.2.0)
|
// Format Handler Registry
|
||||||
export {
|
export {
|
||||||
FormatHandlerRegistry,
|
FormatHandlerRegistry,
|
||||||
globalHandlerRegistry
|
globalHandlerRegistry
|
||||||
} from './FormatHandlerRegistry.js'
|
} from './FormatHandlerRegistry.js'
|
||||||
export type { HandlerRegistration } from './FormatHandlerRegistry.js'
|
export type { HandlerRegistration } from './FormatHandlerRegistry.js'
|
||||||
|
|
||||||
// Image Handler Types (v5.2.0)
|
// Image Handler Types
|
||||||
export type {
|
export type {
|
||||||
ImageMetadata,
|
ImageMetadata,
|
||||||
EXIFData,
|
EXIFData,
|
||||||
|
|
|
||||||
|
|
@ -88,13 +88,13 @@ export interface FormatHandlerOptions {
|
||||||
streaming?: boolean
|
streaming?: boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Progress hooks (v4.5.0)
|
* Progress hooks
|
||||||
* Handlers call these to report progress during processing
|
* Handlers call these to report progress during processing
|
||||||
*/
|
*/
|
||||||
progressHooks?: FormatHandlerProgressHooks
|
progressHooks?: FormatHandlerProgressHooks
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Total file size in bytes (v4.5.0)
|
* Total file size in bytes
|
||||||
* Used for progress percentage calculation
|
* Used for progress percentage calculation
|
||||||
*/
|
*/
|
||||||
totalBytes?: number
|
totalBytes?: number
|
||||||
|
|
@ -168,7 +168,7 @@ export interface IntelligentImportConfig {
|
||||||
/** Enable PDF handler */
|
/** Enable PDF handler */
|
||||||
enablePDF: boolean
|
enablePDF: boolean
|
||||||
|
|
||||||
/** Enable Image handler (v5.2.0) */
|
/** Enable Image handler */
|
||||||
enableImage: boolean
|
enableImage: boolean
|
||||||
|
|
||||||
/** Default options for CSV */
|
/** Default options for CSV */
|
||||||
|
|
@ -180,7 +180,7 @@ export interface IntelligentImportConfig {
|
||||||
/** Default options for PDF */
|
/** Default options for PDF */
|
||||||
pdfDefaults?: Partial<FormatHandlerOptions>
|
pdfDefaults?: Partial<FormatHandlerOptions>
|
||||||
|
|
||||||
/** Default options for Image (v5.2.0) */
|
/** Default options for Image */
|
||||||
imageDefaults?: Partial<FormatHandlerOptions>
|
imageDefaults?: Partial<FormatHandlerOptions>
|
||||||
|
|
||||||
/** Maximum file size to process (bytes) */
|
/** Maximum file size to process (bytes) */
|
||||||
|
|
|
||||||
|
|
@ -168,7 +168,7 @@ export interface TypeMatchResult {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* BrainyTypes - Intelligent type detection for nouns and verbs
|
* BrainyTypes - Intelligent type detection for nouns and verbs
|
||||||
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings
|
* PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings
|
||||||
* Type embeddings are loaded instantly; only input objects are embedded at runtime
|
* Type embeddings are loaded instantly; only input objects are embedded at runtime
|
||||||
*/
|
*/
|
||||||
export class BrainyTypes {
|
export class BrainyTypes {
|
||||||
|
|
|
||||||
|
|
@ -172,7 +172,7 @@ export const coreCommands = {
|
||||||
metadata
|
metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.3.x: Add confidence and weight if provided
|
// Add confidence and weight if provided
|
||||||
if (options.confidence) {
|
if (options.confidence) {
|
||||||
addParams.confidence = parseFloat(options.confidence)
|
addParams.confidence = parseFloat(options.confidence)
|
||||||
}
|
}
|
||||||
|
|
@ -347,7 +347,7 @@ export const coreCommands = {
|
||||||
searchParams.includeRelations = true
|
searchParams.includeRelations = true
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.7.0: VFS is now part of the knowledge graph (included by default)
|
// VFS is now part of the knowledge graph (included by default)
|
||||||
// Users can exclude VFS with --where vfsType exists:false if needed
|
// Users can exclude VFS with --where vfsType exists:false if needed
|
||||||
|
|
||||||
// Triple Intelligence Fusion - custom weighting
|
// Triple Intelligence Fusion - custom weighting
|
||||||
|
|
|
||||||
|
|
@ -312,7 +312,7 @@ ${chalk.cyan('Fork Statistics:')}
|
||||||
},
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Migrate from v4.x to v5.0.0 (one-time)
|
* Migrate storage format (one-time)
|
||||||
*/
|
*/
|
||||||
async migrate(options: MigrateOptions) {
|
async migrate(options: MigrateOptions) {
|
||||||
let spinner: any = null
|
let spinner: any = null
|
||||||
|
|
@ -388,7 +388,7 @@ ${chalk.cyan('Fork Statistics:')}
|
||||||
|
|
||||||
await oldBrain.init()
|
await oldBrain.init()
|
||||||
|
|
||||||
// Create new brain (v5.0.0)
|
// Create new brain
|
||||||
const newBrain = new Brainy({
|
const newBrain = new Brainy({
|
||||||
storage: {
|
storage: {
|
||||||
type: 'filesystem',
|
type: 'filesystem',
|
||||||
|
|
|
||||||
|
|
@ -117,7 +117,7 @@ export const neuralCommand = {
|
||||||
await handleNeighborsCommand(neural, argv)
|
await handleNeighborsCommand(neural, argv)
|
||||||
break
|
break
|
||||||
case 'path':
|
case 'path':
|
||||||
console.log(chalk.yellow('\n⚠️ Semantic path finding coming in v3.21.0'))
|
console.log(chalk.yellow('\n⚠️ Semantic path finding coming soon'))
|
||||||
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
|
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
|
||||||
console.log(chalk.dim('Use "neighbors" and "hierarchy" commands to explore connections'))
|
console.log(chalk.dim('Use "neighbors" and "hierarchy" commands to explore connections'))
|
||||||
break
|
break
|
||||||
|
|
@ -148,7 +148,7 @@ async function promptForAction(): Promise<string> {
|
||||||
{ name: '🎯 Find semantic clusters', value: 'clusters' },
|
{ name: '🎯 Find semantic clusters', value: 'clusters' },
|
||||||
{ name: '🌳 Show item hierarchy', value: 'hierarchy' },
|
{ name: '🌳 Show item hierarchy', value: 'hierarchy' },
|
||||||
{ name: '🕸️ Find semantic neighbors', value: 'neighbors' },
|
{ name: '🕸️ Find semantic neighbors', value: 'neighbors' },
|
||||||
{ name: '🛣️ Find semantic path between items (v3.21.0)', value: 'path', disabled: true },
|
{ name: '🛣️ Find semantic path between items (coming soon)', value: 'path', disabled: true },
|
||||||
{ name: '🚨 Detect outliers', value: 'outliers' },
|
{ name: '🚨 Detect outliers', value: 'outliers' },
|
||||||
{ name: '📊 Generate visualization data', value: 'visualize' }
|
{ name: '📊 Generate visualization data', value: 'visualize' }
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* 💾 Storage Management Commands - v4.0.0
|
* 💾 Storage Management Commands
|
||||||
*
|
*
|
||||||
* Modern interactive CLI for storage lifecycle, cost optimization, and management
|
* Modern interactive CLI for storage lifecycle, cost optimization, and management
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -133,7 +133,7 @@ export const utilityCommands = {
|
||||||
// removeOrphans and rebuildIndex would require new Brainy APIs
|
// removeOrphans and rebuildIndex would require new Brainy APIs
|
||||||
if (options.removeOrphans || options.rebuildIndex) {
|
if (options.removeOrphans || options.rebuildIndex) {
|
||||||
spinner.warn('Advanced cleanup options not yet implemented')
|
spinner.warn('Advanced cleanup options not yet implemented')
|
||||||
console.log(chalk.yellow('\n⚠️ Advanced cleanup features coming in v3.21.0:'))
|
console.log(chalk.yellow('\n⚠️ Advanced cleanup features coming soon:'))
|
||||||
console.log(chalk.dim(' • --remove-orphans: Remove disconnected items'))
|
console.log(chalk.dim(' • --remove-orphans: Remove disconnected items'))
|
||||||
console.log(chalk.dim(' • --rebuild-index: Rebuild vector index'))
|
console.log(chalk.dim(' • --rebuild-index: Rebuild vector index'))
|
||||||
console.log(chalk.dim('\nUse "brainy clean" without options to clear the database'))
|
console.log(chalk.dim('\nUse "brainy clean" without options to clear the database'))
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ let brainyInstance: Brainy | null = null
|
||||||
const getBrainy = async (): Promise<Brainy> => {
|
const getBrainy = async (): Promise<Brainy> => {
|
||||||
if (!brainyInstance) {
|
if (!brainyInstance) {
|
||||||
brainyInstance = new Brainy()
|
brainyInstance = new Brainy()
|
||||||
await brainyInstance.init() // v5.0.1: Initialize brain (VFS auto-initialized here!)
|
await brainyInstance.init() // Initialize brain (VFS auto-initialized here!)
|
||||||
}
|
}
|
||||||
return brainyInstance
|
return brainyInstance
|
||||||
}
|
}
|
||||||
|
|
@ -52,8 +52,8 @@ export const vfsCommands = {
|
||||||
const spinner = ora('Reading file...').start()
|
const spinner = ora('Reading file...').start()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const brain = await getBrainy() // v5.0.1: Await async getBrainy
|
const brain = await getBrainy() // Await async getBrainy
|
||||||
// v5.0.1: VFS auto-initialized, no need for vfs.init()
|
// VFS auto-initialized, no need for vfs.init()
|
||||||
const buffer = await brain.vfs.readFile(path, {
|
const buffer = await brain.vfs.readFile(path, {
|
||||||
encoding: options.encoding as any
|
encoding: options.encoding as any
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -68,7 +68,7 @@ ${chalk.cyan('Examples:')}
|
||||||
$ brainy vfs search "React components"
|
$ brainy vfs search "React components"
|
||||||
$ brainy vfs similar /code/Button.tsx
|
$ brainy vfs similar /code/Button.tsx
|
||||||
|
|
||||||
${chalk.dim('# Storage management (v4.0.0)')}
|
${chalk.dim('# Storage management')}
|
||||||
$ brainy storage status --quota
|
$ brainy storage status --quota
|
||||||
$ brainy storage lifecycle set ${chalk.dim('# Interactive mode')}
|
$ brainy storage lifecycle set ${chalk.dim('# Interactive mode')}
|
||||||
$ brainy storage cost-estimate
|
$ brainy storage cost-estimate
|
||||||
|
|
@ -217,11 +217,11 @@ program
|
||||||
|
|
||||||
program
|
program
|
||||||
.command('path <from> <to>')
|
.command('path <from> <to>')
|
||||||
.description('Find semantic path between items (v3.21.0)')
|
.description('Find semantic path between items')
|
||||||
.option('--steps', 'Show step-by-step path')
|
.option('--steps', 'Show step-by-step path')
|
||||||
.option('--max-hops <number>', 'Maximum path length', '5')
|
.option('--max-hops <number>', 'Maximum path length', '5')
|
||||||
.action(() => {
|
.action(() => {
|
||||||
console.log(chalk.yellow('\n⚠️ Semantic path finding coming in v3.21.0'))
|
console.log(chalk.yellow('\n⚠️ Semantic path finding coming soon'))
|
||||||
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
|
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
|
||||||
console.log(chalk.dim('Use "brainy neighbors" and "brainy hierarchy" to explore connections'))
|
console.log(chalk.dim('Use "brainy neighbors" and "brainy hierarchy" to explore connections'))
|
||||||
})
|
})
|
||||||
|
|
@ -446,7 +446,7 @@ program
|
||||||
vfsCommands.tree(path, options)
|
vfsCommands.tree(path, options)
|
||||||
})
|
})
|
||||||
|
|
||||||
// ===== Storage Management Commands (v4.0.0) =====
|
// ===== Storage Management Commands =====
|
||||||
|
|
||||||
program
|
program
|
||||||
.command('storage')
|
.command('storage')
|
||||||
|
|
@ -610,7 +610,7 @@ program
|
||||||
.option('--iterations <n>', 'Number of iterations', '100')
|
.option('--iterations <n>', 'Number of iterations', '100')
|
||||||
.action(utilityCommands.benchmark)
|
.action(utilityCommands.benchmark)
|
||||||
|
|
||||||
// ===== COW Commands (v5.0.0) - Instant Fork & Branching =====
|
// ===== COW Commands - Instant Fork & Branching =====
|
||||||
|
|
||||||
program
|
program
|
||||||
.command('fork [name]')
|
.command('fork [name]')
|
||||||
|
|
|
||||||
|
|
@ -625,7 +625,7 @@ export async function promptCommand(): Promise<string> {
|
||||||
*/
|
*/
|
||||||
export async function startInteractiveMode() {
|
export async function startInteractiveMode() {
|
||||||
console.log(chalk.cyan('\n🧠 Brainy Interactive Mode\n'))
|
console.log(chalk.cyan('\n🧠 Brainy Interactive Mode\n'))
|
||||||
console.log(chalk.yellow('Interactive REPL mode coming in v3.20.0\n'))
|
console.log(chalk.yellow('Interactive REPL mode coming soon\n'))
|
||||||
console.log(chalk.dim('Use specific commands for now: brainy add, brainy search, etc.'))
|
console.log(chalk.dim('Use specific commands for now: brainy add, brainy search, etc.'))
|
||||||
process.exit(0)
|
process.exit(0)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -57,7 +57,7 @@ export type DistanceFunction = (a: Vector, b: Vector) => number
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Embedding function for converting data to vectors
|
* Embedding function for converting data to vectors
|
||||||
* v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
|
* Now properly typed - accepts string, string array (batch), or object, no `any`
|
||||||
*/
|
*/
|
||||||
export type EmbeddingFunction = (data: string | string[] | Record<string, unknown>) => Promise<Vector>
|
export type EmbeddingFunction = (data: string | string[] | Record<string, unknown>) => Promise<Vector>
|
||||||
|
|
||||||
|
|
@ -72,7 +72,7 @@ export interface EmbeddingModel {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Embed data into a vector
|
* Embed data into a vector
|
||||||
* v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any`
|
* Now properly typed - accepts string, string array (batch), or object, no `any`
|
||||||
*/
|
*/
|
||||||
embed(data: string | string[] | Record<string, unknown>): Promise<Vector>
|
embed(data: string | string[] | Record<string, unknown>): Promise<Vector>
|
||||||
|
|
||||||
|
|
@ -83,9 +83,9 @@ export interface EmbeddingModel {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* HNSW graph noun - Pure vector structure (v4.0.0)
|
* HNSW graph noun - Pure vector structure
|
||||||
*
|
*
|
||||||
* v4.0.0 BREAKING CHANGE: metadata field removed
|
* metadata field removed
|
||||||
* - Stores ONLY vector data for optimal memory usage
|
* - Stores ONLY vector data for optimal memory usage
|
||||||
* - Metadata stored separately and combined on retrieval
|
* - Metadata stored separately and combined on retrieval
|
||||||
* - 25% memory reduction @ 1B scale (no in-memory metadata)
|
* - 25% memory reduction @ 1B scale (no in-memory metadata)
|
||||||
|
|
@ -100,15 +100,15 @@ export interface HNSWNoun {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Lightweight verb for HNSW index storage - Core relational structure (v4.0.0)
|
* Lightweight verb for HNSW index storage - Core relational structure
|
||||||
*
|
*
|
||||||
* Core fields (v3.50.1+): verb/sourceId/targetId are first-class fields
|
* Core fields: verb/sourceId/targetId are first-class fields
|
||||||
* These are NOT metadata - they're the essence of what a verb IS:
|
* These are NOT metadata - they're the essence of what a verb IS:
|
||||||
* - verb: The relationship type (creates, contains, etc.) - needed for routing & display
|
* - verb: The relationship type (creates, contains, etc.) - needed for routing & display
|
||||||
* - sourceId: What entity this verb connects FROM - needed for graph traversal
|
* - sourceId: What entity this verb connects FROM - needed for graph traversal
|
||||||
* - targetId: What entity this verb connects TO - needed for graph traversal
|
* - targetId: What entity this verb connects TO - needed for graph traversal
|
||||||
*
|
*
|
||||||
* v4.0.0 BREAKING CHANGE: metadata field removed
|
* metadata field removed
|
||||||
* - Stores ONLY vector + core relational data
|
* - Stores ONLY vector + core relational data
|
||||||
* - User metadata (weight, custom fields) stored separately
|
* - User metadata (weight, custom fields) stored separately
|
||||||
* - 10x faster metadata-only updates (skip HNSW rebuild)
|
* - 10x faster metadata-only updates (skip HNSW rebuild)
|
||||||
|
|
@ -134,9 +134,9 @@ export interface HNSWVerb {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Noun metadata structure (v4.8.0)
|
* Noun metadata structure
|
||||||
*
|
*
|
||||||
* v4.8.0 BREAKING CHANGE: Now contains ONLY custom user-defined fields
|
* Now contains ONLY custom user-defined fields
|
||||||
* - Standard fields (confidence, weight, timestamps, etc.) moved to top-level in HNSWNounWithMetadata
|
* - Standard fields (confidence, weight, timestamps, etc.) moved to top-level in HNSWNounWithMetadata
|
||||||
* - This interface represents custom metadata stored separately from vector data
|
* - This interface represents custom metadata stored separately from vector data
|
||||||
* - Storage format unchanged (backward compatible at storage layer)
|
* - Storage format unchanged (backward compatible at storage layer)
|
||||||
|
|
@ -162,9 +162,9 @@ export interface NounMetadata {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verb metadata structure (v4.8.0)
|
* Verb metadata structure
|
||||||
*
|
*
|
||||||
* v4.8.0 BREAKING CHANGE: Now contains ONLY custom user-defined fields
|
* Now contains ONLY custom user-defined fields
|
||||||
* - Standard fields (weight, confidence, timestamps, etc.) moved to top-level in HNSWVerbWithMetadata
|
* - Standard fields (weight, confidence, timestamps, etc.) moved to top-level in HNSWVerbWithMetadata
|
||||||
* - This interface represents custom metadata stored separately from vector + core relational data
|
* - This interface represents custom metadata stored separately from vector + core relational data
|
||||||
* - Storage format unchanged (backward compatible at storage layer)
|
* - Storage format unchanged (backward compatible at storage layer)
|
||||||
|
|
@ -190,9 +190,9 @@ export interface VerbMetadata {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Combined noun structure for transport/API boundaries (v4.8.0)
|
* Combined noun structure for transport/API boundaries
|
||||||
*
|
*
|
||||||
* v4.8.0 BREAKING CHANGE: Standard fields moved to top-level
|
* Standard fields moved to top-level
|
||||||
* - ALL standard fields (confidence, weight, timestamps, etc.) are now at top-level
|
* - ALL standard fields (confidence, weight, timestamps, etc.) are now at top-level
|
||||||
* - metadata contains ONLY custom user-defined fields
|
* - metadata contains ONLY custom user-defined fields
|
||||||
* - Provides clean, predictable API: entity.confidence always works
|
* - Provides clean, predictable API: entity.confidence always works
|
||||||
|
|
@ -230,9 +230,9 @@ export interface HNSWNounWithMetadata {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Combined verb structure for transport/API boundaries (v4.8.0)
|
* Combined verb structure for transport/API boundaries
|
||||||
*
|
*
|
||||||
* v4.8.0 BREAKING CHANGE: Standard fields moved to top-level
|
* Standard fields moved to top-level
|
||||||
* - ALL standard fields (weight, confidence, timestamps, etc.) are now at top-level
|
* - ALL standard fields (weight, confidence, timestamps, etc.) are now at top-level
|
||||||
* - metadata contains ONLY custom user-defined fields
|
* - metadata contains ONLY custom user-defined fields
|
||||||
* - Provides clean, predictable API: verb.weight always works
|
* - Provides clean, predictable API: verb.weight always works
|
||||||
|
|
@ -308,7 +308,7 @@ export interface HNSWConfig {
|
||||||
efSearch: number // Size of the dynamic candidate list during search
|
efSearch: number // Size of the dynamic candidate list during search
|
||||||
ml: number // Maximum level
|
ml: number // Maximum level
|
||||||
useDiskBasedIndex?: boolean // Whether to use disk-based index
|
useDiskBasedIndex?: boolean // Whether to use disk-based index
|
||||||
maxConcurrentNeighborWrites?: number // Maximum concurrent neighbor updates during insert (v4.10.0+). Default: unlimited (full concurrency)
|
maxConcurrentNeighborWrites?: number // Maximum concurrent neighbor updates during insert. Default: unlimited (full concurrency)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -548,7 +548,7 @@ export interface StatisticsData {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Change record for getChangesSince (v4.0.0)
|
* Change record for getChangesSince
|
||||||
* Replaces `any[]` with properly typed structure
|
* Replaces `any[]` with properly typed structure
|
||||||
*/
|
*/
|
||||||
export interface Change {
|
export interface Change {
|
||||||
|
|
@ -563,27 +563,27 @@ export interface StorageAdapter {
|
||||||
init(): Promise<void>
|
init(): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save noun - Pure HNSW vector data only (v4.0.0)
|
* Save noun - Pure HNSW vector data only
|
||||||
* @param noun Pure HNSW vector data (no metadata)
|
* @param noun Pure HNSW vector data (no metadata)
|
||||||
* Note: Use saveNounMetadata() to save metadata separately
|
* Note: Use saveNounMetadata() to save metadata separately
|
||||||
*/
|
*/
|
||||||
saveNoun(noun: HNSWNoun): Promise<void>
|
saveNoun(noun: HNSWNoun): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save noun metadata separately (v4.0.0)
|
* Save noun metadata separately
|
||||||
* @param id Noun ID
|
* @param id Noun ID
|
||||||
* @param metadata Noun metadata
|
* @param metadata Noun metadata
|
||||||
*/
|
*/
|
||||||
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
|
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete noun metadata (v4.0.0)
|
* Delete noun metadata
|
||||||
* @param id Noun ID
|
* @param id Noun ID
|
||||||
*/
|
*/
|
||||||
deleteNounMetadata(id: string): Promise<void>
|
deleteNounMetadata(id: string): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get noun with metadata combined (v4.0.0)
|
* Get noun with metadata combined
|
||||||
* @returns Combined HNSWNounWithMetadata or null
|
* @returns Combined HNSWNounWithMetadata or null
|
||||||
*/
|
*/
|
||||||
getNoun(id: string): Promise<HNSWNounWithMetadata | null>
|
getNoun(id: string): Promise<HNSWNounWithMetadata | null>
|
||||||
|
|
@ -622,14 +622,14 @@ export interface StorageAdapter {
|
||||||
deleteNoun(id: string): Promise<void>
|
deleteNoun(id: string): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save verb - Pure HNSW verb with core fields only (v4.0.0)
|
* Save verb - Pure HNSW verb with core fields only
|
||||||
* @param verb Pure HNSW verb data (vector + core fields, no user metadata)
|
* @param verb Pure HNSW verb data (vector + core fields, no user metadata)
|
||||||
* Note: Use saveVerbMetadata() to save metadata separately
|
* Note: Use saveVerbMetadata() to save metadata separately
|
||||||
*/
|
*/
|
||||||
saveVerb(verb: HNSWVerb): Promise<void>
|
saveVerb(verb: HNSWVerb): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get verb with metadata combined (v4.0.0)
|
* Get verb with metadata combined
|
||||||
* @returns Combined HNSWVerbWithMetadata or null
|
* @returns Combined HNSWVerbWithMetadata or null
|
||||||
*/
|
*/
|
||||||
getVerb(id: string): Promise<HNSWVerbWithMetadata | null>
|
getVerb(id: string): Promise<HNSWVerbWithMetadata | null>
|
||||||
|
|
@ -686,14 +686,14 @@ export interface StorageAdapter {
|
||||||
deleteVerb(id: string): Promise<void>
|
deleteVerb(id: string): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save metadata (v4.0.0: now typed)
|
* Save metadata
|
||||||
* @param id Entity ID
|
* @param id Entity ID
|
||||||
* @param metadata Typed noun metadata
|
* @param metadata Typed noun metadata
|
||||||
*/
|
*/
|
||||||
saveMetadata(id: string, metadata: NounMetadata): Promise<void>
|
saveMetadata(id: string, metadata: NounMetadata): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get metadata (v4.0.0: now typed)
|
* Get metadata
|
||||||
* @param id Entity ID
|
* @param id Entity ID
|
||||||
* @returns Typed noun metadata or null
|
* @returns Typed noun metadata or null
|
||||||
*/
|
*/
|
||||||
|
|
@ -702,26 +702,26 @@ export interface StorageAdapter {
|
||||||
/**
|
/**
|
||||||
* Get multiple metadata objects in batches (prevents socket exhaustion)
|
* Get multiple metadata objects in batches (prevents socket exhaustion)
|
||||||
* @param ids Array of IDs to get metadata for
|
* @param ids Array of IDs to get metadata for
|
||||||
* @returns Promise that resolves to a Map of id -> metadata (v4.0.0: typed)
|
* @returns Promise that resolves to a Map of id -> metadata
|
||||||
*/
|
*/
|
||||||
getMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>
|
getMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get noun metadata from storage (v4.0.0: now typed)
|
* Get noun metadata from storage
|
||||||
* @param id The ID of the noun
|
* @param id The ID of the noun
|
||||||
* @returns Promise that resolves to the metadata or null if not found
|
* @returns Promise that resolves to the metadata or null if not found
|
||||||
*/
|
*/
|
||||||
getNounMetadata(id: string): Promise<NounMetadata | null>
|
getNounMetadata(id: string): Promise<NounMetadata | null>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Batch get multiple nouns with vectors (v6.2.0 - N+1 fix)
|
* Batch get multiple nouns with vectors
|
||||||
* @param ids Array of noun IDs to fetch
|
* @param ids Array of noun IDs to fetch
|
||||||
* @returns Map of id → HNSWNounWithMetadata (only successful reads included)
|
* @returns Map of id → HNSWNounWithMetadata (only successful reads included)
|
||||||
*/
|
*/
|
||||||
getNounBatch?(ids: string[]): Promise<Map<string, HNSWNounWithMetadata>>
|
getNounBatch?(ids: string[]): Promise<Map<string, HNSWNounWithMetadata>>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save verb metadata to storage (v4.0.0: now typed)
|
* Save verb metadata to storage
|
||||||
* @param id The ID of the verb
|
* @param id The ID of the verb
|
||||||
* @param metadata The metadata to save
|
* @param metadata The metadata to save
|
||||||
* @returns Promise that resolves when the metadata is saved
|
* @returns Promise that resolves when the metadata is saved
|
||||||
|
|
@ -729,14 +729,14 @@ export interface StorageAdapter {
|
||||||
saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void>
|
saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get verb metadata from storage (v4.0.0: now typed)
|
* Get verb metadata from storage
|
||||||
* @param id The ID of the verb
|
* @param id The ID of the verb
|
||||||
* @returns Promise that resolves to the metadata or null if not found
|
* @returns Promise that resolves to the metadata or null if not found
|
||||||
*/
|
*/
|
||||||
getVerbMetadata(id: string): Promise<VerbMetadata | null>
|
getVerbMetadata(id: string): Promise<VerbMetadata | null>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Batch get multiple verbs (v6.2.0 - N+1 fix)
|
* Batch get multiple verbs
|
||||||
* @param ids Array of verb IDs to fetch
|
* @param ids Array of verb IDs to fetch
|
||||||
* @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
|
* @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
|
||||||
*/
|
*/
|
||||||
|
|
@ -745,7 +745,7 @@ export interface StorageAdapter {
|
||||||
clear(): Promise<void>
|
clear(): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Batch delete multiple objects from storage (v4.0.0)
|
* Batch delete multiple objects from storage
|
||||||
* Efficient deletion of large numbers of entities using cloud provider batch APIs.
|
* Efficient deletion of large numbers of entities using cloud provider batch APIs.
|
||||||
* Significantly faster and cheaper than individual deletes (up to 1000x speedup).
|
* Significantly faster and cheaper than individual deletes (up to 1000x speedup).
|
||||||
*
|
*
|
||||||
|
|
@ -853,7 +853,7 @@ export interface StorageAdapter {
|
||||||
flushStatisticsToStorage(): Promise<void>
|
flushStatisticsToStorage(): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Track field names from a JSON document (v4.0.0: now typed)
|
* Track field names from a JSON document
|
||||||
* @param jsonDocument The JSON document to extract field names from
|
* @param jsonDocument The JSON document to extract field names from
|
||||||
* @param service The service that inserted the data
|
* @param service The service that inserted the data
|
||||||
*/
|
*/
|
||||||
|
|
@ -872,7 +872,7 @@ export interface StorageAdapter {
|
||||||
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>
|
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get changes since a specific timestamp (v4.0.0: now typed)
|
* Get changes since a specific timestamp
|
||||||
* @param timestamp The timestamp to get changes since
|
* @param timestamp The timestamp to get changes since
|
||||||
* @param limit Optional limit on the number of changes to return
|
* @param limit Optional limit on the number of changes to return
|
||||||
* @returns Promise that resolves to an array of properly typed changes
|
* @returns Promise that resolves to an array of properly typed changes
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,7 @@ use js_sys::{Array, Float32Array};
|
||||||
use tokenizers::Tokenizer;
|
use tokenizers::Tokenizer;
|
||||||
use wasm_bindgen::prelude::*;
|
use wasm_bindgen::prelude::*;
|
||||||
|
|
||||||
// v7.2.0: Model weights are NO LONGER embedded in WASM
|
// Model weights are NO LONGER embedded in WASM
|
||||||
//
|
//
|
||||||
// Previous design: 90MB WASM with model weights embedded via include_bytes!()
|
// Previous design: 90MB WASM with model weights embedded via include_bytes!()
|
||||||
// Problem: WASM parsing/compilation took 139+ seconds on throttled CPU (Cloud Run)
|
// Problem: WASM parsing/compilation took 139+ seconds on throttled CPU (Cloud Run)
|
||||||
|
|
@ -78,7 +78,7 @@ impl EmbeddingEngine {
|
||||||
|
|
||||||
/// Load the model and tokenizer from bytes
|
/// Load the model and tokenizer from bytes
|
||||||
///
|
///
|
||||||
/// v7.2.0: This is now the ONLY way to initialize the engine.
|
/// This is now the ONLY way to initialize the engine.
|
||||||
/// Model weights are no longer embedded in WASM for faster initialization.
|
/// Model weights are no longer embedded in WASM for faster initialization.
|
||||||
///
|
///
|
||||||
/// # Arguments
|
/// # Arguments
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
* Pure Rust/WASM implementation for sentence embeddings.
|
* Pure Rust/WASM implementation for sentence embeddings.
|
||||||
* Works with Bun, Node.js, Bun --compile, and browsers.
|
* Works with Bun, Node.js, Bun --compile, and browsers.
|
||||||
*
|
*
|
||||||
* v7.2.0 Architecture (20x faster initialization):
|
* Architecture (20x faster initialization):
|
||||||
* - WASM file: ~2.4MB (inference code only)
|
* - WASM file: ~2.4MB (inference code only)
|
||||||
* - Model files: ~88MB (loaded separately as raw bytes)
|
* - Model files: ~88MB (loaded separately as raw bytes)
|
||||||
* - Init time: ~5-7 seconds (vs 139 seconds with embedded model)
|
* - Init time: ~5-7 seconds (vs 139 seconds with embedded model)
|
||||||
|
|
@ -34,7 +34,7 @@ interface CandleWasmModule {
|
||||||
}
|
}
|
||||||
|
|
||||||
interface CandleEngineInstance {
|
interface CandleEngineInstance {
|
||||||
// v7.2.0: load() is the only initialization method (no more embedded model)
|
// load() is the only initialization method (no more embedded model)
|
||||||
load(modelBytes: Uint8Array, tokenizerBytes: Uint8Array, configBytes: Uint8Array): void
|
load(modelBytes: Uint8Array, tokenizerBytes: Uint8Array, configBytes: Uint8Array): void
|
||||||
is_ready(): boolean
|
is_ready(): boolean
|
||||||
embed(text: string): Float32Array
|
embed(text: string): Float32Array
|
||||||
|
|
@ -54,7 +54,7 @@ let globalInitPromise: Promise<void> | null = null
|
||||||
* Uses the Candle ML framework (Rust/WASM) for inference.
|
* Uses the Candle ML framework (Rust/WASM) for inference.
|
||||||
* Supports all-MiniLM-L6-v2 with 384-dimensional embeddings.
|
* Supports all-MiniLM-L6-v2 with 384-dimensional embeddings.
|
||||||
*
|
*
|
||||||
* v7.2.0: Model weights are loaded separately from WASM for 20x faster init.
|
* Model weights are loaded separately from WASM for 20x faster init.
|
||||||
* For bun --compile deployments, both WASM and model files are automatically
|
* For bun --compile deployments, both WASM and model files are automatically
|
||||||
* embedded in the binary - single file deployment still works.
|
* embedded in the binary - single file deployment still works.
|
||||||
*/
|
*/
|
||||||
|
|
@ -104,7 +104,7 @@ export class CandleEmbeddingEngine {
|
||||||
/**
|
/**
|
||||||
* Perform actual initialization
|
* Perform actual initialization
|
||||||
*
|
*
|
||||||
* v7.2.0: WASM and model files are loaded separately for 20x faster init.
|
* WASM and model files are loaded separately for 20x faster init.
|
||||||
* - WASM (~2.4MB): Compiles in ~3-5 seconds
|
* - WASM (~2.4MB): Compiles in ~3-5 seconds
|
||||||
* - Model (~88MB): Loads as raw bytes in ~1-2 seconds
|
* - Model (~88MB): Loads as raw bytes in ~1-2 seconds
|
||||||
* - Total: ~5-7 seconds (vs 139 seconds with embedded model)
|
* - Total: ~5-7 seconds (vs 139 seconds with embedded model)
|
||||||
|
|
@ -153,7 +153,7 @@ export class CandleEmbeddingEngine {
|
||||||
/**
|
/**
|
||||||
* Load the WASM module
|
* Load the WASM module
|
||||||
*
|
*
|
||||||
* v7.2.0: WASM is now only ~2.4MB (inference code only, no model weights).
|
* WASM is now only ~2.4MB (inference code only, no model weights).
|
||||||
* Uses wasmLoader.ts for cross-environment compatibility.
|
* Uses wasmLoader.ts for cross-environment compatibility.
|
||||||
*/
|
*/
|
||||||
private async loadWasmModule(): Promise<CandleWasmModule> {
|
private async loadWasmModule(): Promise<CandleWasmModule> {
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* Universal Model Loader for Candle Embeddings
|
* Universal Model Loader for Candle Embeddings
|
||||||
*
|
*
|
||||||
* v7.2.0: Model weights are now loaded separately from WASM for faster initialization.
|
* Model weights are now loaded separately from WASM for faster initialization.
|
||||||
* This reduces WASM compilation time from 139 seconds to ~3-5 seconds.
|
* This reduces WASM compilation time from 139 seconds to ~3-5 seconds.
|
||||||
*
|
*
|
||||||
* Loads model files from appropriate source based on environment:
|
* Loads model files from appropriate source based on environment:
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,7 @@ export class GraphAdjacencyIndex {
|
||||||
private lsmTreeVerbsBySource: LSMTree // sourceId -> verbIds
|
private lsmTreeVerbsBySource: LSMTree // sourceId -> verbIds
|
||||||
private lsmTreeVerbsByTarget: LSMTree // targetId -> verbIds
|
private lsmTreeVerbsByTarget: LSMTree // targetId -> verbIds
|
||||||
|
|
||||||
// v5.7.0: ID-only tracking for billion-scale memory optimization
|
// ID-only tracking for billion-scale memory optimization
|
||||||
// Previous: Map<string, GraphVerb> stored full objects (128GB @ 1B verbs)
|
// Previous: Map<string, GraphVerb> stored full objects (128GB @ 1B verbs)
|
||||||
// Now: Set<string> stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction
|
// Now: Set<string> stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction
|
||||||
private verbIdSet = new Set<string>()
|
private verbIdSet = new Set<string>()
|
||||||
|
|
@ -117,7 +117,7 @@ export class GraphAdjacencyIndex {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the graph index (lazy initialization)
|
* Initialize the graph index (lazy initialization)
|
||||||
* v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency
|
* Added defensive auto-rebuild check for verbIdSet consistency
|
||||||
*/
|
*/
|
||||||
private async ensureInitialized(): Promise<void> {
|
private async ensureInitialized(): Promise<void> {
|
||||||
if (this.initialized) {
|
if (this.initialized) {
|
||||||
|
|
@ -129,7 +129,7 @@ export class GraphAdjacencyIndex {
|
||||||
await this.lsmTreeVerbsBySource.init()
|
await this.lsmTreeVerbsBySource.init()
|
||||||
await this.lsmTreeVerbsByTarget.init()
|
await this.lsmTreeVerbsByTarget.init()
|
||||||
|
|
||||||
// v6.3.0: Defensive check - if LSM-trees have data but verbIdSet is empty,
|
// Defensive check - if LSM-trees have data but verbIdSet is empty,
|
||||||
// the index was created without proper rebuild (shouldn't happen with singleton
|
// the index was created without proper rebuild (shouldn't happen with singleton
|
||||||
// pattern but protects against edge cases and future refactoring)
|
// pattern but protects against edge cases and future refactoring)
|
||||||
const lsmTreeSize = this.lsmTreeVerbsBySource.size()
|
const lsmTreeSize = this.lsmTreeVerbsBySource.size()
|
||||||
|
|
@ -151,7 +151,7 @@ export class GraphAdjacencyIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Populate verbIdSet from storage without full rebuild (v6.3.0)
|
* Populate verbIdSet from storage without full rebuild
|
||||||
* Lighter weight than full rebuild - only loads verb IDs, not all verb data
|
* Lighter weight than full rebuild - only loads verb IDs, not all verb data
|
||||||
* @private
|
* @private
|
||||||
*/
|
*/
|
||||||
|
|
@ -192,7 +192,7 @@ export class GraphAdjacencyIndex {
|
||||||
* Core API - Neighbor lookup with LSM-tree storage
|
* Core API - Neighbor lookup with LSM-tree storage
|
||||||
*
|
*
|
||||||
* O(log n) with bloom filter optimization (90% of queries skip disk I/O)
|
* O(log n) with bloom filter optimization (90% of queries skip disk I/O)
|
||||||
* v5.8.0: Added pagination support for high-degree nodes
|
* Added pagination support for high-degree nodes
|
||||||
*
|
*
|
||||||
* @param id Entity ID to get neighbors for
|
* @param id Entity ID to get neighbors for
|
||||||
* @param optionsOrDirection Optional: direction string OR options object
|
* @param optionsOrDirection Optional: direction string OR options object
|
||||||
|
|
@ -252,7 +252,7 @@ export class GraphAdjacencyIndex {
|
||||||
// Convert to array for pagination
|
// Convert to array for pagination
|
||||||
let result = Array.from(neighbors)
|
let result = Array.from(neighbors)
|
||||||
|
|
||||||
// Apply pagination if requested (v5.8.0)
|
// Apply pagination if requested
|
||||||
if (options?.limit !== undefined || options?.offset !== undefined) {
|
if (options?.limit !== undefined || options?.offset !== undefined) {
|
||||||
const offset = options.offset || 0
|
const offset = options.offset || 0
|
||||||
const limit = options.limit !== undefined ? options.limit : result.length
|
const limit = options.limit !== undefined ? options.limit : result.length
|
||||||
|
|
@ -273,8 +273,8 @@ export class GraphAdjacencyIndex {
|
||||||
* Get verb IDs by source - Billion-scale optimization for getVerbsBySource
|
* Get verb IDs by source - Billion-scale optimization for getVerbsBySource
|
||||||
*
|
*
|
||||||
* O(log n) LSM-tree lookup with bloom filter optimization
|
* O(log n) LSM-tree lookup with bloom filter optimization
|
||||||
* v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround)
|
* Filters out deleted verb IDs (tombstone deletion workaround)
|
||||||
* v5.8.0: Added pagination support for entities with many relationships
|
* Added pagination support for entities with many relationships
|
||||||
*
|
*
|
||||||
* @param sourceId Source entity ID
|
* @param sourceId Source entity ID
|
||||||
* @param options Optional configuration
|
* @param options Optional configuration
|
||||||
|
|
@ -318,7 +318,7 @@ export class GraphAdjacencyIndex {
|
||||||
const allIds = verbIds || []
|
const allIds = verbIds || []
|
||||||
let result = allIds.filter(id => this.verbIdSet.has(id))
|
let result = allIds.filter(id => this.verbIdSet.has(id))
|
||||||
|
|
||||||
// Apply pagination if requested (v5.8.0)
|
// Apply pagination if requested
|
||||||
if (options?.limit !== undefined || options?.offset !== undefined) {
|
if (options?.limit !== undefined || options?.offset !== undefined) {
|
||||||
const offset = options.offset || 0
|
const offset = options.offset || 0
|
||||||
const limit = options.limit !== undefined ? options.limit : result.length
|
const limit = options.limit !== undefined ? options.limit : result.length
|
||||||
|
|
@ -332,8 +332,8 @@ export class GraphAdjacencyIndex {
|
||||||
* Get verb IDs by target - Billion-scale optimization for getVerbsByTarget
|
* Get verb IDs by target - Billion-scale optimization for getVerbsByTarget
|
||||||
*
|
*
|
||||||
* O(log n) LSM-tree lookup with bloom filter optimization
|
* O(log n) LSM-tree lookup with bloom filter optimization
|
||||||
* v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround)
|
* Filters out deleted verb IDs (tombstone deletion workaround)
|
||||||
* v5.8.0: Added pagination support for popular target entities
|
* Added pagination support for popular target entities
|
||||||
*
|
*
|
||||||
* @param targetId Target entity ID
|
* @param targetId Target entity ID
|
||||||
* @param options Optional configuration
|
* @param options Optional configuration
|
||||||
|
|
@ -377,7 +377,7 @@ export class GraphAdjacencyIndex {
|
||||||
const allIds = verbIds || []
|
const allIds = verbIds || []
|
||||||
let result = allIds.filter(id => this.verbIdSet.has(id))
|
let result = allIds.filter(id => this.verbIdSet.has(id))
|
||||||
|
|
||||||
// Apply pagination if requested (v5.8.0)
|
// Apply pagination if requested
|
||||||
if (options?.limit !== undefined || options?.offset !== undefined) {
|
if (options?.limit !== undefined || options?.offset !== undefined) {
|
||||||
const offset = options.offset || 0
|
const offset = options.offset || 0
|
||||||
const limit = options.limit !== undefined ? options.limit : result.length
|
const limit = options.limit !== undefined ? options.limit : result.length
|
||||||
|
|
@ -414,7 +414,7 @@ export class GraphAdjacencyIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Batch get multiple verbs with caching (v6.2.0 - N+1 fix)
|
* Batch get multiple verbs with caching
|
||||||
*
|
*
|
||||||
* **Performance**: Eliminates N+1 pattern for verb loading
|
* **Performance**: Eliminates N+1 pattern for verb loading
|
||||||
* - Current: N × getVerbCached() = N × 50ms on GCS = 250ms for 5 verbs
|
* - Current: N × getVerbCached() = N × 50ms on GCS = 250ms for 5 verbs
|
||||||
|
|
@ -433,7 +433,6 @@ export class GraphAdjacencyIndex {
|
||||||
* @param verbIds Array of verb IDs to fetch
|
* @param verbIds Array of verb IDs to fetch
|
||||||
* @returns Map of verbId → GraphVerb (only successful reads included)
|
* @returns Map of verbId → GraphVerb (only successful reads included)
|
||||||
*
|
*
|
||||||
* @since v6.2.0
|
|
||||||
*/
|
*/
|
||||||
async getVerbsBatchCached(verbIds: string[]): Promise<Map<string, GraphVerb>> {
|
async getVerbsBatchCached(verbIds: string[]): Promise<Map<string, GraphVerb>> {
|
||||||
const results = new Map<string, GraphVerb>()
|
const results = new Map<string, GraphVerb>()
|
||||||
|
|
@ -514,7 +513,7 @@ export class GraphAdjacencyIndex {
|
||||||
const targetStats = this.lsmTreeTarget.getStats()
|
const targetStats = this.lsmTreeTarget.getStats()
|
||||||
|
|
||||||
// Note: Exact unique node counts would require full LSM-tree scan
|
// Note: Exact unique node counts would require full LSM-tree scan
|
||||||
// v5.7.0: Using verbIdSet (ID-only tracking) for memory efficiency
|
// Using verbIdSet (ID-only tracking) for memory efficiency
|
||||||
const uniqueSourceNodes = this.verbIdSet.size
|
const uniqueSourceNodes = this.verbIdSet.size
|
||||||
const uniqueTargetNodes = this.verbIdSet.size
|
const uniqueTargetNodes = this.verbIdSet.size
|
||||||
const totalNodes = this.verbIdSet.size
|
const totalNodes = this.verbIdSet.size
|
||||||
|
|
@ -622,13 +621,13 @@ export class GraphAdjacencyIndex {
|
||||||
// Clear current index
|
// Clear current index
|
||||||
this.verbIdSet.clear()
|
this.verbIdSet.clear()
|
||||||
this.totalRelationshipsIndexed = 0
|
this.totalRelationshipsIndexed = 0
|
||||||
// v6.2.4: CRITICAL FIX - Clear relationship counts to prevent accumulation
|
// CRITICAL FIX - Clear relationship counts to prevent accumulation
|
||||||
this.relationshipCountsByType.clear()
|
this.relationshipCountsByType.clear()
|
||||||
|
|
||||||
// Note: LSM-trees will be recreated from storage via their own initialization
|
// Note: LSM-trees will be recreated from storage via their own initialization
|
||||||
// Verb data will be loaded on-demand via UnifiedCache
|
// Verb data will be loaded on-demand via UnifiedCache
|
||||||
|
|
||||||
// Adaptive loading strategy based on storage type (v4.2.4)
|
// Adaptive loading strategy based on storage type
|
||||||
const storageType = this.storage?.constructor.name || ''
|
const storageType = this.storage?.constructor.name || ''
|
||||||
const isLocalStorage =
|
const isLocalStorage =
|
||||||
storageType === 'FileSystemStorage' ||
|
storageType === 'FileSystemStorage' ||
|
||||||
|
|
@ -754,7 +753,7 @@ export class GraphAdjacencyIndex {
|
||||||
bytes += targetStats.memTableMemory
|
bytes += targetStats.memTableMemory
|
||||||
|
|
||||||
// Verb ID set (memory-efficient: IDs only, ~8 bytes per ID pointer)
|
// Verb ID set (memory-efficient: IDs only, ~8 bytes per ID pointer)
|
||||||
// v5.7.0: Previous verbIndex Map stored full objects (128 bytes each = 128GB @ 1B verbs)
|
// Previous verbIndex Map stored full objects (128 bytes each = 128GB @ 1B verbs)
|
||||||
// Now: verbIdSet stores only IDs (~8 bytes each = ~100KB @ 1B verbs) = 1,280,000x reduction
|
// Now: verbIdSet stores only IDs (~8 bytes each = ~100KB @ 1B verbs) = 1,280,000x reduction
|
||||||
bytes += this.verbIdSet.size * 8
|
bytes += this.verbIdSet.size * 8
|
||||||
|
|
||||||
|
|
@ -792,7 +791,7 @@ export class GraphAdjacencyIndex {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush LSM-tree MemTables to disk
|
* Flush LSM-tree MemTables to disk
|
||||||
* CRITICAL FIX (v3.43.2): Now public so it can be called from brain.flush()
|
* CRITICAL FIX: Now public so it can be called from brain.flush()
|
||||||
*/
|
*/
|
||||||
async flush(): Promise<void> {
|
async flush(): Promise<void> {
|
||||||
if (!this.initialized) {
|
if (!this.initialized) {
|
||||||
|
|
|
||||||
|
|
@ -35,18 +35,18 @@ export class HNSWIndex {
|
||||||
private distanceFunction: DistanceFunction
|
private distanceFunction: DistanceFunction
|
||||||
private dimension: number | null = null
|
private dimension: number | null = null
|
||||||
private useParallelization: boolean = true // Whether to use parallelization for performance-critical operations
|
private useParallelization: boolean = true // Whether to use parallelization for performance-critical operations
|
||||||
private storage: BaseStorage | null = null // Storage adapter for HNSW persistence (v3.35.0+)
|
private storage: BaseStorage | null = null // Storage adapter for HNSW persistence
|
||||||
|
|
||||||
// Universal memory management (v3.36.0+)
|
// Universal memory management
|
||||||
private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes
|
private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes
|
||||||
// Always-adaptive caching (v3.36.0+) - no "mode" concept, system adapts automatically
|
// Always-adaptive caching - no "mode" concept, system adapts automatically
|
||||||
|
|
||||||
// COW (Copy-on-Write) support - v5.0.0
|
// COW (Copy-on-Write) support
|
||||||
private cowEnabled: boolean = false
|
private cowEnabled: boolean = false
|
||||||
private cowModifiedNodes: Set<string> = new Set()
|
private cowModifiedNodes: Set<string> = new Set()
|
||||||
private cowParent: HNSWIndex | null = null
|
private cowParent: HNSWIndex | null = null
|
||||||
|
|
||||||
// v6.2.8: Deferred HNSW persistence for cloud storage performance
|
// Deferred HNSW persistence for cloud storage performance
|
||||||
// In deferred mode, HNSW connections are only persisted on flush/close
|
// In deferred mode, HNSW connections are only persisted on flush/close
|
||||||
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
|
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
|
||||||
private persistMode: 'immediate' | 'deferred' = 'immediate'
|
private persistMode: 'immediate' | 'deferred' = 'immediate'
|
||||||
|
|
@ -86,7 +86,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* v6.2.8: Flush dirty HNSW data to storage
|
* Flush dirty HNSW data to storage
|
||||||
*
|
*
|
||||||
* In deferred persistence mode, HNSW connections are tracked as dirty but not
|
* In deferred persistence mode, HNSW connections are tracked as dirty but not
|
||||||
* immediately persisted. Call flush() to persist all pending changes.
|
* immediately persisted. Call flush() to persist all pending changes.
|
||||||
|
|
@ -361,6 +361,19 @@ export class HNSWIndex {
|
||||||
this.entryPointId = id
|
this.entryPointId = id
|
||||||
this.maxLevel = nounLevel
|
this.maxLevel = nounLevel
|
||||||
this.nouns.set(id, noun)
|
this.nouns.set(id, noun)
|
||||||
|
|
||||||
|
// Persist system data for first noun (previously skipped)
|
||||||
|
if (this.storage && this.persistMode === 'immediate') {
|
||||||
|
await this.storage.saveHNSWSystem({
|
||||||
|
entryPointId: this.entryPointId,
|
||||||
|
maxLevel: this.maxLevel
|
||||||
|
}).catch(error => {
|
||||||
|
console.error('Failed to persist initial HNSW system data:', error)
|
||||||
|
})
|
||||||
|
} else if (this.persistMode === 'deferred') {
|
||||||
|
this.dirtySystem = true
|
||||||
|
}
|
||||||
|
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -438,7 +451,7 @@ export class HNSWIndex {
|
||||||
)
|
)
|
||||||
|
|
||||||
// Add bidirectional connections
|
// Add bidirectional connections
|
||||||
// PERFORMANCE OPTIMIZATION (v4.10.0): Collect all neighbor updates for concurrent execution
|
// PERFORMANCE OPTIMIZATION: Collect all neighbor updates for concurrent execution
|
||||||
const neighborUpdates: Array<{
|
const neighborUpdates: Array<{
|
||||||
neighborId: string
|
neighborId: string
|
||||||
promise: Promise<void>
|
promise: Promise<void>
|
||||||
|
|
@ -467,9 +480,9 @@ export class HNSWIndex {
|
||||||
await this.pruneConnections(neighbor, level)
|
await this.pruneConnections(neighbor, level)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Persist updated neighbor HNSW data (v3.35.0+)
|
// Persist updated neighbor HNSW data
|
||||||
//
|
//
|
||||||
// v6.2.8: Deferred persistence mode for cloud storage performance
|
// Deferred persistence mode for cloud storage performance
|
||||||
// In deferred mode, we track dirty nodes instead of persisting immediately
|
// In deferred mode, we track dirty nodes instead of persisting immediately
|
||||||
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
|
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
|
||||||
if (this.storage && this.persistMode === 'immediate') {
|
if (this.storage && this.persistMode === 'immediate') {
|
||||||
|
|
@ -561,8 +574,8 @@ export class HNSWIndex {
|
||||||
this.highLevelNodes.get(nounLevel)!.add(id)
|
this.highLevelNodes.get(nounLevel)!.add(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Persist HNSW graph data to storage (v3.35.0+)
|
// Persist HNSW graph data to storage
|
||||||
// v6.2.8: Respect persistMode setting
|
// Respect persistMode setting
|
||||||
if (this.storage && this.persistMode === 'immediate') {
|
if (this.storage && this.persistMode === 'immediate') {
|
||||||
// IMMEDIATE MODE: Original behavior - persist new entity and system data
|
// IMMEDIATE MODE: Original behavior - persist new entity and system data
|
||||||
const connectionsObj: Record<string, string[]> = {}
|
const connectionsObj: Record<string, string[]> = {}
|
||||||
|
|
@ -594,7 +607,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* O(1) entry point recovery using highLevelNodes index (v6.2.3).
|
* O(1) entry point recovery using highLevelNodes index.
|
||||||
* At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
|
* At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
|
||||||
* For tiny indexes with only level 0-1 nodes, any node works as entry point.
|
* For tiny indexes with only level 0-1 nodes, any node works as entry point.
|
||||||
*/
|
*/
|
||||||
|
|
@ -640,7 +653,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start from the entry point
|
// Start from the entry point
|
||||||
// If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3)
|
// If entry point is null but nouns exist, attempt O(1) recovery
|
||||||
if (!this.entryPointId && this.nouns.size > 0) {
|
if (!this.entryPointId && this.nouns.size > 0) {
|
||||||
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
|
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
|
||||||
if (recoveredId) {
|
if (recoveredId) {
|
||||||
|
|
@ -656,7 +669,7 @@ export class HNSWIndex {
|
||||||
|
|
||||||
let entryPoint = this.nouns.get(this.entryPointId)
|
let entryPoint = this.nouns.get(this.entryPointId)
|
||||||
if (!entryPoint) {
|
if (!entryPoint) {
|
||||||
// Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3)
|
// Entry point ID exists but noun was deleted - O(1) recovery
|
||||||
if (this.nouns.size > 0) {
|
if (this.nouns.size > 0) {
|
||||||
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
|
const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
|
||||||
if (recoveredId) {
|
if (recoveredId) {
|
||||||
|
|
@ -944,7 +957,7 @@ export class HNSWIndex {
|
||||||
/**
|
/**
|
||||||
* Get vector safely (always uses adaptive caching via UnifiedCache)
|
* Get vector safely (always uses adaptive caching via UnifiedCache)
|
||||||
*
|
*
|
||||||
* Production-grade adaptive caching (v3.36.0+):
|
* Production-grade adaptive caching:
|
||||||
* - Vector already loaded: Returns immediately (O(1))
|
* - Vector already loaded: Returns immediately (O(1))
|
||||||
* - Vector in cache: Loads from UnifiedCache (O(1) hash lookup)
|
* - Vector in cache: Loads from UnifiedCache (O(1) hash lookup)
|
||||||
* - Vector on disk: Loads from storage → UnifiedCache (O(disk))
|
* - Vector on disk: Loads from storage → UnifiedCache (O(disk))
|
||||||
|
|
@ -990,7 +1003,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get vector synchronously if available in memory (v3.36.0+)
|
* Get vector synchronously if available in memory
|
||||||
*
|
*
|
||||||
* Sync fast path optimization:
|
* Sync fast path optimization:
|
||||||
* - Vector in memory: Returns immediately (zero overhead)
|
* - Vector in memory: Returns immediately (zero overhead)
|
||||||
|
|
@ -1048,7 +1061,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Calculate distance with sync fast path (v3.36.0+)
|
* Calculate distance with sync fast path
|
||||||
*
|
*
|
||||||
* Eliminates async overhead when vectors are in memory:
|
* Eliminates async overhead when vectors are in memory:
|
||||||
* - Sync path: Vector in memory → returns number (zero overhead)
|
* - Sync path: Vector in memory → returns number (zero overhead)
|
||||||
|
|
@ -1093,7 +1106,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Rebuild HNSW index from persisted graph data (v3.35.0+)
|
* Rebuild HNSW index from persisted graph data
|
||||||
*
|
*
|
||||||
* This is a production-grade O(N) rebuild that restores the pre-computed graph structure
|
* This is a production-grade O(N) rebuild that restores the pre-computed graph structure
|
||||||
* from storage. Much faster than re-building which is O(N log N).
|
* from storage. Much faster than re-building which is O(N log N).
|
||||||
|
|
@ -1157,7 +1170,7 @@ export class HNSWIndex {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 4: Adaptive loading strategy based on storage type (v4.2.4)
|
// Step 4: Adaptive loading strategy based on storage type
|
||||||
// FileSystem/Memory/OPFS: Load all at once (avoids repeated getAllShardedFiles() calls)
|
// FileSystem/Memory/OPFS: Load all at once (avoids repeated getAllShardedFiles() calls)
|
||||||
// Cloud (GCS/S3/R2): Use pagination (efficient native cloud APIs)
|
// Cloud (GCS/S3/R2): Use pagination (efficient native cloud APIs)
|
||||||
const storageType = this.storage?.constructor.name || ''
|
const storageType = this.storage?.constructor.name || ''
|
||||||
|
|
@ -1238,7 +1251,7 @@ export class HNSWIndex {
|
||||||
prodLog.info(`HNSW: Using cloud pagination strategy (${storageType})`)
|
prodLog.info(`HNSW: Using cloud pagination strategy (${storageType})`)
|
||||||
|
|
||||||
let hasMore = true
|
let hasMore = true
|
||||||
let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop)
|
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
|
||||||
|
|
||||||
while (hasMore) {
|
while (hasMore) {
|
||||||
// Fetch batch of nouns from storage (cast needed as method is not in base interface)
|
// Fetch batch of nouns from storage (cast needed as method is not in base interface)
|
||||||
|
|
@ -1249,7 +1262,7 @@ export class HNSWIndex {
|
||||||
nextCursor?: string
|
nextCursor?: string
|
||||||
} = await (this.storage as any).getNounsWithPagination({
|
} = await (this.storage as any).getNounsWithPagination({
|
||||||
limit: batchSize,
|
limit: batchSize,
|
||||||
offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored)
|
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
|
||||||
})
|
})
|
||||||
|
|
||||||
// Set total count on first batch
|
// Set total count on first batch
|
||||||
|
|
@ -1307,11 +1320,11 @@ export class HNSWIndex {
|
||||||
|
|
||||||
// Check for more data
|
// Check for more data
|
||||||
hasMore = result.hasMore
|
hasMore = result.hasMore
|
||||||
offset += batchSize // v5.7.11: Increment offset for next page
|
offset += batchSize // Increment offset for next page
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1))
|
// Step 5: CRITICAL - Recover entry point if missing)
|
||||||
// This ensures consistency even if getHNSWSystem() returned null
|
// This ensures consistency even if getHNSWSystem() returned null
|
||||||
if (this.nouns.size > 0 && this.entryPointId === null) {
|
if (this.nouns.size > 0 && this.entryPointId === null) {
|
||||||
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup')
|
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup')
|
||||||
|
|
@ -1426,7 +1439,7 @@ export class HNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get cache performance statistics for monitoring and diagnostics (v3.36.0+)
|
* Get cache performance statistics for monitoring and diagnostics
|
||||||
*
|
*
|
||||||
* Production-grade monitoring:
|
* Production-grade monitoring:
|
||||||
* - Adaptive caching strategy (preloading vs on-demand)
|
* - Adaptive caching strategy (preloading vs on-demand)
|
||||||
|
|
|
||||||
|
|
@ -118,7 +118,7 @@ export class TypeAwareHNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* v6.2.8: Flush dirty HNSW data to storage for all type-specific indexes
|
* Flush dirty HNSW data to storage for all type-specific indexes
|
||||||
*
|
*
|
||||||
* In deferred persistence mode, HNSW connections are tracked as dirty but not
|
* In deferred persistence mode, HNSW connections are tracked as dirty but not
|
||||||
* immediately persisted. Call flush() to persist all pending changes across
|
* immediately persisted. Call flush() to persist all pending changes across
|
||||||
|
|
@ -502,7 +502,7 @@ export class TypeAwareHNSWIndex {
|
||||||
|
|
||||||
// Load ALL nouns ONCE and route to correct type indexes
|
// Load ALL nouns ONCE and route to correct type indexes
|
||||||
// This is O(N) instead of O(42*N) from the previous parallel approach
|
// This is O(N) instead of O(42*N) from the previous parallel approach
|
||||||
let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop)
|
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
|
||||||
let hasMore = true
|
let hasMore = true
|
||||||
let totalLoaded = 0
|
let totalLoaded = 0
|
||||||
const loadedByType = new Map<NounType, number>()
|
const loadedByType = new Map<NounType, number>()
|
||||||
|
|
@ -515,13 +515,13 @@ export class TypeAwareHNSWIndex {
|
||||||
totalCount?: number
|
totalCount?: number
|
||||||
} = await (this.storage as any).getNounsWithPagination({
|
} = await (this.storage as any).getNounsWithPagination({
|
||||||
limit: batchSize,
|
limit: batchSize,
|
||||||
offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored)
|
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
|
||||||
})
|
})
|
||||||
|
|
||||||
// Route each noun to its type index
|
// Route each noun to its type index
|
||||||
for (const nounData of result.items) {
|
for (const nounData of result.items) {
|
||||||
try {
|
try {
|
||||||
// v7.3.0: Use 'type' property from HNSWNounWithMetadata (not 'nounType')
|
// Use 'type' property from HNSWNounWithMetadata (not 'nounType')
|
||||||
// Previously accessed wrong property, causing N+1 metadata fetches
|
// Previously accessed wrong property, causing N+1 metadata fetches
|
||||||
// getNounsWithPagination already includes type in response
|
// getNounsWithPagination already includes type in response
|
||||||
const nounType = nounData.type
|
const nounType = nounData.type
|
||||||
|
|
@ -578,7 +578,7 @@ export class TypeAwareHNSWIndex {
|
||||||
}
|
}
|
||||||
|
|
||||||
hasMore = result.hasMore
|
hasMore = result.hasMore
|
||||||
offset += batchSize // v5.7.11: Increment offset for next page
|
offset += batchSize // Increment offset for next page
|
||||||
|
|
||||||
// Progress logging
|
// Progress logging
|
||||||
if (totalLoaded % 1000 === 0) {
|
if (totalLoaded % 1000 === 0) {
|
||||||
|
|
@ -593,12 +593,26 @@ export class TypeAwareHNSWIndex {
|
||||||
let entryPointId: string | null = null
|
let entryPointId: string | null = null
|
||||||
|
|
||||||
for (const [id, noun] of (index as any).nouns.entries()) {
|
for (const [id, noun] of (index as any).nouns.entries()) {
|
||||||
if (noun.level > maxLevel) {
|
if (entryPointId === null || noun.level > maxLevel) {
|
||||||
maxLevel = noun.level
|
maxLevel = noun.level
|
||||||
entryPointId = id
|
entryPointId = id
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Recovery: if still null after loop but nouns exist
|
||||||
|
if ((index as any).nouns.size > 0 && !entryPointId) {
|
||||||
|
const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1()
|
||||||
|
entryPointId = recoveredId
|
||||||
|
maxLevel = recoveredLevel
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validation: if entry point doesn't exist in loaded nouns
|
||||||
|
if (entryPointId && !(index as any).nouns.has(entryPointId)) {
|
||||||
|
const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1()
|
||||||
|
entryPointId = recoveredId
|
||||||
|
maxLevel = recoveredLevel
|
||||||
|
}
|
||||||
|
|
||||||
;(index as any).entryPointId = entryPointId
|
;(index as any).entryPointId = entryPointId
|
||||||
;(index as any).maxLevel = maxLevel
|
;(index as any).maxLevel = maxLevel
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -111,7 +111,7 @@ export class FormatDetector {
|
||||||
return 'docx'
|
return 'docx'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Images (v5.2.0: ImageHandler support)
|
// Images (ImageHandler support)
|
||||||
if (mimeType.startsWith('image/')) {
|
if (mimeType.startsWith('image/')) {
|
||||||
return 'image'
|
return 'image'
|
||||||
}
|
}
|
||||||
|
|
@ -134,7 +134,7 @@ export class FormatDetector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// YAML detection (v4.2.0)
|
// YAML detection
|
||||||
if (this.looksLikeYAML(trimmed)) {
|
if (this.looksLikeYAML(trimmed)) {
|
||||||
return {
|
return {
|
||||||
format: 'yaml',
|
format: 'yaml',
|
||||||
|
|
@ -206,7 +206,7 @@ export class FormatDetector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Image formats (v5.2.0)
|
// Image formats
|
||||||
// JPEG: FF D8 FF
|
// JPEG: FF D8 FF
|
||||||
if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) {
|
if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) {
|
||||||
return {
|
return {
|
||||||
|
|
@ -384,7 +384,7 @@ export class FormatDetector {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if content looks like YAML
|
* Check if content looks like YAML
|
||||||
* v4.2.0: Added YAML detection
|
* Added YAML detection
|
||||||
*/
|
*/
|
||||||
private looksLikeYAML(content: string): boolean {
|
private looksLikeYAML(content: string): boolean {
|
||||||
const lines = content.split('\n').filter(l => l.trim()).slice(0, 20)
|
const lines = content.split('\n').filter(l => l.trim()).slice(0, 20)
|
||||||
|
|
|
||||||
|
|
@ -37,10 +37,10 @@ export interface ImportSource {
|
||||||
/** Optional filename hint */
|
/** Optional filename hint */
|
||||||
filename?: string
|
filename?: string
|
||||||
|
|
||||||
/** HTTP headers for URL imports (v4.2.0) */
|
/** HTTP headers for URL imports */
|
||||||
headers?: Record<string, string>
|
headers?: Record<string, string>
|
||||||
|
|
||||||
/** Basic authentication for URL imports (v4.2.0) */
|
/** Basic authentication for URL imports */
|
||||||
auth?: {
|
auth?: {
|
||||||
username: string
|
username: string
|
||||||
password: string
|
password: string
|
||||||
|
|
@ -93,7 +93,7 @@ export interface ValidImportOptions {
|
||||||
/** Create relationships in knowledge graph */
|
/** Create relationships in knowledge graph */
|
||||||
createRelationships?: boolean
|
createRelationships?: boolean
|
||||||
|
|
||||||
/** Create provenance relationships (document → entity) [v4.9.0] */
|
/** Create provenance relationships (document → entity) */
|
||||||
createProvenanceLinks?: boolean
|
createProvenanceLinks?: boolean
|
||||||
|
|
||||||
/** Preserve source file in VFS */
|
/** Preserve source file in VFS */
|
||||||
|
|
@ -144,7 +144,7 @@ export interface ValidImportOptions {
|
||||||
customMetadata?: Record<string, any>
|
customMetadata?: Record<string, any>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Progress callback for tracking import progress (v4.2.0+)
|
* Progress callback for tracking import progress
|
||||||
*
|
*
|
||||||
* **Streaming Architecture** (always enabled):
|
* **Streaming Architecture** (always enabled):
|
||||||
* - Indexes are flushed periodically during import (adaptive intervals)
|
* - Indexes are flushed periodically during import (adaptive intervals)
|
||||||
|
|
@ -227,21 +227,21 @@ export type ImportOptions = ValidImportOptions & DeprecatedImportOptions
|
||||||
|
|
||||||
export interface ImportProgress {
|
export interface ImportProgress {
|
||||||
stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete'
|
stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete'
|
||||||
/** Phase of import - extraction or relationship building (v3.49.0) */
|
/** Phase of import - extraction or relationship building */
|
||||||
phase?: 'extraction' | 'relationships'
|
phase?: 'extraction' | 'relationships'
|
||||||
message: string
|
message: string
|
||||||
processed?: number
|
processed?: number
|
||||||
/** Alias for processed, used in relationship phase (v3.49.0) */
|
/** Alias for processed, used in relationship phase */
|
||||||
current?: number
|
current?: number
|
||||||
total?: number
|
total?: number
|
||||||
entities?: number
|
entities?: number
|
||||||
relationships?: number
|
relationships?: number
|
||||||
/** Rows per second (v3.38.0) */
|
/** Rows per second */
|
||||||
throughput?: number
|
throughput?: number
|
||||||
/** Estimated time remaining in ms (v3.38.0) */
|
/** Estimated time remaining in ms */
|
||||||
eta?: number
|
eta?: number
|
||||||
/**
|
/**
|
||||||
* Whether data is queryable at this point (v4.2.0+)
|
* Whether data is queryable at this point
|
||||||
*
|
*
|
||||||
* When true, indexes have been flushed and queries will return up-to-date results.
|
* When true, indexes have been flushed and queries will return up-to-date results.
|
||||||
* When false, data exists in storage but indexes may not be current (queries may be slower/incomplete).
|
* When false, data exists in storage but indexes may not be current (queries may be slower/incomplete).
|
||||||
|
|
@ -357,7 +357,7 @@ export class ImportCoordinator {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Import from any source with auto-detection
|
* Import from any source with auto-detection
|
||||||
* v4.2.0: Now supports URL imports with authentication
|
* Now supports URL imports with authentication
|
||||||
*/
|
*/
|
||||||
async import(
|
async import(
|
||||||
source: Buffer | string | object | ImportSource,
|
source: Buffer | string | object | ImportSource,
|
||||||
|
|
@ -365,10 +365,10 @@ export class ImportCoordinator {
|
||||||
): Promise<ImportResult> {
|
): Promise<ImportResult> {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
||||||
// Validate options (v4.0.0+: Reject deprecated v3.x options)
|
// Validate options (Reject deprecated options)
|
||||||
this.validateOptions(options)
|
this.validateOptions(options)
|
||||||
|
|
||||||
// Normalize source (v4.2.0: handles URL fetching)
|
// Normalize source (handles URL fetching)
|
||||||
const normalizedSource = await this.normalizeSource(source, options.format)
|
const normalizedSource = await this.normalizeSource(source, options.format)
|
||||||
|
|
||||||
// Report detection stage
|
// Report detection stage
|
||||||
|
|
@ -387,10 +387,10 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set defaults early (needed for tracking context)
|
// Set defaults early (needed for tracking context)
|
||||||
// CRITICAL FIX (v4.3.2): Spread options FIRST, then apply defaults
|
// CRITICAL FIX: Spread options FIRST, then apply defaults
|
||||||
// Previously: ...options at the end overwrote normalized defaults with undefined
|
// Previously: ...options at the end overwrote normalized defaults with undefined
|
||||||
// Now: Defaults properly override undefined values
|
// Now: Defaults properly override undefined values
|
||||||
// v4.4.0: Enable AI features by default for smarter imports
|
// Enable AI features by default for smarter imports
|
||||||
const opts = {
|
const opts = {
|
||||||
...options, // Spread first to get all options
|
...options, // Spread first to get all options
|
||||||
vfsPath: options.vfsPath || `/imports/${Date.now()}`,
|
vfsPath: options.vfsPath || `/imports/${Date.now()}`,
|
||||||
|
|
@ -399,13 +399,13 @@ export class ImportCoordinator {
|
||||||
createRelationships: options.createRelationships !== false,
|
createRelationships: options.createRelationships !== false,
|
||||||
preserveSource: options.preserveSource !== false,
|
preserveSource: options.preserveSource !== false,
|
||||||
enableDeduplication: options.enableDeduplication !== false,
|
enableDeduplication: options.enableDeduplication !== false,
|
||||||
enableNeuralExtraction: options.enableNeuralExtraction !== false, // v4.4.0: Default true
|
enableNeuralExtraction: options.enableNeuralExtraction !== false, // Default true
|
||||||
enableRelationshipInference: options.enableRelationshipInference !== false, // v4.4.0: Default true
|
enableRelationshipInference: options.enableRelationshipInference !== false, // Default true
|
||||||
enableConceptExtraction: options.enableConceptExtraction !== false, // Already defaults to true
|
enableConceptExtraction: options.enableConceptExtraction !== false, // Already defaults to true
|
||||||
deduplicationThreshold: options.deduplicationThreshold || 0.85
|
deduplicationThreshold: options.deduplicationThreshold || 0.85
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate tracking context (v4.10.0+: Unified import/project tracking)
|
// Generate tracking context (Unified import/project tracking)
|
||||||
const importId = options.importId || uuidv4()
|
const importId = options.importId || uuidv4()
|
||||||
const projectId = options.projectId || this.deriveProjectId(opts.vfsPath)
|
const projectId = options.projectId || this.deriveProjectId(opts.vfsPath)
|
||||||
const trackingContext: TrackingContext = {
|
const trackingContext: TrackingContext = {
|
||||||
|
|
@ -441,13 +441,13 @@ export class ImportCoordinator {
|
||||||
groupBy: opts.groupBy,
|
groupBy: opts.groupBy,
|
||||||
customGrouping: opts.customGrouping,
|
customGrouping: opts.customGrouping,
|
||||||
preserveSource: opts.preserveSource,
|
preserveSource: opts.preserveSource,
|
||||||
// v5.1.2: Fix sourceBuffer for file paths - type is 'path' not 'buffer' from normalizeSource()
|
// Fix sourceBuffer for file paths - type is 'path' not 'buffer' from normalizeSource()
|
||||||
sourceBuffer: Buffer.isBuffer(normalizedSource.data) ? normalizedSource.data as Buffer : undefined,
|
sourceBuffer: Buffer.isBuffer(normalizedSource.data) ? normalizedSource.data as Buffer : undefined,
|
||||||
sourceFilename: normalizedSource.filename || `import.${detection.format}`,
|
sourceFilename: normalizedSource.filename || `import.${detection.format}`,
|
||||||
createRelationshipFile: true,
|
createRelationshipFile: true,
|
||||||
createMetadataFile: true,
|
createMetadataFile: true,
|
||||||
trackingContext, // v4.10.0: Pass tracking metadata to VFS
|
trackingContext, // Pass tracking metadata to VFS
|
||||||
// v4.11.1: Pass progress callback for VFS creation updates
|
// Pass progress callback for VFS creation updates
|
||||||
onProgress: (vfsProgress) => {
|
onProgress: (vfsProgress) => {
|
||||||
options.onProgress?.({
|
options.onProgress?.({
|
||||||
stage: 'storing-vfs',
|
stage: 'storing-vfs',
|
||||||
|
|
@ -473,7 +473,7 @@ export class ImportCoordinator {
|
||||||
sourceFilename: normalizedSource.filename || `import.${detection.format}`,
|
sourceFilename: normalizedSource.filename || `import.${detection.format}`,
|
||||||
format: detection.format
|
format: detection.format
|
||||||
},
|
},
|
||||||
trackingContext // v4.10.0: Pass tracking metadata to graph creation
|
trackingContext // Pass tracking metadata to graph creation
|
||||||
)
|
)
|
||||||
|
|
||||||
// Report complete
|
// Report complete
|
||||||
|
|
@ -520,7 +520,7 @@ export class ImportCoordinator {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CRITICAL FIX (v3.43.2): Auto-flush all indexes before returning
|
// CRITICAL FIX: Auto-flush all indexes before returning
|
||||||
// Ensures imported data survives server restarts
|
// Ensures imported data survives server restarts
|
||||||
// Bug #5: Import data was only in memory, lost on restart
|
// Bug #5: Import data was only in memory, lost on restart
|
||||||
options.onProgress?.({
|
options.onProgress?.({
|
||||||
|
|
@ -535,7 +535,7 @@ export class ImportCoordinator {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Normalize source to ImportSource
|
* Normalize source to ImportSource
|
||||||
* v4.2.0: Now async to support URL fetching
|
* Now async to support URL fetching
|
||||||
*/
|
*/
|
||||||
private async normalizeSource(
|
private async normalizeSource(
|
||||||
source: Buffer | string | object | ImportSource,
|
source: Buffer | string | object | ImportSource,
|
||||||
|
|
@ -616,7 +616,7 @@ export class ImportCoordinator {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch content from URL
|
* Fetch content from URL
|
||||||
* v4.2.0: Supports authentication and custom headers
|
* Supports authentication and custom headers
|
||||||
*/
|
*/
|
||||||
private async fetchUrl(source: ImportSource): Promise<ImportSource> {
|
private async fetchUrl(source: ImportSource): Promise<ImportSource> {
|
||||||
const url = typeof source.data === 'string' ? source.data : String(source.data)
|
const url = typeof source.data === 'string' ? source.data : String(source.data)
|
||||||
|
|
@ -722,7 +722,7 @@ export class ImportCoordinator {
|
||||||
format: SupportedFormat,
|
format: SupportedFormat,
|
||||||
options: ImportOptions
|
options: ImportOptions
|
||||||
): Promise<any> {
|
): Promise<any> {
|
||||||
// v5.2.0: Check if IntelligentImportAugmentation already extracted data
|
// Check if IntelligentImportAugmentation already extracted data
|
||||||
if ((options as any)._intelligentImport && (options as any)._extractedData) {
|
if ((options as any)._intelligentImport && (options as any)._extractedData) {
|
||||||
const extractedData = (options as any)._extractedData
|
const extractedData = (options as any)._extractedData
|
||||||
// Convert extracted data to ExtractedRow format
|
// Convert extracted data to ExtractedRow format
|
||||||
|
|
@ -760,7 +760,7 @@ export class ImportCoordinator {
|
||||||
enableConceptExtraction: options.enableConceptExtraction !== false,
|
enableConceptExtraction: options.enableConceptExtraction !== false,
|
||||||
confidenceThreshold: options.confidenceThreshold || 0.6,
|
confidenceThreshold: options.confidenceThreshold || 0.6,
|
||||||
onProgress: (stats: any) => {
|
onProgress: (stats: any) => {
|
||||||
// Enhanced progress reporting (v3.38.0) with throughput and ETA
|
// Enhanced progress reporting with throughput and ETA
|
||||||
const message = stats.throughput
|
const message = stats.throughput
|
||||||
? `Extracting entities from ${format} (${stats.throughput} rows/sec, ETA: ${Math.round(stats.eta / 1000)}s)...`
|
? `Extracting entities from ${format} (${stats.throughput} rows/sec, ETA: ${Math.round(stats.eta / 1000)}s)...`
|
||||||
: `Extracting entities from ${format}...`
|
: `Extracting entities from ${format}...`
|
||||||
|
|
@ -825,7 +825,7 @@ export class ImportCoordinator {
|
||||||
return await this.docxImporter.extract(docxBuffer, extractOptions)
|
return await this.docxImporter.extract(docxBuffer, extractOptions)
|
||||||
|
|
||||||
case 'image':
|
case 'image':
|
||||||
// v5.2.0: Images are handled by IntelligentImportAugmentation
|
// Images are handled by IntelligentImportAugmentation
|
||||||
// If we reach here, augmentation didn't process it - return minimal result
|
// If we reach here, augmentation didn't process it - return minimal result
|
||||||
const imageName = source.filename || 'image'
|
const imageName = source.filename || 'image'
|
||||||
const imageId = `image-${Date.now()}`
|
const imageId = `image-${Date.now()}`
|
||||||
|
|
@ -867,7 +867,7 @@ export class ImportCoordinator {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create entities and relationships in knowledge graph
|
* Create entities and relationships in knowledge graph
|
||||||
* v4.9.0: Added sourceInfo parameter for document entity creation
|
* Added sourceInfo parameter for document entity creation
|
||||||
*/
|
*/
|
||||||
private async createGraphEntities(
|
private async createGraphEntities(
|
||||||
extractionResult: any,
|
extractionResult: any,
|
||||||
|
|
@ -877,7 +877,7 @@ export class ImportCoordinator {
|
||||||
sourceFilename: string
|
sourceFilename: string
|
||||||
format: string
|
format: string
|
||||||
},
|
},
|
||||||
trackingContext?: TrackingContext // v4.10.0: Import/project tracking
|
trackingContext?: TrackingContext // Import/project tracking
|
||||||
): Promise<{
|
): Promise<{
|
||||||
entities: Array<{ id: string; name: string; type: NounType; vfsPath?: string; metadata?: Record<string, any> }>
|
entities: Array<{ id: string; name: string; type: NounType; vfsPath?: string; metadata?: Record<string, any> }>
|
||||||
relationships: Array<{ id: string; from: string; to: string; type: VerbType }>
|
relationships: Array<{ id: string; from: string; to: string; type: VerbType }>
|
||||||
|
|
@ -891,7 +891,7 @@ export class ImportCoordinator {
|
||||||
let mergedCount = 0
|
let mergedCount = 0
|
||||||
let newCount = 0
|
let newCount = 0
|
||||||
|
|
||||||
// CRITICAL FIX (v4.3.2): Default to true when undefined
|
// CRITICAL FIX: Default to true when undefined
|
||||||
// Previously: if (!options.createEntities) treated undefined as false
|
// Previously: if (!options.createEntities) treated undefined as false
|
||||||
// Now: Only skip when explicitly set to false
|
// Now: Only skip when explicitly set to false
|
||||||
if (options.createEntities === false) {
|
if (options.createEntities === false) {
|
||||||
|
|
@ -908,7 +908,7 @@ export class ImportCoordinator {
|
||||||
// Extract rows/sections/entities from result (unified across formats)
|
// Extract rows/sections/entities from result (unified across formats)
|
||||||
const rows = extractionResult.rows || extractionResult.sections || extractionResult.entities || []
|
const rows = extractionResult.rows || extractionResult.sections || extractionResult.entities || []
|
||||||
|
|
||||||
// Progressive flush interval - adjusts based on current count (v4.2.0+)
|
// Progressive flush interval - adjusts based on current count
|
||||||
// Starts at 100, increases to 1000 at 1K entities, then 5000 at 10K
|
// Starts at 100, increases to 1000 at 1K entities, then 5000 at 10K
|
||||||
// This works for both known totals (files) and unknown totals (streaming APIs)
|
// This works for both known totals (files) and unknown totals (streaming APIs)
|
||||||
let currentFlushInterval = 100 // Start with frequent updates for better UX
|
let currentFlushInterval = 100 // Start with frequent updates for better UX
|
||||||
|
|
@ -938,7 +938,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// v4.9.0: Create document entity for import source
|
// Create document entity for import source
|
||||||
// ============================================
|
// ============================================
|
||||||
let documentEntityId: string | null = null
|
let documentEntityId: string | null = null
|
||||||
let provenanceCount = 0
|
let provenanceCount = 0
|
||||||
|
|
@ -957,7 +957,7 @@ export class ImportCoordinator {
|
||||||
vfsPath: vfsResult.rootPath,
|
vfsPath: vfsResult.rootPath,
|
||||||
totalRows: rows.length,
|
totalRows: rows.length,
|
||||||
byType: this.countByType(rows),
|
byType: this.countByType(rows),
|
||||||
// v4.10.0: Import tracking metadata
|
// Import tracking metadata
|
||||||
...(trackingContext && {
|
...(trackingContext && {
|
||||||
importIds: [trackingContext.importId],
|
importIds: [trackingContext.importId],
|
||||||
projectId: trackingContext.projectId,
|
projectId: trackingContext.projectId,
|
||||||
|
|
@ -973,7 +973,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// v4.11.0: Batch entity creation using addMany()
|
// Batch entity creation using addMany()
|
||||||
// Replaces entity-by-entity loop for 10-100x performance improvement on cloud storage
|
// Replaces entity-by-entity loop for 10-100x performance improvement on cloud storage
|
||||||
// ============================================
|
// ============================================
|
||||||
|
|
||||||
|
|
@ -1038,7 +1038,7 @@ export class ImportCoordinator {
|
||||||
name: entity.name,
|
name: entity.name,
|
||||||
type: entity.type,
|
type: entity.type,
|
||||||
vfsPath: vfsFile?.path,
|
vfsPath: vfsFile?.path,
|
||||||
metadata: entity.metadata // v5.2.0: Include metadata in return (for ImageHandler, etc)
|
metadata: entity.metadata // Include metadata in return (for ImageHandler, etc)
|
||||||
})
|
})
|
||||||
newCount++
|
newCount++
|
||||||
}
|
}
|
||||||
|
|
@ -1090,7 +1090,7 @@ export class ImportCoordinator {
|
||||||
const importSource = vfsResult.rootPath
|
const importSource = vfsResult.rootPath
|
||||||
let entityId: string
|
let entityId: string
|
||||||
|
|
||||||
// v5.7.0: No deduplication during import (12-24x speedup)
|
// No deduplication during import (12-24x speedup)
|
||||||
// Background deduplication runs 5 minutes after import completes
|
// Background deduplication runs 5 minutes after import completes
|
||||||
entityId = await this.brain.add({
|
entityId = await this.brain.add({
|
||||||
data: entity.description || entity.name,
|
data: entity.description || entity.name,
|
||||||
|
|
@ -1101,7 +1101,7 @@ export class ImportCoordinator {
|
||||||
confidence: entity.confidence,
|
confidence: entity.confidence,
|
||||||
vfsPath: vfsFile?.path,
|
vfsPath: vfsFile?.path,
|
||||||
importedFrom: 'import-coordinator',
|
importedFrom: 'import-coordinator',
|
||||||
// v4.10.0: Import tracking metadata
|
// Import tracking metadata
|
||||||
...(trackingContext && {
|
...(trackingContext && {
|
||||||
importId: trackingContext.importId, // Used for background dedup
|
importId: trackingContext.importId, // Used for background dedup
|
||||||
importIds: [trackingContext.importId],
|
importIds: [trackingContext.importId],
|
||||||
|
|
@ -1126,11 +1126,11 @@ export class ImportCoordinator {
|
||||||
name: entity.name,
|
name: entity.name,
|
||||||
type: entity.type,
|
type: entity.type,
|
||||||
vfsPath: vfsFile?.path,
|
vfsPath: vfsFile?.path,
|
||||||
metadata: entity.metadata // v5.2.0: Include metadata in return (for ImageHandler, etc)
|
metadata: entity.metadata // Include metadata in return (for ImageHandler, etc)
|
||||||
})
|
})
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// v4.9.0: Create provenance relationship (document → entity)
|
// Create provenance relationship (document → entity)
|
||||||
// ============================================
|
// ============================================
|
||||||
if (documentEntityId && options.createProvenanceLinks !== false) {
|
if (documentEntityId && options.createProvenanceLinks !== false) {
|
||||||
await this.brain.relate({
|
await this.brain.relate({
|
||||||
|
|
@ -1144,7 +1144,7 @@ export class ImportCoordinator {
|
||||||
rowNumber: row.rowNumber,
|
rowNumber: row.rowNumber,
|
||||||
extractedAt: Date.now(),
|
extractedAt: Date.now(),
|
||||||
format: sourceInfo?.format,
|
format: sourceInfo?.format,
|
||||||
// v4.10.0: Import tracking metadata
|
// Import tracking metadata
|
||||||
...(trackingContext && {
|
...(trackingContext && {
|
||||||
importIds: [trackingContext.importId],
|
importIds: [trackingContext.importId],
|
||||||
projectId: trackingContext.projectId,
|
projectId: trackingContext.projectId,
|
||||||
|
|
@ -1161,7 +1161,7 @@ export class ImportCoordinator {
|
||||||
if (options.createRelationships && row.relationships) {
|
if (options.createRelationships && row.relationships) {
|
||||||
for (const rel of row.relationships) {
|
for (const rel of row.relationships) {
|
||||||
try {
|
try {
|
||||||
// CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop
|
// CRITICAL FIX: Prevent infinite placeholder creation loop
|
||||||
// Find or create target entity using EXACT matching only
|
// Find or create target entity using EXACT matching only
|
||||||
let targetEntityId: string | undefined
|
let targetEntityId: string | undefined
|
||||||
|
|
||||||
|
|
@ -1195,7 +1195,7 @@ export class ImportCoordinator {
|
||||||
name: rel.to,
|
name: rel.to,
|
||||||
placeholder: true,
|
placeholder: true,
|
||||||
inferredFrom: entity.name,
|
inferredFrom: entity.name,
|
||||||
// v4.10.0: Import tracking metadata
|
// Import tracking metadata
|
||||||
...(trackingContext && {
|
...(trackingContext && {
|
||||||
importIds: [trackingContext.importId],
|
importIds: [trackingContext.importId],
|
||||||
projectId: trackingContext.projectId,
|
projectId: trackingContext.projectId,
|
||||||
|
|
@ -1221,11 +1221,11 @@ export class ImportCoordinator {
|
||||||
from: entityId,
|
from: entityId,
|
||||||
to: targetEntityId,
|
to: targetEntityId,
|
||||||
type: rel.type,
|
type: rel.type,
|
||||||
confidence: rel.confidence, // v4.2.0: Top-level field
|
confidence: rel.confidence, // Top-level field
|
||||||
weight: rel.weight || 1.0, // v4.2.0: Top-level field
|
weight: rel.weight || 1.0, // Top-level field
|
||||||
metadata: {
|
metadata: {
|
||||||
evidence: rel.evidence,
|
evidence: rel.evidence,
|
||||||
// v4.10.0: Import tracking metadata (will be merged in batch creation)
|
// Import tracking metadata (will be merged in batch creation)
|
||||||
...(trackingContext && {
|
...(trackingContext && {
|
||||||
importIds: [trackingContext.importId],
|
importIds: [trackingContext.importId],
|
||||||
projectId: trackingContext.projectId,
|
projectId: trackingContext.projectId,
|
||||||
|
|
@ -1242,7 +1242,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Streaming import: Progressive flush with dynamic interval adjustment (v4.2.0+)
|
// Streaming import: Progressive flush with dynamic interval adjustment
|
||||||
entitiesSinceFlush++
|
entitiesSinceFlush++
|
||||||
|
|
||||||
if (entitiesSinceFlush >= currentFlushInterval) {
|
if (entitiesSinceFlush >= currentFlushInterval) {
|
||||||
|
|
@ -1307,7 +1307,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch create all relationships using brain.relateMany() for performance
|
// Batch create all relationships using brain.relateMany() for performance
|
||||||
// v4.9.0: Enhanced with type-based inference and semantic metadata
|
// Enhanced with type-based inference and semantic metadata
|
||||||
if (options.createRelationships && relationships.length > 0) {
|
if (options.createRelationships && relationships.length > 0) {
|
||||||
try {
|
try {
|
||||||
const relationshipParams = relationships.map(rel => {
|
const relationshipParams = relationships.map(rel => {
|
||||||
|
|
@ -1331,7 +1331,7 @@ export class ImportCoordinator {
|
||||||
type: verbType, // Enhanced type
|
type: verbType, // Enhanced type
|
||||||
metadata: {
|
metadata: {
|
||||||
...((rel as any).metadata || {}),
|
...((rel as any).metadata || {}),
|
||||||
relationshipType: 'semantic', // v4.9.0: Distinguish from VFS/provenance
|
relationshipType: 'semantic', // Distinguish from VFS/provenance
|
||||||
inferredType: verbType !== rel.type, // Track if type was enhanced
|
inferredType: verbType !== rel.type, // Track if type was enhanced
|
||||||
originalType: rel.type
|
originalType: rel.type
|
||||||
}
|
}
|
||||||
|
|
@ -1369,7 +1369,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.7.0: Schedule background deduplication (debounced 5 minutes)
|
// Schedule background deduplication (debounced 5 minutes)
|
||||||
if (trackingContext && trackingContext.importId) {
|
if (trackingContext && trackingContext.importId) {
|
||||||
this.backgroundDedup.scheduleDedup(trackingContext.importId)
|
this.backgroundDedup.scheduleDedup(trackingContext.importId)
|
||||||
}
|
}
|
||||||
|
|
@ -1457,7 +1457,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// YAML: entities -> rows (v4.2.0)
|
// YAML: entities -> rows
|
||||||
if (format === 'yaml') {
|
if (format === 'yaml') {
|
||||||
const rows = result.entities.map((entity: any) => ({
|
const rows = result.entities.map((entity: any) => ({
|
||||||
entity,
|
entity,
|
||||||
|
|
@ -1477,7 +1477,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DOCX: entities -> rows (v4.2.0)
|
// DOCX: entities -> rows
|
||||||
if (format === 'docx') {
|
if (format === 'docx') {
|
||||||
const rows = result.entities.map((entity: any) => ({
|
const rows = result.entities.map((entity: any) => ({
|
||||||
entity,
|
entity,
|
||||||
|
|
@ -1502,7 +1502,7 @@ export class ImportCoordinator {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validate options and reject deprecated v3.x options (v4.0.0+)
|
* Validate options and reject deprecated v3.x options
|
||||||
* Throws clear errors with migration guidance
|
* Throws clear errors with migration guidance
|
||||||
*/
|
*/
|
||||||
private validateOptions(options: any): void {
|
private validateOptions(options: any): void {
|
||||||
|
|
@ -1657,7 +1657,7 @@ ${optionDetails}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get progressive flush interval based on CURRENT entity count (v4.2.0+)
|
* Get progressive flush interval based on CURRENT entity count
|
||||||
*
|
*
|
||||||
* Unlike adaptive intervals (which require knowing total count upfront),
|
* Unlike adaptive intervals (which require knowing total count upfront),
|
||||||
* progressive intervals adjust dynamically as import proceeds.
|
* progressive intervals adjust dynamically as import proceeds.
|
||||||
|
|
@ -1699,7 +1699,7 @@ ${optionDetails}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Infer relationship type based on entity types and context
|
* Infer relationship type based on entity types and context
|
||||||
* v4.9.0: Semantic relationship enhancement
|
* Semantic relationship enhancement
|
||||||
*
|
*
|
||||||
* @param sourceType - Type of source entity
|
* @param sourceType - Type of source entity
|
||||||
* @param targetType - Type of target entity
|
* @param targetType - Type of target entity
|
||||||
|
|
@ -1769,7 +1769,7 @@ ${optionDetails}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Count entities by type for document metadata
|
* Count entities by type for document metadata
|
||||||
* v4.9.0: Used for document entity statistics
|
* Used for document entity statistics
|
||||||
*
|
*
|
||||||
* @param rows - Extracted rows from import
|
* @param rows - Extracted rows from import
|
||||||
* @returns Record of entity type counts
|
* @returns Record of entity type counts
|
||||||
|
|
|
||||||
|
|
@ -42,17 +42,17 @@ export interface SmartCSVOptions extends FormatHandlerOptions {
|
||||||
csvDelimiter?: string
|
csvDelimiter?: string
|
||||||
csvHeaders?: boolean
|
csvHeaders?: boolean
|
||||||
|
|
||||||
/** Progress callback (v3.39.0: Enhanced with performance metrics) */
|
/** Progress callback (Enhanced with performance metrics) */
|
||||||
onProgress?: (stats: {
|
onProgress?: (stats: {
|
||||||
processed: number
|
processed: number
|
||||||
total: number
|
total: number
|
||||||
entities: number
|
entities: number
|
||||||
relationships: number
|
relationships: number
|
||||||
/** Rows per second (v3.39.0) */
|
/** Rows per second */
|
||||||
throughput?: number
|
throughput?: number
|
||||||
/** Estimated time remaining in ms (v3.39.0) */
|
/** Estimated time remaining in ms */
|
||||||
eta?: number
|
eta?: number
|
||||||
/** Current phase (v3.39.0) */
|
/** Current phase */
|
||||||
phase?: string
|
phase?: string
|
||||||
}) => void
|
}) => void
|
||||||
}
|
}
|
||||||
|
|
@ -169,7 +169,7 @@ export class SmartCSVImporter {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse CSV using existing handler
|
// Parse CSV using existing handler
|
||||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
// Pass progress hooks to handler for file parsing progress
|
||||||
const processedData = await this.csvHandler.process(buffer, {
|
const processedData = await this.csvHandler.process(buffer, {
|
||||||
...options,
|
...options,
|
||||||
csvDelimiter: opts.csvDelimiter,
|
csvDelimiter: opts.csvDelimiter,
|
||||||
|
|
@ -217,7 +217,7 @@ export class SmartCSVImporter {
|
||||||
// Detect column names
|
// Detect column names
|
||||||
const columns = this.detectColumns(rows[0], opts)
|
const columns = this.detectColumns(rows[0], opts)
|
||||||
|
|
||||||
// Process each row with BATCHED PARALLEL PROCESSING (v3.39.0)
|
// Process each row with BATCHED PARALLEL PROCESSING
|
||||||
const extractedRows: ExtractedRow[] = []
|
const extractedRows: ExtractedRow[] = []
|
||||||
const entityMap = new Map<string, string>()
|
const entityMap = new Map<string, string>()
|
||||||
const stats = {
|
const stats = {
|
||||||
|
|
@ -375,7 +375,7 @@ export class SmartCSVImporter {
|
||||||
total: rows.length,
|
total: rows.length,
|
||||||
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
|
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
|
||||||
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
|
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
|
||||||
// Additional performance metrics (v3.39.0)
|
// Additional performance metrics
|
||||||
throughput: Math.round(rowsPerSecond * 10) / 10,
|
throughput: Math.round(rowsPerSecond * 10) / 10,
|
||||||
eta: Math.round(estimatedTimeRemaining),
|
eta: Math.round(estimatedTimeRemaining),
|
||||||
phase: 'extracting'
|
phase: 'extracting'
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
* - NaturalLanguageProcessor for relationship inference
|
* - NaturalLanguageProcessor for relationship inference
|
||||||
* - Hierarchical relationship creation based on heading hierarchy
|
* - Hierarchical relationship creation based on heading hierarchy
|
||||||
*
|
*
|
||||||
* v4.2.0: New format handler
|
* New format handler
|
||||||
* NO MOCKS - Production-ready implementation
|
* NO MOCKS - Production-ready implementation
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -187,7 +187,7 @@ export class SmartDOCXImporter {
|
||||||
await this.init()
|
await this.init()
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report parsing start
|
// Report parsing start
|
||||||
options.onProgress?.({
|
options.onProgress?.({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
entities: 0,
|
entities: 0,
|
||||||
|
|
@ -200,7 +200,7 @@ export class SmartDOCXImporter {
|
||||||
// Extract HTML for structure analysis (headings, tables)
|
// Extract HTML for structure analysis (headings, tables)
|
||||||
const htmlResult = await mammoth.convertToHtml({ buffer })
|
const htmlResult = await mammoth.convertToHtml({ buffer })
|
||||||
|
|
||||||
// v4.5.0: Report parsing complete
|
// Report parsing complete
|
||||||
options.onProgress?.({
|
options.onProgress?.({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
entities: 0,
|
entities: 0,
|
||||||
|
|
|
||||||
|
|
@ -36,17 +36,17 @@ export interface SmartExcelOptions extends FormatHandlerOptions {
|
||||||
typeColumn?: string // e.g., "Type", "Category"
|
typeColumn?: string // e.g., "Type", "Category"
|
||||||
relatedColumn?: string // e.g., "Related Terms", "See Also"
|
relatedColumn?: string // e.g., "Related Terms", "See Also"
|
||||||
|
|
||||||
/** Progress callback (v3.38.0: Enhanced with performance metrics) */
|
/** Progress callback (Enhanced with performance metrics) */
|
||||||
onProgress?: (stats: {
|
onProgress?: (stats: {
|
||||||
processed: number
|
processed: number
|
||||||
total: number
|
total: number
|
||||||
entities: number
|
entities: number
|
||||||
relationships: number
|
relationships: number
|
||||||
/** Rows per second (v3.38.0) */
|
/** Rows per second */
|
||||||
throughput?: number
|
throughput?: number
|
||||||
/** Estimated time remaining in ms (v3.38.0) */
|
/** Estimated time remaining in ms */
|
||||||
eta?: number
|
eta?: number
|
||||||
/** Current phase (v3.38.0) */
|
/** Current phase */
|
||||||
phase?: string
|
phase?: string
|
||||||
}) => void
|
}) => void
|
||||||
}
|
}
|
||||||
|
|
@ -111,7 +111,7 @@ export interface SmartExcelResult {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Sheet-specific data for VFS extraction (v4.2.0) */
|
/** Sheet-specific data for VFS extraction */
|
||||||
sheets?: Array<{
|
sheets?: Array<{
|
||||||
name: string
|
name: string
|
||||||
rows: ExtractedRow[]
|
rows: ExtractedRow[]
|
||||||
|
|
@ -161,7 +161,7 @@ export class SmartExcelImporter {
|
||||||
const opts = {
|
const opts = {
|
||||||
enableNeuralExtraction: true,
|
enableNeuralExtraction: true,
|
||||||
enableRelationshipInference: true,
|
enableRelationshipInference: true,
|
||||||
// CONCEPT EXTRACTION PRODUCTION-READY (v3.33.0+):
|
// CONCEPT EXTRACTION PRODUCTION-READY:
|
||||||
// Type embeddings are now pre-computed at build time - zero runtime cost!
|
// Type embeddings are now pre-computed at build time - zero runtime cost!
|
||||||
// All 42 noun types + 127 verb types instantly available
|
// All 42 noun types + 127 verb types instantly available
|
||||||
//
|
//
|
||||||
|
|
@ -184,7 +184,7 @@ export class SmartExcelImporter {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse Excel using existing handler
|
// Parse Excel using existing handler
|
||||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
// Pass progress hooks to handler for file parsing progress
|
||||||
const processedData = await this.excelHandler.process(buffer, {
|
const processedData = await this.excelHandler.process(buffer, {
|
||||||
...options,
|
...options,
|
||||||
totalBytes: buffer.length,
|
totalBytes: buffer.length,
|
||||||
|
|
@ -227,7 +227,7 @@ export class SmartExcelImporter {
|
||||||
return this.emptyResult(startTime)
|
return this.emptyResult(startTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CRITICAL FIX (v4.8.6): Detect columns per-sheet, not globally
|
// CRITICAL FIX: Detect columns per-sheet, not globally
|
||||||
// Different sheets may have different column structures (Term vs Name, etc.)
|
// Different sheets may have different column structures (Term vs Name, etc.)
|
||||||
// Group rows by sheet and detect columns for each sheet separately
|
// Group rows by sheet and detect columns for each sheet separately
|
||||||
const rowsBySheet = new Map<string, typeof rows>()
|
const rowsBySheet = new Map<string, typeof rows>()
|
||||||
|
|
@ -247,7 +247,7 @@ export class SmartExcelImporter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each row with BATCHED PARALLEL PROCESSING (v3.38.0)
|
// Process each row with BATCHED PARALLEL PROCESSING
|
||||||
const extractedRows: ExtractedRow[] = []
|
const extractedRows: ExtractedRow[] = []
|
||||||
const entityMap = new Map<string, string>()
|
const entityMap = new Map<string, string>()
|
||||||
const stats = {
|
const stats = {
|
||||||
|
|
@ -269,7 +269,7 @@ export class SmartExcelImporter {
|
||||||
chunk.map(async (row, chunkIndex) => {
|
chunk.map(async (row, chunkIndex) => {
|
||||||
const i = chunkStart + chunkIndex
|
const i = chunkStart + chunkIndex
|
||||||
|
|
||||||
// CRITICAL FIX (v4.8.6): Use sheet-specific column mapping
|
// CRITICAL FIX: Use sheet-specific column mapping
|
||||||
const sheet = row._sheet || 'default'
|
const sheet = row._sheet || 'default'
|
||||||
const columns = columnsBySheet.get(sheet) || this.detectColumns(row, opts)
|
const columns = columnsBySheet.get(sheet) || this.detectColumns(row, opts)
|
||||||
|
|
||||||
|
|
@ -353,7 +353,7 @@ export class SmartExcelImporter {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// v4.9.0: Enhanced column-based relationship detection
|
// Enhanced column-based relationship detection
|
||||||
// ============================================
|
// ============================================
|
||||||
// Parse explicit "Related Terms" column
|
// Parse explicit "Related Terms" column
|
||||||
if (relatedTerms) {
|
if (relatedTerms) {
|
||||||
|
|
@ -379,7 +379,7 @@ export class SmartExcelImporter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.9.0: Check for additional relationship-indicating columns
|
// Check for additional relationship-indicating columns
|
||||||
// Expanded patterns for various relationship types
|
// Expanded patterns for various relationship types
|
||||||
const relationshipColumnPatterns = [
|
const relationshipColumnPatterns = [
|
||||||
{ pattern: /^(location|home|lives in|resides|dwelling|place)$/i, defaultType: VerbType.LocatedAt },
|
{ pattern: /^(location|home|lives in|resides|dwelling|place)$/i, defaultType: VerbType.LocatedAt },
|
||||||
|
|
@ -485,14 +485,14 @@ export class SmartExcelImporter {
|
||||||
total: rows.length,
|
total: rows.length,
|
||||||
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
|
entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
|
||||||
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
|
relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
|
||||||
// Additional performance metrics (v3.38.0)
|
// Additional performance metrics
|
||||||
throughput: Math.round(rowsPerSecond * 10) / 10,
|
throughput: Math.round(rowsPerSecond * 10) / 10,
|
||||||
eta: Math.round(estimatedTimeRemaining),
|
eta: Math.round(estimatedTimeRemaining),
|
||||||
phase: 'extracting'
|
phase: 'extracting'
|
||||||
} as any)
|
} as any)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Group rows by sheet for VFS extraction (v4.2.0)
|
// Group rows by sheet for VFS extraction
|
||||||
const sheetGroups = new Map<string, ExtractedRow[]>()
|
const sheetGroups = new Map<string, ExtractedRow[]>()
|
||||||
extractedRows.forEach((extractedRow, index) => {
|
extractedRows.forEach((extractedRow, index) => {
|
||||||
const originalRow = rows[index]
|
const originalRow = rows[index]
|
||||||
|
|
|
||||||
|
|
@ -167,7 +167,7 @@ export class SmartJSONImporter {
|
||||||
...options
|
...options
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report parsing start
|
// Report parsing start
|
||||||
opts.onProgress({
|
opts.onProgress({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
entities: 0,
|
entities: 0,
|
||||||
|
|
@ -186,7 +186,7 @@ export class SmartJSONImporter {
|
||||||
jsonData = data
|
jsonData = data
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report parsing complete, starting traversal
|
// Report parsing complete, starting traversal
|
||||||
opts.onProgress({
|
opts.onProgress({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
entities: 0,
|
entities: 0,
|
||||||
|
|
@ -228,7 +228,7 @@ export class SmartJSONImporter {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// v4.5.0: Report completion
|
// Report completion
|
||||||
opts.onProgress({
|
opts.onProgress({
|
||||||
processed: nodesProcessed,
|
processed: nodesProcessed,
|
||||||
entities: entities.length,
|
entities: entities.length,
|
||||||
|
|
|
||||||
|
|
@ -174,7 +174,7 @@ export class SmartMarkdownImporter {
|
||||||
...options
|
...options
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report parsing start
|
// Report parsing start
|
||||||
opts.onProgress({
|
opts.onProgress({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
total: 0,
|
total: 0,
|
||||||
|
|
@ -185,7 +185,7 @@ export class SmartMarkdownImporter {
|
||||||
// Parse markdown into sections
|
// Parse markdown into sections
|
||||||
const parsedSections = this.parseMarkdown(markdown, opts)
|
const parsedSections = this.parseMarkdown(markdown, opts)
|
||||||
|
|
||||||
// v4.5.0: Report parsing complete
|
// Report parsing complete
|
||||||
opts.onProgress({
|
opts.onProgress({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
total: parsedSections.length,
|
total: parsedSections.length,
|
||||||
|
|
@ -218,7 +218,7 @@ export class SmartMarkdownImporter {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report completion
|
// Report completion
|
||||||
const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0)
|
const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0)
|
||||||
const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0)
|
const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0)
|
||||||
opts.onProgress({
|
opts.onProgress({
|
||||||
|
|
|
||||||
|
|
@ -40,17 +40,17 @@ export interface SmartPDFOptions extends FormatHandlerOptions {
|
||||||
/** Group by page or full document */
|
/** Group by page or full document */
|
||||||
groupBy?: 'page' | 'document'
|
groupBy?: 'page' | 'document'
|
||||||
|
|
||||||
/** Progress callback (v3.39.0: Enhanced with performance metrics) */
|
/** Progress callback (Enhanced with performance metrics) */
|
||||||
onProgress?: (stats: {
|
onProgress?: (stats: {
|
||||||
processed: number
|
processed: number
|
||||||
total: number
|
total: number
|
||||||
entities: number
|
entities: number
|
||||||
relationships: number
|
relationships: number
|
||||||
/** Sections per second (v3.39.0) */
|
/** Sections per second */
|
||||||
throughput?: number
|
throughput?: number
|
||||||
/** Estimated time remaining in ms (v3.39.0) */
|
/** Estimated time remaining in ms */
|
||||||
eta?: number
|
eta?: number
|
||||||
/** Current phase (v3.39.0) */
|
/** Current phase */
|
||||||
phase?: string
|
phase?: string
|
||||||
}) => void
|
}) => void
|
||||||
}
|
}
|
||||||
|
|
@ -181,7 +181,7 @@ export class SmartPDFImporter {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse PDF using existing handler
|
// Parse PDF using existing handler
|
||||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
// Pass progress hooks to handler for file parsing progress
|
||||||
const processedData = await this.pdfHandler.process(buffer, {
|
const processedData = await this.pdfHandler.process(buffer, {
|
||||||
...options,
|
...options,
|
||||||
totalBytes: buffer.length,
|
totalBytes: buffer.length,
|
||||||
|
|
@ -228,7 +228,7 @@ export class SmartPDFImporter {
|
||||||
// Group data by page or combine into single document
|
// Group data by page or combine into single document
|
||||||
const grouped = this.groupData(data, opts)
|
const grouped = this.groupData(data, opts)
|
||||||
|
|
||||||
// Process each group with BATCHED PARALLEL PROCESSING (v3.39.0)
|
// Process each group with BATCHED PARALLEL PROCESSING
|
||||||
const sections: ExtractedSection[] = []
|
const sections: ExtractedSection[] = []
|
||||||
const entityMap = new Map<string, string>()
|
const entityMap = new Map<string, string>()
|
||||||
const stats = {
|
const stats = {
|
||||||
|
|
@ -270,7 +270,7 @@ export class SmartPDFImporter {
|
||||||
total: totalGroups,
|
total: totalGroups,
|
||||||
entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
|
entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
|
||||||
relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0),
|
relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0),
|
||||||
// Additional performance metrics (v3.39.0)
|
// Additional performance metrics
|
||||||
throughput: Math.round(sectionsPerSecond * 10) / 10,
|
throughput: Math.round(sectionsPerSecond * 10) / 10,
|
||||||
eta: Math.round(estimatedTimeRemaining),
|
eta: Math.round(estimatedTimeRemaining),
|
||||||
phase: 'extracting'
|
phase: 'extracting'
|
||||||
|
|
@ -367,7 +367,7 @@ export class SmartPDFImporter {
|
||||||
|
|
||||||
const combinedText = texts.join('\n\n')
|
const combinedText = texts.join('\n\n')
|
||||||
|
|
||||||
// Parallel extraction: entities AND concepts at the same time (v3.39.0)
|
// Parallel extraction: entities AND concepts at the same time
|
||||||
const [extractedEntities, concepts] = await Promise.all([
|
const [extractedEntities, concepts] = await Promise.all([
|
||||||
// Extract entities if enabled
|
// Extract entities if enabled
|
||||||
options.enableNeuralExtraction && combinedText.length > 0
|
options.enableNeuralExtraction && combinedText.length > 0
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
* - NaturalLanguageProcessor for relationship inference
|
* - NaturalLanguageProcessor for relationship inference
|
||||||
* - Hierarchical relationship creation (parent-child, contains, etc.)
|
* - Hierarchical relationship creation (parent-child, contains, etc.)
|
||||||
*
|
*
|
||||||
* v4.2.0: New format handler
|
* New format handler
|
||||||
* NO MOCKS - Production-ready implementation
|
* NO MOCKS - Production-ready implementation
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -159,7 +159,7 @@ export class SmartYAMLImporter {
|
||||||
): Promise<SmartYAMLResult> {
|
): Promise<SmartYAMLResult> {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
||||||
// v4.5.0: Report parsing start
|
// Report parsing start
|
||||||
options.onProgress?.({
|
options.onProgress?.({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
entities: 0,
|
entities: 0,
|
||||||
|
|
@ -178,7 +178,7 @@ export class SmartYAMLImporter {
|
||||||
throw new Error(`Failed to parse YAML: ${error.message}`)
|
throw new Error(`Failed to parse YAML: ${error.message}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.5.0: Report parsing complete
|
// Report parsing complete
|
||||||
options.onProgress?.({
|
options.onProgress?.({
|
||||||
processed: 0,
|
processed: 0,
|
||||||
entities: 0,
|
entities: 0,
|
||||||
|
|
|
||||||
|
|
@ -40,10 +40,10 @@ export interface VFSStructureOptions {
|
||||||
/** Create metadata file */
|
/** Create metadata file */
|
||||||
createMetadataFile?: boolean
|
createMetadataFile?: boolean
|
||||||
|
|
||||||
/** Import tracking context (v4.10.0) */
|
/** Import tracking context */
|
||||||
trackingContext?: TrackingContext
|
trackingContext?: TrackingContext
|
||||||
|
|
||||||
/** Progress callback (v4.11.1) - Reports VFS creation progress */
|
/** Progress callback - Reports VFS creation progress */
|
||||||
onProgress?: (progress: {
|
onProgress?: (progress: {
|
||||||
stage: 'directories' | 'entities' | 'metadata'
|
stage: 'directories' | 'entities' | 'metadata'
|
||||||
message: string
|
message: string
|
||||||
|
|
@ -98,7 +98,7 @@ export class VFSStructureGenerator {
|
||||||
// Get brain's cached VFS instance (creates if doesn't exist)
|
// Get brain's cached VFS instance (creates if doesn't exist)
|
||||||
this.vfs = this.brain.vfs
|
this.vfs = this.brain.vfs
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.2): Always call vfs.init() explicitly
|
// CRITICAL FIX: Always call vfs.init() explicitly
|
||||||
// The previous code tried to check if initialized via stat('/') but this was unreliable
|
// The previous code tried to check if initialized via stat('/') but this was unreliable
|
||||||
// vfs.init() is idempotent, so calling it multiple times is safe
|
// vfs.init() is idempotent, so calling it multiple times is safe
|
||||||
await this.vfs.init()
|
await this.vfs.init()
|
||||||
|
|
@ -123,7 +123,7 @@ export class VFSStructureGenerator {
|
||||||
// Ensure VFS is initialized
|
// Ensure VFS is initialized
|
||||||
await this.init()
|
await this.init()
|
||||||
|
|
||||||
// v4.11.1: Calculate total operations for progress tracking
|
// Calculate total operations for progress tracking
|
||||||
const groups = this.groupEntities(importResult, options)
|
const groups = this.groupEntities(importResult, options)
|
||||||
const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0)
|
const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0)
|
||||||
const totalOperations =
|
const totalOperations =
|
||||||
|
|
@ -161,7 +161,7 @@ export class VFSStructureGenerator {
|
||||||
try {
|
try {
|
||||||
await this.vfs.mkdir(options.rootPath, {
|
await this.vfs.mkdir(options.rootPath, {
|
||||||
recursive: true,
|
recursive: true,
|
||||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
metadata: trackingMetadata // Add tracking metadata
|
||||||
})
|
})
|
||||||
result.directories.push(options.rootPath)
|
result.directories.push(options.rootPath)
|
||||||
result.operations++
|
result.operations++
|
||||||
|
|
@ -179,7 +179,7 @@ export class VFSStructureGenerator {
|
||||||
if (options.preserveSource && options.sourceBuffer && options.sourceFilename) {
|
if (options.preserveSource && options.sourceBuffer && options.sourceFilename) {
|
||||||
const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}`
|
const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}`
|
||||||
await this.vfs.writeFile(sourcePath, options.sourceBuffer, {
|
await this.vfs.writeFile(sourcePath, options.sourceBuffer, {
|
||||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
metadata: trackingMetadata // Add tracking metadata
|
||||||
})
|
})
|
||||||
result.files.push({
|
result.files.push({
|
||||||
path: sourcePath,
|
path: sourcePath,
|
||||||
|
|
@ -200,7 +200,7 @@ export class VFSStructureGenerator {
|
||||||
try {
|
try {
|
||||||
await this.vfs.mkdir(groupPath, {
|
await this.vfs.mkdir(groupPath, {
|
||||||
recursive: true,
|
recursive: true,
|
||||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
metadata: trackingMetadata // Add tracking metadata
|
||||||
})
|
})
|
||||||
result.directories.push(groupPath)
|
result.directories.push(groupPath)
|
||||||
result.operations++
|
result.operations++
|
||||||
|
|
@ -242,7 +242,7 @@ export class VFSStructureGenerator {
|
||||||
|
|
||||||
await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2), {
|
await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2), {
|
||||||
metadata: {
|
metadata: {
|
||||||
...trackingMetadata, // v4.10.0: Add tracking metadata
|
...trackingMetadata, // Add tracking metadata
|
||||||
entityId: extracted.entity.id
|
entityId: extracted.entity.id
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -253,7 +253,7 @@ export class VFSStructureGenerator {
|
||||||
})
|
})
|
||||||
result.operations++
|
result.operations++
|
||||||
|
|
||||||
// v4.11.1: Report progress every 10 entities (or on last entity)
|
// Report progress every 10 entities (or on last entity)
|
||||||
if (entityCount % 10 === 0 || entityCount === entities.length) {
|
if (entityCount % 10 === 0 || entityCount === entities.length) {
|
||||||
reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`)
|
reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`)
|
||||||
}
|
}
|
||||||
|
|
@ -280,7 +280,7 @@ export class VFSStructureGenerator {
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2), {
|
await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2), {
|
||||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
metadata: trackingMetadata // Add tracking metadata
|
||||||
})
|
})
|
||||||
result.files.push({
|
result.files.push({
|
||||||
path: relationshipsPath,
|
path: relationshipsPath,
|
||||||
|
|
@ -322,7 +322,7 @@ export class VFSStructureGenerator {
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2), {
|
await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2), {
|
||||||
metadata: trackingMetadata // v4.10.0: Add tracking metadata
|
metadata: trackingMetadata // Add tracking metadata
|
||||||
})
|
})
|
||||||
result.files.push({
|
result.files.push({
|
||||||
path: metadataPath,
|
path: metadataPath,
|
||||||
|
|
@ -332,7 +332,7 @@ export class VFSStructureGenerator {
|
||||||
reportProgress('metadata', 'Created metadata file')
|
reportProgress('metadata', 'Created metadata file')
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.11.1: Final progress update
|
// Final progress update
|
||||||
if (options.onProgress) {
|
if (options.onProgress) {
|
||||||
options.onProgress({
|
options.onProgress({
|
||||||
stage: 'metadata',
|
stage: 'metadata',
|
||||||
|
|
@ -355,7 +355,7 @@ export class VFSStructureGenerator {
|
||||||
): Map<string, typeof importResult.rows> {
|
): Map<string, typeof importResult.rows> {
|
||||||
const groups = new Map<string, typeof importResult.rows>()
|
const groups = new Map<string, typeof importResult.rows>()
|
||||||
|
|
||||||
// Handle sheet-based grouping (v4.2.0)
|
// Handle sheet-based grouping
|
||||||
if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) {
|
if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) {
|
||||||
for (const sheet of importResult.sheets) {
|
for (const sheet of importResult.sheets) {
|
||||||
groups.set(sheet.name, sheet.rows)
|
groups.set(sheet.name, sheet.rows)
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,7 @@ export type {
|
||||||
NeuralImportOptions
|
NeuralImportOptions
|
||||||
} from './cortex/neuralImport.js'
|
} from './cortex/neuralImport.js'
|
||||||
|
|
||||||
// Export Neural Entity Extraction (v5.7.6 - Workshop request)
|
// Export Neural Entity Extraction
|
||||||
export { NeuralEntityExtractor } from './neural/entityExtractor.js'
|
export { NeuralEntityExtractor } from './neural/entityExtractor.js'
|
||||||
export { SmartExtractor } from './neural/SmartExtractor.js'
|
export { SmartExtractor } from './neural/SmartExtractor.js'
|
||||||
export { SmartRelationshipExtractor } from './neural/SmartRelationshipExtractor.js'
|
export { SmartRelationshipExtractor } from './neural/SmartRelationshipExtractor.js'
|
||||||
|
|
@ -220,7 +220,7 @@ export {
|
||||||
// FileSystemStorage is exported separately to avoid browser build issues
|
// FileSystemStorage is exported separately to avoid browser build issues
|
||||||
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
|
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
|
||||||
|
|
||||||
// Export COW (Copy-on-Write) infrastructure for v5.0.0
|
// Export COW (Copy-on-Write) infrastructure
|
||||||
// Enables premium augmentations to implement temporal features
|
// Enables premium augmentations to implement temporal features
|
||||||
import { CommitLog } from './storage/cow/CommitLog.js'
|
import { CommitLog } from './storage/cow/CommitLog.js'
|
||||||
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
|
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
|
||||||
|
|
@ -543,7 +543,7 @@ export type {
|
||||||
MCPTool
|
MCPTool
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============= Integration Hub (v7.4.0) =============
|
// ============= Integration Hub =============
|
||||||
// Connect Brainy to Excel, Power BI, Google Sheets, and more
|
// Connect Brainy to Excel, Power BI, Google Sheets, and more
|
||||||
// Enable with: new Brainy({ integrations: true })
|
// Enable with: new Brainy({ integrations: true })
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
* - Real-time dashboards (SSE streaming)
|
* - Real-time dashboards (SSE streaming)
|
||||||
* - External webhooks (push notifications)
|
* - External webhooks (push notifications)
|
||||||
*
|
*
|
||||||
* @example Enable integrations (v7.4.0 - recommended)
|
* @example Enable integrations (recommended)
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* import { Brainy } from '@soulcraft/brainy'
|
* import { Brainy } from '@soulcraft/brainy'
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* Unified Index Interface (v3.35.0+)
|
* Unified Index Interface
|
||||||
*
|
*
|
||||||
* Standardizes index lifecycle across all index types in Brainy.
|
* Standardizes index lifecycle across all index types in Brainy.
|
||||||
* All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
|
* All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
|
||||||
|
|
@ -57,7 +57,7 @@ export interface RebuildOptions {
|
||||||
* - On-demand: Large datasets loaded adaptively via UnifiedCache
|
* - On-demand: Large datasets loaded adaptively via UnifiedCache
|
||||||
*
|
*
|
||||||
* This option is kept for backwards compatibility but is ignored.
|
* This option is kept for backwards compatibility but is ignored.
|
||||||
* The system always uses adaptive caching (v3.36.0+).
|
* The system always uses adaptive caching.
|
||||||
*/
|
*/
|
||||||
lazy?: boolean
|
lazy?: boolean
|
||||||
|
|
||||||
|
|
@ -104,7 +104,7 @@ export interface IIndex {
|
||||||
* - Provide progress reporting for large datasets
|
* - Provide progress reporting for large datasets
|
||||||
* - Recover gracefully from partial failures
|
* - Recover gracefully from partial failures
|
||||||
*
|
*
|
||||||
* Adaptive Caching (v3.36.0+):
|
* Adaptive Caching:
|
||||||
* System automatically chooses optimal strategy:
|
* System automatically chooses optimal strategy:
|
||||||
* - Small datasets: Preload all data at init for zero-latency access
|
* - Small datasets: Preload all data at init for zero-latency access
|
||||||
* - Large datasets: Load on-demand via UnifiedCache for memory efficiency
|
* - Large datasets: Load on-demand via UnifiedCache for memory efficiency
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
* Neural Entity Extractor using Brainy's NounTypes
|
* Neural Entity Extractor using Brainy's NounTypes
|
||||||
* Uses embeddings and similarity matching for accurate type detection
|
* Uses embeddings and similarity matching for accurate type detection
|
||||||
*
|
*
|
||||||
* v4.2.0: Now powered by SmartExtractor for ultra-neural classification
|
* Now powered by SmartExtractor for ultra-neural classification
|
||||||
* PRODUCTION-READY with caching support
|
* PRODUCTION-READY with caching support
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -24,7 +24,7 @@ export interface ExtractedEntity {
|
||||||
type: NounType
|
type: NounType
|
||||||
position: { start: number; end: number }
|
position: { start: number; end: number }
|
||||||
confidence: number
|
confidence: number
|
||||||
weight?: number // v4.2.0: Entity importance/salience
|
weight?: number // Entity importance/salience
|
||||||
vector?: Vector
|
vector?: Vector
|
||||||
metadata?: any
|
metadata?: any
|
||||||
}
|
}
|
||||||
|
|
@ -39,7 +39,7 @@ export class NeuralEntityExtractor {
|
||||||
// Entity extraction cache
|
// Entity extraction cache
|
||||||
private cache: EntityExtractionCache
|
private cache: EntityExtractionCache
|
||||||
|
|
||||||
// Runtime embedding cache for performance (v3.38.0)
|
// Runtime embedding cache for performance
|
||||||
// Caches candidate embeddings during an extraction session to avoid redundant model calls
|
// Caches candidate embeddings during an extraction session to avoid redundant model calls
|
||||||
private embeddingCache: Map<string, Vector> = new Map()
|
private embeddingCache: Map<string, Vector> = new Map()
|
||||||
private embeddingCacheStats = {
|
private embeddingCacheStats = {
|
||||||
|
|
@ -48,7 +48,7 @@ export class NeuralEntityExtractor {
|
||||||
size: 0
|
size: 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.2.0: SmartExtractor for ultra-neural classification
|
// SmartExtractor for ultra-neural classification
|
||||||
private smartExtractor: SmartExtractor
|
private smartExtractor: SmartExtractor
|
||||||
|
|
||||||
constructor(brain: Brainy | Brainy<any>, cacheOptions?: EntityCacheOptions) {
|
constructor(brain: Brainy | Brainy<any>, cacheOptions?: EntityCacheOptions) {
|
||||||
|
|
@ -63,7 +63,7 @@ export class NeuralEntityExtractor {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize type embeddings for neural matching
|
* Initialize type embeddings for neural matching
|
||||||
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed embeddings from build time
|
* PRODUCTION OPTIMIZATION: Uses pre-computed embeddings from build time
|
||||||
* Zero runtime cost - embeddings are loaded instantly from embedded data
|
* Zero runtime cost - embeddings are loaded instantly from embedded data
|
||||||
*/
|
*/
|
||||||
private async initializeTypeEmbeddings(requestedTypes?: NounType[]): Promise<void> {
|
private async initializeTypeEmbeddings(requestedTypes?: NounType[]): Promise<void> {
|
||||||
|
|
@ -109,7 +109,7 @@ export class NeuralEntityExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
): Promise<ExtractedEntity[]> {
|
): Promise<ExtractedEntity[]> {
|
||||||
// PRODUCTION OPTIMIZATION (v3.33.0): Load pre-computed type embeddings
|
// PRODUCTION OPTIMIZATION: Load pre-computed type embeddings
|
||||||
// Zero runtime cost - embeddings were computed at build time
|
// Zero runtime cost - embeddings were computed at build time
|
||||||
await this.initializeTypeEmbeddings(options?.types)
|
await this.initializeTypeEmbeddings(options?.types)
|
||||||
|
|
||||||
|
|
@ -138,7 +138,7 @@ export class NeuralEntityExtractor {
|
||||||
// Step 1: Extract potential entities using patterns
|
// Step 1: Extract potential entities using patterns
|
||||||
const candidates = await this.extractCandidates(text)
|
const candidates = await this.extractCandidates(text)
|
||||||
|
|
||||||
// Step 2: Classify each candidate using SmartExtractor (v4.2.0)
|
// Step 2: Classify each candidate using SmartExtractor
|
||||||
for (const candidate of candidates) {
|
for (const candidate of candidates) {
|
||||||
// Use SmartExtractor for unified neural + rule-based classification
|
// Use SmartExtractor for unified neural + rule-based classification
|
||||||
const classification = await this.smartExtractor.extract(candidate.text, {
|
const classification = await this.smartExtractor.extract(candidate.text, {
|
||||||
|
|
@ -357,7 +357,7 @@ export class NeuralEntityExtractor {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get embedding for text with caching (v3.38.0)
|
* Get embedding for text with caching
|
||||||
*
|
*
|
||||||
* PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
|
* PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
|
||||||
* to avoid redundant model calls for repeated text (common in large imports)
|
* to avoid redundant model calls for repeated text (common in large imports)
|
||||||
|
|
@ -509,7 +509,7 @@ export class NeuralEntityExtractor {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear embedding cache (v3.38.0)
|
* Clear embedding cache
|
||||||
*
|
*
|
||||||
* Clears the runtime embedding cache. Useful for:
|
* Clears the runtime embedding cache. Useful for:
|
||||||
* - Freeing memory after large imports
|
* - Freeing memory after large imports
|
||||||
|
|
@ -525,7 +525,7 @@ export class NeuralEntityExtractor {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get embedding cache statistics (v3.38.0)
|
* Get embedding cache statistics
|
||||||
*
|
*
|
||||||
* Returns performance metrics for the embedding cache:
|
* Returns performance metrics for the embedding cache:
|
||||||
* - hits: Number of cache hits (avoided model calls)
|
* - hits: Number of cache hits (avoided model calls)
|
||||||
|
|
|
||||||
|
|
@ -99,7 +99,7 @@ export class NaturalLanguageProcessor {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize embeddings for all NounTypes and VerbTypes
|
* Initialize embeddings for all NounTypes and VerbTypes
|
||||||
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings
|
* PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings
|
||||||
* Zero runtime cost - embeddings are loaded instantly from embedded data
|
* Zero runtime cost - embeddings are loaded instantly from embedded data
|
||||||
*/
|
*/
|
||||||
private async initializeTypeEmbeddings(): Promise<void> {
|
private async initializeTypeEmbeddings(): Promise<void> {
|
||||||
|
|
|
||||||
|
|
@ -122,7 +122,7 @@ export class PatternSignal {
|
||||||
])
|
])
|
||||||
|
|
||||||
// Location patterns - MEDIUM PRIORITY (city/country format - requires more context)
|
// Location patterns - MEDIUM PRIORITY (city/country format - requires more context)
|
||||||
// v4.11.2: Lower priority to avoid matching person names with commas
|
// Lower priority to avoid matching person names with commas
|
||||||
this.addPatterns(NounType.Location, 0.75, [
|
this.addPatterns(NounType.Location, 0.75, [
|
||||||
/\b[A-Z][a-z]+,\s*(?:Japan|China|France|Germany|Italy|Spain|Canada|Mexico|Brazil|India|Australia|Russia|UK|USA)\b/
|
/\b[A-Z][a-z]+,\s*(?:Japan|China|France|Germany|Italy|Spain|Canada|Mexico|Brazil|India|Australia|Russia|UK|USA)\b/
|
||||||
])
|
])
|
||||||
|
|
@ -170,7 +170,7 @@ export class PatternSignal {
|
||||||
// Technology patterns (Thing type)
|
// Technology patterns (Thing type)
|
||||||
this.addPatterns(NounType.Thing, 0.82, [
|
this.addPatterns(NounType.Thing, 0.82, [
|
||||||
/\b(?:JavaScript|TypeScript|Python|Java|Go|Rust|Swift|Kotlin)\b/,
|
/\b(?:JavaScript|TypeScript|Python|Java|Go|Rust|Swift|Kotlin)\b/,
|
||||||
/\bC\+\+(?!\w)/, // v4.11.2: Special handling for C++ (word boundary doesn't work with +)
|
/\bC\+\+(?!\w)/, // Special handling for C++ (word boundary doesn't work with +)
|
||||||
/\b(?:React|Vue|Angular|Node|Express|Django|Flask|Rails)\b/,
|
/\b(?:React|Vue|Angular|Node|Express|Django|Flask|Rails)\b/,
|
||||||
/\b(?:AWS|Azure|GCP|Docker|Kubernetes|Git|GitHub|GitLab)\b/,
|
/\b(?:AWS|Azure|GCP|Docker|Kubernetes|Git|GitHub|GitLab)\b/,
|
||||||
/\b(?:API|SDK|CLI|IDE|framework|library|package|module)\b/i,
|
/\b(?:API|SDK|CLI|IDE|framework|library|package|module)\b/i,
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* Brainy Setup - Minimal Polyfills
|
* Brainy Setup - Minimal Polyfills
|
||||||
*
|
*
|
||||||
* ARCHITECTURE (v7.0.0):
|
* ARCHITECTURE:
|
||||||
* Brainy uses Candle WASM (Rust-based) for embeddings.
|
* Brainy uses Candle WASM (Rust-based) for embeddings.
|
||||||
* No transformers.js or ONNX Runtime dependency, no hacks required.
|
* No transformers.js or ONNX Runtime dependency, no hacks required.
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
* 4. SAS Token
|
* 4. SAS Token
|
||||||
* 5. Azure AD (OAuth2) via DefaultAzureCredential
|
* 5. Azure AD (OAuth2) via DefaultAzureCredential
|
||||||
*
|
*
|
||||||
* v4.0.0: Fully compatible with metadata/vector separation architecture
|
* Fully compatible with metadata/vector separation architecture
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
|
@ -65,7 +65,7 @@ const MAX_AZURE_PAGE_SIZE = 5000
|
||||||
* 3. Storage Account Key - if accountName + accountKey provided
|
* 3. Storage Account Key - if accountName + accountKey provided
|
||||||
* 4. SAS Token - if accountName + sasToken provided
|
* 4. SAS Token - if accountName + sasToken provided
|
||||||
*
|
*
|
||||||
* v5.4.0: Type-aware storage now built into BaseStorage
|
* Type-aware storage now built into BaseStorage
|
||||||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||||||
* - Removed pagination overrides
|
* - Removed pagination overrides
|
||||||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
|
|
@ -105,7 +105,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
// Request coalescer for deduplication
|
// Request coalescer for deduplication
|
||||||
private requestCoalescer: RequestCoalescer | null = null
|
private requestCoalescer: RequestCoalescer | null = null
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled for consistent performance
|
// Write buffering always enabled for consistent performance
|
||||||
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
||||||
|
|
||||||
// Multi-level cache manager for efficient data access
|
// Multi-level cache manager for efficient data access
|
||||||
|
|
@ -115,7 +115,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
// Module logger
|
// Module logger
|
||||||
private logger = createModuleLogger('AzureBlobStorage')
|
private logger = createModuleLogger('AzureBlobStorage')
|
||||||
|
|
||||||
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
// HNSW mutex locks to prevent read-modify-write races
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -162,7 +162,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialization mode for fast cold starts (v7.3.0+)
|
* Initialization mode for fast cold starts
|
||||||
*
|
*
|
||||||
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
|
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
|
||||||
* strict locally. Zero-config optimization.
|
* strict locally. Zero-config optimization.
|
||||||
|
|
@ -171,7 +171,6 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
* - `'strict'`: Traditional blocking init. Validates container and loads counts
|
* - `'strict'`: Traditional blocking init. Validates container and loads counts
|
||||||
* before init() returns.
|
* before init() returns.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
initMode?: InitMode
|
initMode?: InitMode
|
||||||
|
|
||||||
|
|
@ -185,7 +184,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
this.sasToken = options.sasToken
|
this.sasToken = options.sasToken
|
||||||
this.readOnly = options.readOnly || false
|
this.readOnly = options.readOnly || false
|
||||||
|
|
||||||
// v7.3.0: Handle initMode
|
// Handle initMode
|
||||||
if (options.initMode) {
|
if (options.initMode) {
|
||||||
this.initMode = options.initMode
|
this.initMode = options.initMode
|
||||||
}
|
}
|
||||||
|
|
@ -201,7 +200,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
|
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
|
||||||
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
|
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled - no env var check needed
|
// Write buffering always enabled - no env var check needed
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -216,7 +215,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
* Recent Azure improvements make parallel downloads very efficient
|
* Recent Azure improvements make parallel downloads very efficient
|
||||||
*
|
*
|
||||||
* @returns Azure Blob-optimized batch configuration
|
* @returns Azure Blob-optimized batch configuration
|
||||||
* @since v5.12.0 - Updated for native batch API
|
* Updated for native batch API
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -241,7 +240,6 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
*
|
*
|
||||||
* @param paths - Array of Azure blob paths to read
|
* @param paths - Array of Azure blob paths to read
|
||||||
* @returns Map of path -> parsed JSON data (only successful reads)
|
* @returns Map of path -> parsed JSON data (only successful reads)
|
||||||
* @since v5.12.0
|
|
||||||
*/
|
*/
|
||||||
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -297,7 +295,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Initialize the storage adapter
|
* Initialize the storage adapter
|
||||||
*
|
*
|
||||||
* v7.3.0: Supports progressive initialization for fast cold starts
|
* Supports progressive initialization for fast cold starts
|
||||||
*
|
*
|
||||||
* | Mode | Init Time | When |
|
* | Mode | Init Time | When |
|
||||||
* |------|-----------|------|
|
* |------|-----------|------|
|
||||||
|
|
@ -404,7 +402,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
this.verbCacheManager.clear()
|
this.verbCacheManager.clear()
|
||||||
prodLog.info('✅ Cache cleared - starting fresh')
|
prodLog.info('✅ Cache cleared - starting fresh')
|
||||||
|
|
||||||
// v7.3.0: Progressive vs Strict initialization
|
// Progressive vs Strict initialization
|
||||||
if (effectiveMode === 'progressive') {
|
if (effectiveMode === 'progressive') {
|
||||||
// PROGRESSIVE MODE: Fast init, background validation
|
// PROGRESSIVE MODE: Fast init, background validation
|
||||||
// Mark as initialized immediately - ready to accept operations
|
// Mark as initialized immediately - ready to accept operations
|
||||||
|
|
@ -413,7 +411,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
|
|
||||||
prodLog.info(`✅ Azure progressive init complete: ${this.containerName} (validation deferred)`)
|
prodLog.info(`✅ Azure progressive init complete: ${this.containerName} (validation deferred)`)
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
|
|
||||||
// Schedule background tasks (non-blocking)
|
// Schedule background tasks (non-blocking)
|
||||||
|
|
@ -434,7 +432,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
await this.initializeCounts()
|
await this.initializeCounts()
|
||||||
this.countsLoaded = true
|
this.countsLoaded = true
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
|
|
||||||
// Mark background tasks as complete (nothing to do in background)
|
// Mark background tasks as complete (nothing to do in background)
|
||||||
|
|
@ -447,7 +445,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Progressive Initialization (v7.3.0+)
|
// Progressive Initialization
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -461,7 +459,6 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @override
|
* @override
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async runBackgroundInit(): Promise<void> {
|
protected async runBackgroundInit(): Promise<void> {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
@ -485,7 +482,6 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
* bucketValidated/bucketValidationError for lazy use.
|
* bucketValidated/bucketValidationError for lazy use.
|
||||||
*
|
*
|
||||||
* @private
|
* @private
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
private async validateContainerInBackground(): Promise<void> {
|
private async validateContainerInBackground(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|
@ -517,7 +513,6 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
* Load counts from storage in background.
|
* Load counts from storage in background.
|
||||||
*
|
*
|
||||||
* @private
|
* @private
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
private async loadCountsInBackground(): Promise<void> {
|
private async loadCountsInBackground(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|
@ -541,7 +536,6 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
* @throws Error if container validation fails
|
* @throws Error if container validation fails
|
||||||
* @protected
|
* @protected
|
||||||
* @override
|
* @override
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async ensureValidatedForWrite(): Promise<void> {
|
protected async ensureValidatedForWrite(): Promise<void> {
|
||||||
// If already validated, nothing to do
|
// If already validated, nothing to do
|
||||||
|
|
@ -665,7 +659,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush noun buffer to Azure
|
* Flush noun buffer to Azure
|
||||||
|
|
@ -697,20 +691,20 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
await Promise.all(writes)
|
await Promise.all(writes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
|
// Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save a node to storage
|
* Save a node to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.nounWriteBuffer) {
|
if (this.nounWriteBuffer) {
|
||||||
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
||||||
|
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
||||||
this.nounCacheManager.set(node.id, node)
|
this.nounCacheManager.set(node.id, node)
|
||||||
}
|
}
|
||||||
|
|
@ -791,7 +785,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
|
// Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a node from storage
|
* Get a node from storage
|
||||||
|
|
@ -889,18 +883,18 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
|
// Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write an object to a specific path in Azure
|
* Write an object to a specific path in Azure
|
||||||
* Primitive operation required by base class
|
* Primitive operation required by base class
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy container validation on first write in progressive mode.
|
* Performs lazy container validation on first write in progressive mode.
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
// v7.3.0: Lazy container validation for progressive init
|
// Lazy container validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -954,12 +948,12 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
* Delete an object from a specific path in Azure
|
* Delete an object from a specific path in Azure
|
||||||
* Primitive operation required by base class
|
* Primitive operation required by base class
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy container validation on first delete in progressive mode.
|
* Performs lazy container validation on first delete in progressive mode.
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
// v7.3.0: Lazy container validation for progressive init
|
// Lazy container validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -1209,20 +1203,20 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
|
// Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save an edge to storage
|
* Save an edge to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveEdge(edge: Edge): Promise<void> {
|
protected async saveEdge(edge: Edge): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.verbWriteBuffer) {
|
if (this.verbWriteBuffer) {
|
||||||
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
|
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
|
||||||
|
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
this.verbCacheManager.set(edge.id, edge)
|
this.verbCacheManager.set(edge.id, edge)
|
||||||
|
|
||||||
await this.verbWriteBuffer.add(edge.id, edge)
|
await this.verbWriteBuffer.add(edge.id, edge)
|
||||||
|
|
@ -1255,7 +1249,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
])
|
])
|
||||||
),
|
),
|
||||||
|
|
||||||
// CORE RELATIONAL DATA (v4.0.0)
|
// CORE RELATIONAL DATA
|
||||||
verb: edge.verb,
|
verb: edge.verb,
|
||||||
sourceId: edge.sourceId,
|
sourceId: edge.sourceId,
|
||||||
targetId: edge.targetId,
|
targetId: edge.targetId,
|
||||||
|
|
@ -1280,7 +1274,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
// Update cache
|
// Update cache
|
||||||
this.verbCacheManager.set(edge.id, edge)
|
this.verbCacheManager.set(edge.id, edge)
|
||||||
|
|
||||||
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
// Count tracking happens in baseStorage.saveVerbMetadata_internal
|
||||||
// This fixes the race condition where metadata didn't exist yet
|
// This fixes the race condition where metadata didn't exist yet
|
||||||
|
|
||||||
this.logger.trace(`Edge ${edge.id} saved successfully`)
|
this.logger.trace(`Edge ${edge.id} saved successfully`)
|
||||||
|
|
@ -1298,7 +1292,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
|
// Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get an edge from storage
|
* Get an edge from storage
|
||||||
|
|
@ -1335,7 +1329,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
connections.set(Number(level), new Set(verbIds as string[]))
|
connections.set(Number(level), new Set(verbIds as string[]))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
// Return HNSWVerb with core relational fields (NO metadata field)
|
||||||
const edge: Edge = {
|
const edge: Edge = {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
|
|
@ -1346,7 +1340,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
sourceId: data.sourceId,
|
sourceId: data.sourceId,
|
||||||
targetId: data.targetId
|
targetId: data.targetId
|
||||||
|
|
||||||
// ✅ NO metadata field in v4.0.0
|
// ✅ NO metadata field
|
||||||
// User metadata retrieved separately via getVerbMetadata()
|
// User metadata retrieved separately via getVerbMetadata()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1375,13 +1369,13 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
|
// Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation
|
// Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation
|
// Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation
|
// Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear all data from storage
|
* Clear all data from storage
|
||||||
|
|
@ -1393,7 +1387,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
this.logger.info('🧹 Clearing all data from Azure container...')
|
this.logger.info('🧹 Clearing all data from Azure container...')
|
||||||
|
|
||||||
// Delete all blobs in container
|
// Delete all blobs in container
|
||||||
// v5.6.1: listBlobsFlat() returns ALL blobs including _cow/ prefix
|
// listBlobsFlat() returns ALL blobs including _cow/ prefix
|
||||||
// This correctly deletes COW version control data (commits, trees, blobs, refs)
|
// This correctly deletes COW version control data (commits, trees, blobs, refs)
|
||||||
for await (const blob of this.containerClient!.listBlobsFlat()) {
|
for await (const blob of this.containerClient!.listBlobsFlat()) {
|
||||||
if (blob.name) {
|
if (blob.name) {
|
||||||
|
|
@ -1402,7 +1396,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
// Reset COW managers (but don't disable COW - it's always enabled)
|
||||||
// COW will re-initialize automatically on next use
|
// COW will re-initialize automatically on next use
|
||||||
this.refManager = undefined
|
this.refManager = undefined
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
|
|
@ -1461,12 +1455,12 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
* Fixes bug where clear() doesn't persist across instance restarts
|
||||||
* @returns true if marker blob exists, false otherwise
|
* @returns true if marker blob exists, false otherwise
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -1643,7 +1637,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun's vector for HNSW rebuild
|
* Get a noun's vector for HNSW rebuild
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getNounVector(id: string): Promise<number[] | null> {
|
public async getNounVector(id: string): Promise<number[] | null> {
|
||||||
const noun = await this.getNoun(id)
|
const noun = await this.getNoun(id)
|
||||||
|
|
@ -1653,7 +1647,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW graph data for a noun
|
* Save HNSW graph data for a noun
|
||||||
*
|
*
|
||||||
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
||||||
*/
|
*/
|
||||||
public async saveHNSWData(nounId: string, hnswData: {
|
public async saveHNSWData(nounId: string, hnswData: {
|
||||||
|
|
@ -1662,7 +1656,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const lockKey = `hnsw/${nounId}`
|
const lockKey = `hnsw/${nounId}`
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
|
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
|
||||||
// Problem: Without mutex, concurrent operations can:
|
// Problem: Without mutex, concurrent operations can:
|
||||||
// 1. Thread A reads noun (connections: [1,2,3])
|
// 1. Thread A reads noun (connections: [1,2,3])
|
||||||
// 2. Thread B reads noun (connections: [1,2,3])
|
// 2. Thread B reads noun (connections: [1,2,3])
|
||||||
|
|
@ -1682,7 +1676,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
this.hnswLocks.set(lockKey, lockPromise)
|
this.hnswLocks.set(lockKey, lockPromise)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
// Use BaseStorage's getNoun (type-first paths)
|
||||||
// Read existing noun data (if exists)
|
// Read existing noun data (if exists)
|
||||||
const existingNoun = await this.getNoun(nounId)
|
const existingNoun = await this.getNoun(nounId)
|
||||||
|
|
||||||
|
|
@ -1704,7 +1698,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
connections: connectionsMap
|
connections: connectionsMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
|
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
|
||||||
await this.saveNoun(updatedNoun)
|
await this.saveNoun(updatedNoun)
|
||||||
} finally {
|
} finally {
|
||||||
// Release lock (ALWAYS runs, even if error thrown)
|
// Release lock (ALWAYS runs, even if error thrown)
|
||||||
|
|
@ -1715,7 +1709,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get HNSW graph data for a noun
|
* Get HNSW graph data for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getHNSWData(nounId: string): Promise<{
|
public async getHNSWData(nounId: string): Promise<{
|
||||||
level: number
|
level: number
|
||||||
|
|
@ -1744,7 +1738,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW system data (entry point, max level)
|
* Save HNSW system data (entry point, max level)
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
|
* CRITICAL FIX: Optimistic locking with ETags to prevent race conditions
|
||||||
*/
|
*/
|
||||||
public async saveHNSWSystem(systemData: {
|
public async saveHNSWSystem(systemData: {
|
||||||
entryPointId: string | null
|
entryPointId: string | null
|
||||||
|
|
@ -1840,7 +1834,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the access tier for a specific blob (v4.0.0 cost optimization)
|
* Set the access tier for a specific blob (cost optimization)
|
||||||
* Azure Blob Storage tiers:
|
* Azure Blob Storage tiers:
|
||||||
* - Hot: $0.0184/GB/month - Frequently accessed data
|
* - Hot: $0.0184/GB/month - Frequently accessed data
|
||||||
* - Cool: $0.01/GB/month - Infrequently accessed data (45% cheaper)
|
* - Cool: $0.01/GB/month - Infrequently accessed data (45% cheaper)
|
||||||
|
|
@ -1906,7 +1900,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set access tier for multiple blobs in batch (v4.0.0 cost optimization)
|
* Set access tier for multiple blobs in batch (cost optimization)
|
||||||
* Efficiently move large numbers of blobs between tiers for cost optimization
|
* Efficiently move large numbers of blobs between tiers for cost optimization
|
||||||
*
|
*
|
||||||
* @param blobs - Array of blob names and their target tiers
|
* @param blobs - Array of blob names and their target tiers
|
||||||
|
|
@ -2128,7 +2122,7 @@ export class AzureBlobStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set lifecycle management policy for automatic tier transitions and deletions (v4.0.0)
|
* Set lifecycle management policy for automatic tier transitions and deletions
|
||||||
* Automates cost optimization by moving old data to cheaper tiers or deleting it
|
* Automates cost optimization by moving old data to cheaper tiers or deleting it
|
||||||
*
|
*
|
||||||
* Azure Lifecycle Management rules run once per day and apply to the entire container.
|
* Azure Lifecycle Management rules run once per day and apply to the entire container.
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/field
|
||||||
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
|
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// Progressive Initialization Types (v7.3.0+)
|
// Progressive Initialization Types
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -29,7 +29,6 @@ import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
|
||||||
* fast cold starts in serverless environments while maintaining strict
|
* fast cold starts in serverless environments while maintaining strict
|
||||||
* validation for local development.
|
* validation for local development.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*
|
*
|
||||||
* | Mode | Description | Use Case |
|
* | Mode | Description | Use Case |
|
||||||
* |------|-------------|----------|
|
* |------|-------------|----------|
|
||||||
|
|
@ -96,7 +95,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
|
|
||||||
abstract getVerbMetadata(id: string): Promise<any | null>
|
abstract getVerbMetadata(id: string): Promise<any | null>
|
||||||
|
|
||||||
// HNSW Index Persistence (v3.35.0+)
|
// HNSW Index Persistence
|
||||||
// These methods enable HNSW index rebuilding after container restarts
|
// These methods enable HNSW index rebuilding after container restarts
|
||||||
|
|
||||||
abstract getNounVector(id: string): Promise<number[] | null>
|
abstract getNounVector(id: string): Promise<number[] | null>
|
||||||
|
|
@ -139,7 +138,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* relateMany(), and import operations to automatically adapt to storage capabilities.
|
* relateMany(), and import operations to automatically adapt to storage capabilities.
|
||||||
*
|
*
|
||||||
* @returns Batch configuration optimized for this storage type
|
* @returns Batch configuration optimized for this storage type
|
||||||
* @since v4.11.0
|
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
// Conservative defaults that work safely across all storage types
|
// Conservative defaults that work safely across all storage types
|
||||||
|
|
@ -295,7 +293,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
}> = new Map()
|
}> = new Map()
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Progressive Initialization State (v7.3.0+)
|
// Progressive Initialization State
|
||||||
// =============================================
|
// =============================================
|
||||||
// These properties enable fast cold starts in cloud environments
|
// These properties enable fast cold starts in cloud environments
|
||||||
// by deferring validation and count loading to background tasks.
|
// by deferring validation and count loading to background tasks.
|
||||||
|
|
@ -308,7 +306,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* - `'strict'`: Always validate during init (traditional behavior)
|
* - `'strict'`: Always validate during init (traditional behavior)
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected initMode: InitMode = 'auto'
|
protected initMode: InitMode = 'auto'
|
||||||
|
|
||||||
|
|
@ -319,7 +316,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* after the first successful write operation or background validation.
|
* after the first successful write operation or background validation.
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected bucketValidated = false
|
protected bucketValidated = false
|
||||||
|
|
||||||
|
|
@ -329,7 +325,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* Stored here so subsequent operations can fail fast with the same error.
|
* Stored here so subsequent operations can fail fast with the same error.
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected bucketValidationError: Error | null = null
|
protected bucketValidationError: Error | null = null
|
||||||
|
|
||||||
|
|
@ -340,7 +335,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* Operations work immediately; counts become accurate after background load.
|
* Operations work immediately; counts become accurate after background load.
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected countsLoaded = false
|
protected countsLoaded = false
|
||||||
|
|
||||||
|
|
@ -350,7 +344,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* Useful for tests and diagnostics to ensure full initialization.
|
* Useful for tests and diagnostics to ensure full initialization.
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected backgroundTasksComplete = false
|
protected backgroundTasksComplete = false
|
||||||
|
|
||||||
|
|
@ -361,7 +354,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* before proceeding (e.g., in tests).
|
* before proceeding (e.g., in tests).
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected backgroundInitPromise: Promise<void> | null = null
|
protected backgroundInitPromise: Promise<void> | null = null
|
||||||
|
|
||||||
|
|
@ -1057,7 +1049,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
protected readonly COUNT_CACHE_TTL = 60000 // 1 minute cache TTL
|
protected readonly COUNT_CACHE_TTL = 60000 // 1 minute cache TTL
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Smart Count Batching (v3.32.3+)
|
// Smart Count Batching
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
// Count batching state - mirrors statistics batching pattern
|
// Count batching state - mirrors statistics batching pattern
|
||||||
|
|
@ -1112,7 +1104,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
const mutex = getGlobalMutex()
|
const mutex = getGlobalMutex()
|
||||||
await mutex.runExclusive(`count-entity-${type}`, async () => {
|
await mutex.runExclusive(`count-entity-${type}`, async () => {
|
||||||
this.incrementEntityCount(type)
|
this.incrementEntityCount(type)
|
||||||
// Smart batching (v3.32.3+): Adapts to storage type
|
// Smart batching: Adapts to storage type
|
||||||
// - Cloud storage (GCS, S3): Batches 10 ops OR 5 seconds
|
// - Cloud storage (GCS, S3): Batches 10 ops OR 5 seconds
|
||||||
// - Local storage (File, Memory): Persists immediately
|
// - Local storage (File, Memory): Persists immediately
|
||||||
await this.scheduleCountPersist()
|
await this.scheduleCountPersist()
|
||||||
|
|
@ -1147,13 +1139,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
const mutex = getGlobalMutex()
|
const mutex = getGlobalMutex()
|
||||||
await mutex.runExclusive(`count-entity-${type}`, async () => {
|
await mutex.runExclusive(`count-entity-${type}`, async () => {
|
||||||
this.decrementEntityCount(type)
|
this.decrementEntityCount(type)
|
||||||
// Smart batching (v3.32.3+): Adapts to storage type
|
// Smart batching: Adapts to storage type
|
||||||
await this.scheduleCountPersist()
|
await this.scheduleCountPersist()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment verb count - O(1) operation (v4.1.2: now synchronous)
|
* Increment verb count - O(1) operation (now synchronous)
|
||||||
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
|
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
|
||||||
* @param type The verb type
|
* @param type The verb type
|
||||||
*/
|
*/
|
||||||
|
|
@ -1168,7 +1160,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Thread-safe increment for verb counts (v4.1.2)
|
* Thread-safe increment for verb counts
|
||||||
* Uses mutex for single-node, distributed consensus for multi-node
|
* Uses mutex for single-node, distributed consensus for multi-node
|
||||||
* @param type The verb type
|
* @param type The verb type
|
||||||
*/
|
*/
|
||||||
|
|
@ -1176,13 +1168,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
const mutex = getGlobalMutex()
|
const mutex = getGlobalMutex()
|
||||||
await mutex.runExclusive(`count-verb-${type}`, async () => {
|
await mutex.runExclusive(`count-verb-${type}`, async () => {
|
||||||
this.incrementVerbCount(type)
|
this.incrementVerbCount(type)
|
||||||
// Smart batching (v3.32.3+): Adapts to storage type
|
// Smart batching: Adapts to storage type
|
||||||
await this.scheduleCountPersist()
|
await this.scheduleCountPersist()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Decrement verb count - O(1) operation (v4.1.2: now synchronous)
|
* Decrement verb count - O(1) operation (now synchronous)
|
||||||
* @param type The verb type
|
* @param type The verb type
|
||||||
*/
|
*/
|
||||||
protected decrementVerbCount(type: string): void {
|
protected decrementVerbCount(type: string): void {
|
||||||
|
|
@ -1203,20 +1195,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Thread-safe decrement for verb counts (v4.1.2)
|
* Thread-safe decrement for verb counts
|
||||||
* @param type The verb type
|
* @param type The verb type
|
||||||
*/
|
*/
|
||||||
protected async decrementVerbCountSafe(type: string): Promise<void> {
|
protected async decrementVerbCountSafe(type: string): Promise<void> {
|
||||||
const mutex = getGlobalMutex()
|
const mutex = getGlobalMutex()
|
||||||
await mutex.runExclusive(`count-verb-${type}`, async () => {
|
await mutex.runExclusive(`count-verb-${type}`, async () => {
|
||||||
this.decrementVerbCount(type)
|
this.decrementVerbCount(type)
|
||||||
// Smart batching (v3.32.3+): Adapts to storage type
|
// Smart batching: Adapts to storage type
|
||||||
await this.scheduleCountPersist()
|
await this.scheduleCountPersist()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Smart Batching Methods (v3.32.3+)
|
// Smart Batching Methods
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1235,7 +1227,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Progressive Initialization Methods (v7.3.0+)
|
// Progressive Initialization Methods
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1254,7 +1246,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
*
|
*
|
||||||
* @returns `true` if running in a detected cloud environment
|
* @returns `true` if running in a detected cloud environment
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected detectCloudEnvironment(): boolean {
|
protected detectCloudEnvironment(): boolean {
|
||||||
return !!(
|
return !!(
|
||||||
|
|
@ -1274,7 +1265,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
*
|
*
|
||||||
* @returns The resolved init mode (`'progressive'` or `'strict'`)
|
* @returns The resolved init mode (`'progressive'` or `'strict'`)
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected resolveInitMode(): 'progressive' | 'strict' {
|
protected resolveInitMode(): 'progressive' | 'strict' {
|
||||||
if (this.initMode === 'auto') {
|
if (this.initMode === 'auto') {
|
||||||
|
|
@ -1295,7 +1285,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* Always call `super.scheduleBackgroundInit()` first.
|
* Always call `super.scheduleBackgroundInit()` first.
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected scheduleBackgroundInit(): void {
|
protected scheduleBackgroundInit(): void {
|
||||||
// Create a promise that tracks all background work
|
// Create a promise that tracks all background work
|
||||||
|
|
@ -1320,7 +1309,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* The default implementation does nothing (for local storage adapters).
|
* The default implementation does nothing (for local storage adapters).
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async runBackgroundInit(): Promise<void> {
|
protected async runBackgroundInit(): Promise<void> {
|
||||||
// Default implementation: nothing to do for local storage
|
// Default implementation: nothing to do for local storage
|
||||||
|
|
@ -1348,7 +1336,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* ```
|
* ```
|
||||||
*
|
*
|
||||||
* @public
|
* @public
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
public async awaitBackgroundInit(): Promise<void> {
|
public async awaitBackgroundInit(): Promise<void> {
|
||||||
if (this.backgroundInitPromise) {
|
if (this.backgroundInitPromise) {
|
||||||
|
|
@ -1361,7 +1348,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
*
|
*
|
||||||
* @returns `true` if all background tasks are complete
|
* @returns `true` if all background tasks are complete
|
||||||
* @public
|
* @public
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
public isBackgroundInitComplete(): boolean {
|
public isBackgroundInitComplete(): boolean {
|
||||||
return this.backgroundTasksComplete
|
return this.backgroundTasksComplete
|
||||||
|
|
@ -1379,7 +1365,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
*
|
*
|
||||||
* @throws Error if bucket validation fails
|
* @throws Error if bucket validation fails
|
||||||
* @protected
|
* @protected
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async ensureValidatedForWrite(): Promise<void> {
|
protected async ensureValidatedForWrite(): Promise<void> {
|
||||||
// If already validated, nothing to do
|
// If already validated, nothing to do
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ try {
|
||||||
* File system storage adapter for Node.js environments
|
* File system storage adapter for Node.js environments
|
||||||
* Uses the file system to store data in the specified directory structure
|
* Uses the file system to store data in the specified directory structure
|
||||||
*
|
*
|
||||||
* v5.4.0: Type-aware storage now built into BaseStorage
|
* Type-aware storage now built into BaseStorage
|
||||||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||||||
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
||||||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
|
|
@ -91,12 +91,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
private lockTimers: Map<string, NodeJS.Timeout> = new Map() // Track timers for cleanup
|
private lockTimers: Map<string, NodeJS.Timeout> = new Map() // Track timers for cleanup
|
||||||
private allTimers: Set<NodeJS.Timeout> = new Set() // Track all timers for cleanup
|
private allTimers: Set<NodeJS.Timeout> = new Set() // Track all timers for cleanup
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
|
// CRITICAL FIX: Mutex locks for HNSW concurrency control
|
||||||
// Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops)
|
// Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops)
|
||||||
// Matches MemoryStorage and OPFSStorage behavior (tested in production)
|
// Matches MemoryStorage and OPFSStorage behavior (tested in production)
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
// Compression configuration (v4.0.0)
|
// Compression configuration
|
||||||
private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings
|
private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings
|
||||||
private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced)
|
private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced)
|
||||||
|
|
||||||
|
|
@ -136,7 +136,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* - Parallel processing supported
|
* - Parallel processing supported
|
||||||
*
|
*
|
||||||
* @returns FileSystem-optimized batch configuration
|
* @returns FileSystem-optimized batch configuration
|
||||||
* @since v4.11.0
|
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -179,7 +178,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Initialize directory paths now that path module is loaded
|
// Initialize directory paths now that path module is loaded
|
||||||
// Clean directory structure (v4.7.2+)
|
// Clean directory structure
|
||||||
this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw')
|
this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw')
|
||||||
this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw')
|
this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw')
|
||||||
this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference
|
this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference
|
||||||
|
|
@ -245,7 +244,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// Always use fixed depth after migration/detection
|
// Always use fixed depth after migration/detection
|
||||||
this.cachedShardingDepth = this.SHARDING_DEPTH
|
this.cachedShardingDepth = this.SHARDING_DEPTH
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error initializing FileSystemStorage:', error)
|
console.error('Error initializing FileSystemStorage:', error)
|
||||||
|
|
@ -281,7 +280,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save a node to storage
|
* Save a node to storage
|
||||||
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
|
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
|
||||||
*/
|
*/
|
||||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -303,7 +302,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// ATOMIC WRITE SEQUENCE (v4.10.3):
|
// ATOMIC WRITE SEQUENCE:
|
||||||
// 1. Write to temp file
|
// 1. Write to temp file
|
||||||
await this.ensureDirectoryExists(path.dirname(tempPath))
|
await this.ensureDirectoryExists(path.dirname(tempPath))
|
||||||
await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2))
|
await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2))
|
||||||
|
|
@ -320,7 +319,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
throw error
|
throw error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
|
// Count tracking happens in baseStorage.saveNounMetadata_internal
|
||||||
// This fixes the race condition where metadata didn't exist yet
|
// This fixes the race condition where metadata didn't exist yet
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -361,7 +360,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all nodes from storage
|
* Get all nodes from storage
|
||||||
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
|
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
|
||||||
* Previously only scanned flat directory, causing rebuild to find 0 entities
|
* Previously only scanned flat directory, causing rebuild to find 0 entities
|
||||||
*/
|
*/
|
||||||
protected async getAllNodes(): Promise<HNSWNode[]> {
|
protected async getAllNodes(): Promise<HNSWNode[]> {
|
||||||
|
|
@ -406,7 +405,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get nodes by noun type
|
* Get nodes by noun type
|
||||||
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
|
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
|
||||||
* @param nounType The noun type to filter by
|
* @param nounType The noun type to filter by
|
||||||
* @returns Promise that resolves to an array of nodes of the specified noun type
|
* @returns Promise that resolves to an array of nodes of the specified noun type
|
||||||
*/
|
*/
|
||||||
|
|
@ -462,7 +461,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
const filePath = this.getNodePath(id)
|
const filePath = this.getNodePath(id)
|
||||||
|
|
||||||
// Load metadata to get type for count update (v4.0.0: separate storage)
|
// Load metadata to get type for count update (separate storage)
|
||||||
try {
|
try {
|
||||||
const metadata = await this.getNounMetadata(id)
|
const metadata = await this.getNounMetadata(id)
|
||||||
if (metadata) {
|
if (metadata) {
|
||||||
|
|
@ -490,13 +489,13 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save an edge to storage
|
* Save an edge to storage
|
||||||
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
|
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
|
||||||
*/
|
*/
|
||||||
protected async saveEdge(edge: Edge): Promise<void> {
|
protected async saveEdge(edge: Edge): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// Convert connections Map to a serializable format
|
// Convert connections Map to a serializable format
|
||||||
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
|
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
|
||||||
// These fields are essential for 90% of operations - no metadata lookup needed
|
// These fields are essential for 90% of operations - no metadata lookup needed
|
||||||
const serializableEdge = {
|
const serializableEdge = {
|
||||||
id: edge.id,
|
id: edge.id,
|
||||||
|
|
@ -505,7 +504,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
Array.from(set as Set<string>)
|
Array.from(set as Set<string>)
|
||||||
),
|
),
|
||||||
|
|
||||||
// CORE RELATIONAL DATA (v3.50.1+)
|
// CORE RELATIONAL DATA
|
||||||
verb: edge.verb,
|
verb: edge.verb,
|
||||||
sourceId: edge.sourceId,
|
sourceId: edge.sourceId,
|
||||||
targetId: edge.targetId,
|
targetId: edge.targetId,
|
||||||
|
|
@ -518,7 +517,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// ATOMIC WRITE SEQUENCE (v4.10.3):
|
// ATOMIC WRITE SEQUENCE:
|
||||||
// 1. Write to temp file
|
// 1. Write to temp file
|
||||||
await this.ensureDirectoryExists(path.dirname(tempPath))
|
await this.ensureDirectoryExists(path.dirname(tempPath))
|
||||||
await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2))
|
await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2))
|
||||||
|
|
@ -535,7 +534,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
throw error
|
throw error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
// Count tracking happens in baseStorage.saveVerbMetadata_internal
|
||||||
// This fixes the race condition where metadata didn't exist yet
|
// This fixes the race condition where metadata didn't exist yet
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -556,7 +555,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
// Return HNSWVerb with core relational fields (NO metadata field)
|
||||||
return {
|
return {
|
||||||
id: parsedEdge.id,
|
id: parsedEdge.id,
|
||||||
vector: parsedEdge.vector,
|
vector: parsedEdge.vector,
|
||||||
|
|
@ -567,7 +566,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
sourceId: parsedEdge.sourceId,
|
sourceId: parsedEdge.sourceId,
|
||||||
targetId: parsedEdge.targetId
|
targetId: parsedEdge.targetId
|
||||||
|
|
||||||
// ✅ NO metadata field in v4.0.0
|
// ✅ NO metadata field
|
||||||
// User metadata retrieved separately via getVerbMetadata()
|
// User metadata retrieved separately via getVerbMetadata()
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
|
|
@ -580,7 +579,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all edges from storage
|
* Get all edges from storage
|
||||||
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
|
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
|
||||||
* Previously only scanned flat directory, causing rebuild to find 0 relationships
|
* Previously only scanned flat directory, causing rebuild to find 0 relationships
|
||||||
*/
|
*/
|
||||||
protected async getAllEdges(): Promise<Edge[]> {
|
protected async getAllEdges(): Promise<Edge[]> {
|
||||||
|
|
@ -608,7 +607,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
connections.set(Number(level), new Set(nodeIds as string[]))
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Include core relational fields (NO metadata field)
|
// Include core relational fields (NO metadata field)
|
||||||
allEdges.push({
|
allEdges.push({
|
||||||
id: parsedEdge.id,
|
id: parsedEdge.id,
|
||||||
vector: parsedEdge.vector,
|
vector: parsedEdge.vector,
|
||||||
|
|
@ -619,7 +618,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
sourceId: parsedEdge.sourceId,
|
sourceId: parsedEdge.sourceId,
|
||||||
targetId: parsedEdge.targetId
|
targetId: parsedEdge.targetId
|
||||||
|
|
||||||
// ✅ NO metadata field in v4.0.0
|
// ✅ NO metadata field
|
||||||
// User metadata retrieved separately via getVerbMetadata()
|
// User metadata retrieved separately via getVerbMetadata()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -696,8 +695,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Primitive operation: Write object to path
|
* Primitive operation: Write object to path
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
* v4.0.0: Supports gzip compression for 60-80% disk savings
|
* Supports gzip compression for 60-80% disk savings
|
||||||
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
|
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
|
||||||
*/
|
*/
|
||||||
protected async writeObjectToPath(pathStr: string, data: any): Promise<void> {
|
protected async writeObjectToPath(pathStr: string, data: any): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -711,7 +710,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
const tempPath = `${compressedPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
const tempPath = `${compressedPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// ATOMIC WRITE SEQUENCE (v4.10.3):
|
// ATOMIC WRITE SEQUENCE:
|
||||||
// 1. Compress and write to temp file
|
// 1. Compress and write to temp file
|
||||||
const jsonString = JSON.stringify(data, null, 2)
|
const jsonString = JSON.stringify(data, null, 2)
|
||||||
const compressed = await new Promise<Buffer>((resolve, reject) => {
|
const compressed = await new Promise<Buffer>((resolve, reject) => {
|
||||||
|
|
@ -748,7 +747,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
const tempPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
const tempPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// ATOMIC WRITE SEQUENCE (v4.10.3):
|
// ATOMIC WRITE SEQUENCE:
|
||||||
// 1. Write to temp file
|
// 1. Write to temp file
|
||||||
await fs.promises.writeFile(tempPath, JSON.stringify(data, null, 2))
|
await fs.promises.writeFile(tempPath, JSON.stringify(data, null, 2))
|
||||||
|
|
||||||
|
|
@ -770,7 +769,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* Primitive operation: Read object from path
|
* Primitive operation: Read object from path
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
* Enhanced error handling for corrupted metadata files (Bug #3 mitigation)
|
* Enhanced error handling for corrupted metadata files (Bug #3 mitigation)
|
||||||
* v4.0.0: Supports reading both compressed (.gz) and uncompressed files for backward compatibility
|
* Supports reading both compressed (.gz) and uncompressed files for backward compatibility
|
||||||
*/
|
*/
|
||||||
protected async readObjectFromPath(pathStr: string): Promise<any | null> {
|
protected async readObjectFromPath(pathStr: string): Promise<any | null> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -822,7 +821,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Primitive operation: Delete object from path
|
* Primitive operation: Delete object from path
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
* v4.0.0: Deletes both compressed and uncompressed versions (for cleanup)
|
* Deletes both compressed and uncompressed versions (for cleanup)
|
||||||
*/
|
*/
|
||||||
protected async deleteObjectFromPath(pathStr: string): Promise<void> {
|
protected async deleteObjectFromPath(pathStr: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -863,7 +862,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Primitive operation: List objects under path prefix
|
* Primitive operation: List objects under path prefix
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
* v4.0.0: Handles both .json and .json.gz files, normalizes paths
|
* Handles both .json and .json.gz files, normalizes paths
|
||||||
*/
|
*/
|
||||||
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
|
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -877,7 +876,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
for (const entry of entries) {
|
for (const entry of entries) {
|
||||||
if (entry.isFile()) {
|
if (entry.isFile()) {
|
||||||
// v5.3.5: Handle multiple compression formats for broad compatibility
|
// Handle multiple compression formats for broad compatibility
|
||||||
// - .json.gz: Standard entity/metadata files (JSON compressed)
|
// - .json.gz: Standard entity/metadata files (JSON compressed)
|
||||||
// - .gz: COW files (refs, blobs, commits - raw compressed)
|
// - .gz: COW files (refs, blobs, commits - raw compressed)
|
||||||
// - .json: Uncompressed JSON files
|
// - .json: Uncompressed JSON files
|
||||||
|
|
@ -890,7 +889,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
seen.add(normalizedPath)
|
seen.add(normalizedPath)
|
||||||
}
|
}
|
||||||
} else if (entry.name.endsWith('.gz')) {
|
} else if (entry.name.endsWith('.gz')) {
|
||||||
// v5.3.5 fix: COW files stored as .gz (not .json.gz)
|
// COW files stored as .gz (not .json.gz)
|
||||||
// Strip .gz extension and return path
|
// Strip .gz extension and return path
|
||||||
const normalizedName = entry.name.slice(0, -3) // Remove .gz
|
const normalizedName = entry.name.slice(0, -3) // Remove .gz
|
||||||
const normalizedPath = path.join(prefix, normalizedName)
|
const normalizedPath = path.join(prefix, normalizedName)
|
||||||
|
|
@ -966,7 +965,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* Get nouns with pagination support
|
* Get nouns with pagination support
|
||||||
* @param options Pagination options
|
* @param options Pagination options
|
||||||
*/
|
*/
|
||||||
// v5.4.0: Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
|
// Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear all data from storage
|
* Clear all data from storage
|
||||||
|
|
@ -1002,7 +1001,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.10.4: Clear the entire branches/ directory (branch-based storage)
|
// Clear the entire branches/ directory (branch-based storage)
|
||||||
// Bug fix: Data is stored in branches/main/entities/, not just entities/
|
// Bug fix: Data is stored in branches/main/entities/, not just entities/
|
||||||
// The branch-based structure was introduced for COW support
|
// The branch-based structure was introduced for COW support
|
||||||
const branchesDir = path.join(this.rootDir, 'branches')
|
const branchesDir = path.join(this.rootDir, 'branches')
|
||||||
|
|
@ -1016,7 +1015,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
await removeDirectoryContents(this.indexDir)
|
await removeDirectoryContents(this.indexDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.6.1: Remove COW (copy-on-write) version control data
|
// Remove COW (copy-on-write) version control data
|
||||||
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
|
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
|
||||||
// Must be deleted to fully clear all data including version history
|
// Must be deleted to fully clear all data including version history
|
||||||
const cowDir = path.join(this.rootDir, '_cow')
|
const cowDir = path.join(this.rootDir, '_cow')
|
||||||
|
|
@ -1024,7 +1023,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// Delete the entire _cow/ directory (not just contents)
|
// Delete the entire _cow/ directory (not just contents)
|
||||||
await fs.promises.rm(cowDir, { recursive: true, force: true })
|
await fs.promises.rm(cowDir, { recursive: true, force: true })
|
||||||
|
|
||||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
// Reset COW managers (but don't disable COW - it's always enabled)
|
||||||
// COW will re-initialize automatically on next use
|
// COW will re-initialize automatically on next use
|
||||||
this.refManager = undefined
|
this.refManager = undefined
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
|
|
@ -1035,12 +1034,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.statisticsCache = null
|
this.statisticsCache = null
|
||||||
this.statisticsModified = false
|
this.statisticsModified = false
|
||||||
|
|
||||||
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
|
// Reset entity counters (inherited from BaseStorageAdapter)
|
||||||
// These in-memory counters must be reset to 0 after clearing all data
|
// These in-memory counters must be reset to 0 after clearing all data
|
||||||
;(this as any).totalNounCount = 0
|
;(this as any).totalNounCount = 0
|
||||||
;(this as any).totalVerbCount = 0
|
;(this as any).totalVerbCount = 0
|
||||||
|
|
||||||
// v7.3.1: Clear write-through cache (inherited from BaseStorage)
|
// Clear write-through cache (inherited from BaseStorage)
|
||||||
// Without this, readWithInheritance() would return stale cached data
|
// Without this, readWithInheritance() would return stale cached data
|
||||||
// after clear(), causing "ghost" entities to appear
|
// after clear(), causing "ghost" entities to appear
|
||||||
this.clearWriteCache()
|
this.clearWriteCache()
|
||||||
|
|
@ -1074,12 +1073,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
* Fixes bug where clear() doesn't persist across instance restarts
|
||||||
* @returns true if marker file exists, false otherwise
|
* @returns true if marker file exists, false otherwise
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -1152,7 +1151,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
||||||
|
|
||||||
// CRITICAL FIX (v3.43.2): Use persisted counts instead of directory reads
|
// CRITICAL FIX: Use persisted counts instead of directory reads
|
||||||
// This is O(1) instead of O(n), and handles sharded structure correctly
|
// This is O(1) instead of O(n), and handles sharded structure correctly
|
||||||
const nounsCount = this.totalNounCount
|
const nounsCount = this.totalNounCount
|
||||||
const verbsCount = this.totalVerbCount
|
const verbsCount = this.totalVerbCount
|
||||||
|
|
@ -1210,8 +1209,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
||||||
// v5.4.0: Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
|
// Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Acquire a file-based lock for coordinating operations across multiple processes
|
* Acquire a file-based lock for coordinating operations across multiple processes
|
||||||
|
|
@ -1503,14 +1502,14 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalVerbCount = validVerbFiles.length
|
this.totalVerbCount = validVerbFiles.length
|
||||||
|
|
||||||
// Sample some files to get type distribution (don't read all)
|
// Sample some files to get type distribution (don't read all)
|
||||||
// v4.0.0: Load metadata separately for type information
|
// Load metadata separately for type information
|
||||||
const sampleSize = Math.min(100, validNounFiles.length)
|
const sampleSize = Math.min(100, validNounFiles.length)
|
||||||
for (let i = 0; i < sampleSize; i++) {
|
for (let i = 0; i < sampleSize; i++) {
|
||||||
try {
|
try {
|
||||||
const file = validNounFiles[i]
|
const file = validNounFiles[i]
|
||||||
const id = file.replace('.json', '')
|
const id = file.replace('.json', '')
|
||||||
|
|
||||||
// v4.0.0: Load metadata from separate storage for type info
|
// Load metadata from separate storage for type info
|
||||||
const metadata = await this.getNounMetadata(id)
|
const metadata = await this.getNounMetadata(id)
|
||||||
if (metadata) {
|
if (metadata) {
|
||||||
const type = metadata.noun || 'default'
|
const type = metadata.noun || 'default'
|
||||||
|
|
@ -2148,7 +2147,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
const edge = JSON.parse(data)
|
const edge = JSON.parse(data)
|
||||||
const metadata = await this.getVerbMetadata(id)
|
const metadata = await this.getVerbMetadata(id)
|
||||||
|
|
||||||
// v4.8.1: Don't skip verbs without metadata - metadata is optional
|
// Don't skip verbs without metadata - metadata is optional
|
||||||
// FIX: This was the root cause of the VFS bug (11 versions)
|
// FIX: This was the root cause of the VFS bug (11 versions)
|
||||||
// Verbs can exist without metadata files (e.g., from imports/migrations)
|
// Verbs can exist without metadata files (e.g., from imports/migrations)
|
||||||
|
|
||||||
|
|
@ -2162,7 +2161,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
connections = connectionsMap
|
connections = connectionsMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.8.0: Extract standard fields from metadata to top-level
|
// Extract standard fields from metadata to top-level
|
||||||
const metadataObj = (metadata || {}) as VerbMetadata
|
const metadataObj = (metadata || {}) as VerbMetadata
|
||||||
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj
|
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj
|
||||||
|
|
||||||
|
|
@ -2309,12 +2308,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// HNSW Index Persistence (v3.35.0+)
|
// HNSW Index Persistence
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get vector for a noun
|
* Get vector for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getNounVector(id: string): Promise<number[] | null> {
|
public async getNounVector(id: string): Promise<number[] | null> {
|
||||||
const noun = await this.getNoun(id)
|
const noun = await this.getNoun(id)
|
||||||
|
|
@ -2324,7 +2323,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW graph data for a noun
|
* Save HNSW graph data for a noun
|
||||||
*
|
*
|
||||||
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
||||||
*/
|
*/
|
||||||
public async saveHNSWData(nounId: string, hnswData: {
|
public async saveHNSWData(nounId: string, hnswData: {
|
||||||
|
|
@ -2333,7 +2332,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const lockKey = `hnsw/${nounId}`
|
const lockKey = `hnsw/${nounId}`
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
|
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
|
||||||
// Problem: Without mutex, concurrent operations can:
|
// Problem: Without mutex, concurrent operations can:
|
||||||
// 1. Thread A reads noun (connections: [1,2,3])
|
// 1. Thread A reads noun (connections: [1,2,3])
|
||||||
// 2. Thread B reads noun (connections: [1,2,3])
|
// 2. Thread B reads noun (connections: [1,2,3])
|
||||||
|
|
@ -2353,7 +2352,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.hnswLocks.set(lockKey, lockPromise)
|
this.hnswLocks.set(lockKey, lockPromise)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
// Use BaseStorage's getNoun (type-first paths)
|
||||||
// Read existing noun data (if exists)
|
// Read existing noun data (if exists)
|
||||||
const existingNoun = await this.getNoun(nounId)
|
const existingNoun = await this.getNoun(nounId)
|
||||||
|
|
||||||
|
|
@ -2375,7 +2374,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
connections: connectionsMap
|
connections: connectionsMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write)
|
// Use BaseStorage's saveNoun (type-first paths, atomic write)
|
||||||
await this.saveNoun(updatedNoun)
|
await this.saveNoun(updatedNoun)
|
||||||
} finally {
|
} finally {
|
||||||
// Release lock (ALWAYS runs, even if error thrown)
|
// Release lock (ALWAYS runs, even if error thrown)
|
||||||
|
|
@ -2386,7 +2385,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get HNSW graph data for a noun
|
* Get HNSW graph data for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getHNSWData(nounId: string): Promise<{
|
public async getHNSWData(nounId: string): Promise<{
|
||||||
level: number
|
level: number
|
||||||
|
|
@ -2415,7 +2414,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW system data (entry point, max level)
|
* Save HNSW system data (entry point, max level)
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Mutex lock + atomic write to prevent race conditions
|
* CRITICAL FIX: Mutex lock + atomic write to prevent race conditions
|
||||||
*/
|
*/
|
||||||
public async saveHNSWSystem(systemData: {
|
public async saveHNSWSystem(systemData: {
|
||||||
entryPointId: string | null
|
entryPointId: string | null
|
||||||
|
|
@ -2425,7 +2424,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
|
|
||||||
const lockKey = 'hnsw/system'
|
const lockKey = 'hnsw/system'
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex lock to serialize system updates
|
// CRITICAL FIX: Mutex lock to serialize system updates
|
||||||
// System data (entry point, max level) updated frequently during HNSW construction
|
// System data (entry point, max level) updated frequently during HNSW construction
|
||||||
// Without mutex, concurrent updates can lose data (same as entity-level problem)
|
// Without mutex, concurrent updates can lose data (same as entity-level problem)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@ const MAX_GCS_PAGE_SIZE = 5000
|
||||||
* 3. Service Account Credentials Object (if credentials provided)
|
* 3. Service Account Credentials Object (if credentials provided)
|
||||||
* 4. HMAC Keys (if accessKeyId/secretAccessKey provided)
|
* 4. HMAC Keys (if accessKeyId/secretAccessKey provided)
|
||||||
*
|
*
|
||||||
* v5.4.0: Type-aware storage now built into BaseStorage
|
* Type-aware storage now built into BaseStorage
|
||||||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||||||
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
||||||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
|
|
@ -110,7 +110,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
// Request coalescer for deduplication
|
// Request coalescer for deduplication
|
||||||
private requestCoalescer: RequestCoalescer | null = null
|
private requestCoalescer: RequestCoalescer | null = null
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled for consistent performance
|
// Write buffering always enabled for consistent performance
|
||||||
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
||||||
|
|
||||||
// Multi-level cache manager for efficient data access
|
// Multi-level cache manager for efficient data access
|
||||||
|
|
@ -120,7 +120,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
// Module logger
|
// Module logger
|
||||||
private logger = createModuleLogger('GcsStorage')
|
private logger = createModuleLogger('GcsStorage')
|
||||||
|
|
||||||
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
// HNSW mutex locks to prevent read-modify-write races
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
// Configuration options
|
// Configuration options
|
||||||
|
|
@ -166,7 +166,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
secretAccessKey?: string
|
secretAccessKey?: string
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialization mode for fast cold starts (v7.3.0+)
|
* Initialization mode for fast cold starts
|
||||||
*
|
*
|
||||||
* - `'auto'` (default): Progressive in cloud environments (Cloud Run, Lambda),
|
* - `'auto'` (default): Progressive in cloud environments (Cloud Run, Lambda),
|
||||||
* strict locally. Zero-config optimization.
|
* strict locally. Zero-config optimization.
|
||||||
|
|
@ -175,19 +175,18 @@ export class GcsStorage extends BaseStorage {
|
||||||
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
||||||
* before init() returns.
|
* before init() returns.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
initMode?: InitMode
|
initMode?: InitMode
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated Use `initMode: 'progressive'` instead.
|
* @deprecated Use `initMode: 'progressive'` instead.
|
||||||
* Will be removed in v8.0.0.
|
* Will be removed in a future version.
|
||||||
*/
|
*/
|
||||||
skipInitialScan?: boolean
|
skipInitialScan?: boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated Use `initMode: 'progressive'` instead.
|
* @deprecated Use `initMode: 'progressive'` instead.
|
||||||
* Will be removed in v8.0.0.
|
* Will be removed in a future version.
|
||||||
*/
|
*/
|
||||||
skipCountsFile?: boolean
|
skipCountsFile?: boolean
|
||||||
|
|
||||||
|
|
@ -206,7 +205,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
this.accessKeyId = options.accessKeyId
|
this.accessKeyId = options.accessKeyId
|
||||||
this.secretAccessKey = options.secretAccessKey
|
this.secretAccessKey = options.secretAccessKey
|
||||||
|
|
||||||
// v7.3.0: Handle initMode and deprecated skip* flags
|
// Handle initMode and deprecated skip* flags
|
||||||
if (options.initMode) {
|
if (options.initMode) {
|
||||||
this.initMode = options.initMode
|
this.initMode = options.initMode
|
||||||
}
|
}
|
||||||
|
|
@ -215,14 +214,14 @@ export class GcsStorage extends BaseStorage {
|
||||||
if (options.skipInitialScan) {
|
if (options.skipInitialScan) {
|
||||||
console.warn(
|
console.warn(
|
||||||
'[GcsStorage] DEPRECATION WARNING: skipInitialScan is deprecated. ' +
|
'[GcsStorage] DEPRECATION WARNING: skipInitialScan is deprecated. ' +
|
||||||
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
|
'Use initMode: "progressive" instead. Will be removed in a future version.'
|
||||||
)
|
)
|
||||||
this.skipInitialScan = true
|
this.skipInitialScan = true
|
||||||
}
|
}
|
||||||
if (options.skipCountsFile) {
|
if (options.skipCountsFile) {
|
||||||
console.warn(
|
console.warn(
|
||||||
'[GcsStorage] DEPRECATION WARNING: skipCountsFile is deprecated. ' +
|
'[GcsStorage] DEPRECATION WARNING: skipCountsFile is deprecated. ' +
|
||||||
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
|
'Use initMode: "progressive" instead. Will be removed in a future version.'
|
||||||
)
|
)
|
||||||
this.skipCountsFile = true
|
this.skipCountsFile = true
|
||||||
}
|
}
|
||||||
|
|
@ -240,13 +239,13 @@ export class GcsStorage extends BaseStorage {
|
||||||
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
|
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
|
||||||
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
|
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled - no env var check needed
|
// Write buffering always enabled - no env var check needed
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the storage adapter
|
* Initialize the storage adapter
|
||||||
*
|
*
|
||||||
* v7.3.0: Supports progressive initialization for fast cold starts
|
* Supports progressive initialization for fast cold starts
|
||||||
*
|
*
|
||||||
* | Mode | Init Time | When |
|
* | Mode | Init Time | When |
|
||||||
* |------|-----------|------|
|
* |------|-----------|------|
|
||||||
|
|
@ -337,14 +336,14 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
|
// CRITICAL FIX: Clear any stale cache entries from previous runs
|
||||||
// This prevents cache poisoning from causing silent failures on container restart
|
// This prevents cache poisoning from causing silent failures on container restart
|
||||||
prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning')
|
prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning')
|
||||||
this.nounCacheManager.clear()
|
this.nounCacheManager.clear()
|
||||||
this.verbCacheManager.clear()
|
this.verbCacheManager.clear()
|
||||||
prodLog.info('✅ Cache cleared - starting fresh')
|
prodLog.info('✅ Cache cleared - starting fresh')
|
||||||
|
|
||||||
// v7.3.0: Progressive vs Strict initialization
|
// Progressive vs Strict initialization
|
||||||
if (effectiveMode === 'progressive') {
|
if (effectiveMode === 'progressive') {
|
||||||
// PROGRESSIVE MODE: Fast init, background validation
|
// PROGRESSIVE MODE: Fast init, background validation
|
||||||
// Mark as initialized immediately - ready to accept operations
|
// Mark as initialized immediately - ready to accept operations
|
||||||
|
|
@ -353,7 +352,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
|
|
||||||
prodLog.info(`✅ GCS progressive init complete: ${this.bucketName} (validation deferred)`)
|
prodLog.info(`✅ GCS progressive init complete: ${this.bucketName} (validation deferred)`)
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
|
|
||||||
// Schedule background tasks (non-blocking)
|
// Schedule background tasks (non-blocking)
|
||||||
|
|
@ -373,7 +372,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
await this.initializeCounts()
|
await this.initializeCounts()
|
||||||
this.countsLoaded = true
|
this.countsLoaded = true
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
|
|
||||||
// Mark background tasks as complete (nothing to do in background)
|
// Mark background tasks as complete (nothing to do in background)
|
||||||
|
|
@ -386,7 +385,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Progressive Initialization (v7.3.0+)
|
// Progressive Initialization
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -400,7 +399,6 @@ export class GcsStorage extends BaseStorage {
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @override
|
* @override
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async runBackgroundInit(): Promise<void> {
|
protected async runBackgroundInit(): Promise<void> {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
@ -423,7 +421,6 @@ export class GcsStorage extends BaseStorage {
|
||||||
* Stores result in bucketValidated/bucketValidationError for lazy use.
|
* Stores result in bucketValidated/bucketValidationError for lazy use.
|
||||||
*
|
*
|
||||||
* @private
|
* @private
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
private async validateBucketInBackground(): Promise<void> {
|
private async validateBucketInBackground(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|
@ -451,7 +448,6 @@ export class GcsStorage extends BaseStorage {
|
||||||
* Uses the existing initializeCounts() logic but in background.
|
* Uses the existing initializeCounts() logic but in background.
|
||||||
*
|
*
|
||||||
* @private
|
* @private
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
private async loadCountsInBackground(): Promise<void> {
|
private async loadCountsInBackground(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|
@ -475,7 +471,6 @@ export class GcsStorage extends BaseStorage {
|
||||||
* @throws Error if bucket does not exist or is not accessible
|
* @throws Error if bucket does not exist or is not accessible
|
||||||
* @protected
|
* @protected
|
||||||
* @override
|
* @override
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async ensureValidatedForWrite(): Promise<void> {
|
protected async ensureValidatedForWrite(): Promise<void> {
|
||||||
// If already validated, nothing to do
|
// If already validated, nothing to do
|
||||||
|
|
@ -562,7 +557,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Override base class to enable smart batching for cloud storage (v3.32.3+)
|
* Override base class to enable smart batching for cloud storage
|
||||||
*
|
*
|
||||||
* GCS is cloud storage with network latency (~50ms per write).
|
* GCS is cloud storage with network latency (~50ms per write).
|
||||||
* Smart batching reduces writes from 1000 ops → 100 batches.
|
* Smart batching reduces writes from 1000 ops → 100 batches.
|
||||||
|
|
@ -596,7 +591,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush noun buffer to GCS
|
* Flush noun buffer to GCS
|
||||||
|
|
@ -628,20 +623,20 @@ export class GcsStorage extends BaseStorage {
|
||||||
await Promise.all(writes)
|
await Promise.all(writes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
|
// Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save a node to storage
|
* Save a node to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.nounWriteBuffer) {
|
if (this.nounWriteBuffer) {
|
||||||
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
||||||
|
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
||||||
this.nounCacheManager.set(node.id, node)
|
this.nounCacheManager.set(node.id, node)
|
||||||
}
|
}
|
||||||
|
|
@ -657,10 +652,10 @@ export class GcsStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save a node directly to GCS (bypass buffer)
|
* Save a node directly to GCS (bypass buffer)
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
* Performs lazy bucket validation on first write in progressive mode.
|
||||||
*/
|
*/
|
||||||
private async saveNodeDirect(node: HNSWNode): Promise<void> {
|
private async saveNodeDirect(node: HNSWNode): Promise<void> {
|
||||||
// v7.3.0: Lazy bucket validation for progressive init
|
// Lazy bucket validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
// Apply backpressure before starting operation
|
// Apply backpressure before starting operation
|
||||||
|
|
@ -695,14 +690,14 @@ export class GcsStorage extends BaseStorage {
|
||||||
resumable: false // For small objects, non-resumable is faster
|
resumable: false // For small objects, non-resumable is faster
|
||||||
})
|
})
|
||||||
|
|
||||||
// CRITICAL FIX (v3.37.8): Only cache nodes with non-empty vectors
|
// CRITICAL FIX: Only cache nodes with non-empty vectors
|
||||||
// This prevents cache pollution from HNSW's lazy-loading nodes (vector: [])
|
// This prevents cache pollution from HNSW's lazy-loading nodes (vector: [])
|
||||||
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
||||||
this.nounCacheManager.set(node.id, node)
|
this.nounCacheManager.set(node.id, node)
|
||||||
}
|
}
|
||||||
// Note: Empty vectors are intentional during HNSW lazy mode - not logged
|
// Note: Empty vectors are intentional during HNSW lazy mode - not logged
|
||||||
|
|
||||||
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
|
// Count tracking happens in baseStorage.saveNounMetadata_internal
|
||||||
// This fixes the race condition where metadata didn't exist yet
|
// This fixes the race condition where metadata didn't exist yet
|
||||||
|
|
||||||
this.logger.trace(`Node ${node.id} saved successfully`)
|
this.logger.trace(`Node ${node.id} saved successfully`)
|
||||||
|
|
@ -721,7 +716,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
|
// Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a node from storage
|
* Get a node from storage
|
||||||
|
|
@ -732,7 +727,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
// Check cache first
|
// Check cache first
|
||||||
const cached: HNSWNode | null = await this.nounCacheManager.get(id)
|
const cached: HNSWNode | null = await this.nounCacheManager.get(id)
|
||||||
|
|
||||||
// Validate cached object before returning (v3.37.8+)
|
// Validate cached object before returning
|
||||||
if (cached !== undefined && cached !== null) {
|
if (cached !== undefined && cached !== null) {
|
||||||
// Validate cached object has required fields (including non-empty vector!)
|
// Validate cached object has required fields (including non-empty vector!)
|
||||||
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
|
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
|
||||||
|
|
@ -831,18 +826,18 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
|
// Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write an object to a specific path in GCS
|
* Write an object to a specific path in GCS
|
||||||
* Primitive operation required by base class
|
* Primitive operation required by base class
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
* Performs lazy bucket validation on first write in progressive mode.
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
// v7.3.0: Lazy bucket validation for progressive init
|
// Lazy bucket validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -892,7 +887,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Batch read multiple objects from GCS (v5.12.0 - Cloud Storage Optimization)
|
* Batch read multiple objects from GCS
|
||||||
*
|
*
|
||||||
* **Performance**: GCS-optimized parallel downloads
|
* **Performance**: GCS-optimized parallel downloads
|
||||||
* - Uses Promise.all() for concurrent requests
|
* - Uses Promise.all() for concurrent requests
|
||||||
|
|
@ -908,7 +903,6 @@ export class GcsStorage extends BaseStorage {
|
||||||
* @returns Map of path → data (only successful reads included)
|
* @returns Map of path → data (only successful reads included)
|
||||||
*
|
*
|
||||||
* @public - Called by baseStorage.readBatchFromAdapter()
|
* @public - Called by baseStorage.readBatchFromAdapter()
|
||||||
* @since v5.12.0
|
|
||||||
*/
|
*/
|
||||||
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -960,12 +954,11 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get GCS-specific batch configuration (v5.12.0)
|
* Get GCS-specific batch configuration
|
||||||
*
|
*
|
||||||
* GCS performs well with high concurrency due to HTTP/2 multiplexing
|
* GCS performs well with high concurrency due to HTTP/2 multiplexing
|
||||||
*
|
*
|
||||||
* @public - Overrides BaseStorage.getBatchConfig()
|
* @public - Overrides BaseStorage.getBatchConfig()
|
||||||
* @since v5.12.0
|
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -984,12 +977,12 @@ export class GcsStorage extends BaseStorage {
|
||||||
* Delete an object from a specific path in GCS
|
* Delete an object from a specific path in GCS
|
||||||
* Primitive operation required by base class
|
* Primitive operation required by base class
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
|
* Performs lazy bucket validation on first delete in progressive mode.
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
// v7.3.0: Lazy bucket validation for progressive init
|
// Lazy bucket validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -1034,20 +1027,20 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
|
// Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save an edge to storage
|
* Save an edge to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveEdge(edge: Edge): Promise<void> {
|
protected async saveEdge(edge: Edge): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.verbWriteBuffer) {
|
if (this.verbWriteBuffer) {
|
||||||
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
|
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
|
||||||
|
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
this.verbCacheManager.set(edge.id, edge)
|
this.verbCacheManager.set(edge.id, edge)
|
||||||
|
|
||||||
await this.verbWriteBuffer.add(edge.id, edge)
|
await this.verbWriteBuffer.add(edge.id, edge)
|
||||||
|
|
@ -1061,10 +1054,10 @@ export class GcsStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save an edge directly to GCS (bypass buffer)
|
* Save an edge directly to GCS (bypass buffer)
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
* Performs lazy bucket validation on first write in progressive mode.
|
||||||
*/
|
*/
|
||||||
private async saveEdgeDirect(edge: Edge): Promise<void> {
|
private async saveEdgeDirect(edge: Edge): Promise<void> {
|
||||||
// v7.3.0: Lazy bucket validation for progressive init
|
// Lazy bucket validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
const requestId = await this.applyBackpressure()
|
const requestId = await this.applyBackpressure()
|
||||||
|
|
@ -1073,7 +1066,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
this.logger.trace(`Saving edge ${edge.id}`)
|
this.logger.trace(`Saving edge ${edge.id}`)
|
||||||
|
|
||||||
// Convert connections Map to serializable format
|
// Convert connections Map to serializable format
|
||||||
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
|
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
|
||||||
// These fields are essential for 90% of operations - no metadata lookup needed
|
// These fields are essential for 90% of operations - no metadata lookup needed
|
||||||
const serializableEdge = {
|
const serializableEdge = {
|
||||||
id: edge.id,
|
id: edge.id,
|
||||||
|
|
@ -1085,7 +1078,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
])
|
])
|
||||||
),
|
),
|
||||||
|
|
||||||
// CORE RELATIONAL DATA (v3.50.1+)
|
// CORE RELATIONAL DATA
|
||||||
verb: edge.verb,
|
verb: edge.verb,
|
||||||
sourceId: edge.sourceId,
|
sourceId: edge.sourceId,
|
||||||
targetId: edge.targetId,
|
targetId: edge.targetId,
|
||||||
|
|
@ -1107,7 +1100,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
// Update cache
|
// Update cache
|
||||||
this.verbCacheManager.set(edge.id, edge)
|
this.verbCacheManager.set(edge.id, edge)
|
||||||
|
|
||||||
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
// Count tracking happens in baseStorage.saveVerbMetadata_internal
|
||||||
// This fixes the race condition where metadata didn't exist yet
|
// This fixes the race condition where metadata didn't exist yet
|
||||||
|
|
||||||
this.logger.trace(`Edge ${edge.id} saved successfully`)
|
this.logger.trace(`Edge ${edge.id} saved successfully`)
|
||||||
|
|
@ -1125,7 +1118,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
|
// Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get an edge from storage
|
* Get an edge from storage
|
||||||
|
|
@ -1161,7 +1154,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
connections.set(Number(level), new Set(verbIds as string[]))
|
connections.set(Number(level), new Set(verbIds as string[]))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
// Return HNSWVerb with core relational fields (NO metadata field)
|
||||||
const edge: Edge = {
|
const edge: Edge = {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
|
|
@ -1172,7 +1165,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
sourceId: data.sourceId,
|
sourceId: data.sourceId,
|
||||||
targetId: data.targetId
|
targetId: data.targetId
|
||||||
|
|
||||||
// ✅ NO metadata field in v4.0.0
|
// ✅ NO metadata field
|
||||||
// User metadata retrieved separately via getVerbMetadata()
|
// User metadata retrieved separately via getVerbMetadata()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1201,13 +1194,13 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
|
// Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed pagination overrides - use BaseStorage's type-first implementation
|
// Removed pagination overrides - use BaseStorage's type-first implementation
|
||||||
// - getNounsWithPagination, getNodesWithPagination, getVerbsWithPagination
|
// - getNounsWithPagination, getNodesWithPagination, getVerbsWithPagination
|
||||||
// - getNouns, getVerbs (public wrappers)
|
// - getNouns, getVerbs (public wrappers)
|
||||||
|
|
||||||
// v5.4.0: Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation
|
// Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation
|
||||||
// (getNounsByNounType_internal, getVerbsBySource_internal, getVerbsByTarget_internal, getVerbsByType_internal)
|
// (getNounsByNounType_internal, getVerbsBySource_internal, getVerbsByTarget_internal, getVerbsByType_internal)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1280,7 +1273,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.11.0: Clear ALL data using correct paths
|
// Clear ALL data using correct paths
|
||||||
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
||||||
await deleteObjectsWithPrefix('branches/')
|
await deleteObjectsWithPrefix('branches/')
|
||||||
|
|
||||||
|
|
@ -1290,7 +1283,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
// Delete system metadata
|
// Delete system metadata
|
||||||
await deleteObjectsWithPrefix('_system/')
|
await deleteObjectsWithPrefix('_system/')
|
||||||
|
|
||||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
// Reset COW managers (but don't disable COW - it's always enabled)
|
||||||
// COW will re-initialize automatically on next use
|
// COW will re-initialize automatically on next use
|
||||||
this.refManager = undefined
|
this.refManager = undefined
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
|
|
@ -1352,12 +1345,12 @@ export class GcsStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
* Fixes bug where clear() doesn't persist across instance restarts
|
||||||
* @returns true if marker object exists, false otherwise
|
* @returns true if marker object exists, false otherwise
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -1413,7 +1406,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
if (error.code === 404) {
|
if (error.code === 404) {
|
||||||
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
|
// CRITICAL FIX: Statistics file doesn't exist yet (first restart)
|
||||||
// Return minimal stats with counts instead of null
|
// Return minimal stats with counts instead of null
|
||||||
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
||||||
this.logger.trace('Statistics file not found - returning minimal stats with counts')
|
this.logger.trace('Statistics file not found - returning minimal stats with counts')
|
||||||
|
|
@ -1607,11 +1600,11 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// HNSW Index Persistence (v3.35.0+)
|
// HNSW Index Persistence
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun's vector for HNSW rebuild
|
* Get a noun's vector for HNSW rebuild
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getNounVector(id: string): Promise<number[] | null> {
|
public async getNounVector(id: string): Promise<number[] | null> {
|
||||||
const noun = await this.getNoun(id)
|
const noun = await this.getNoun(id)
|
||||||
|
|
@ -1621,7 +1614,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW graph data for a noun
|
* Save HNSW graph data for a noun
|
||||||
*
|
*
|
||||||
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
||||||
*/
|
*/
|
||||||
public async saveHNSWData(nounId: string, hnswData: {
|
public async saveHNSWData(nounId: string, hnswData: {
|
||||||
|
|
@ -1630,7 +1623,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const lockKey = `hnsw/${nounId}`
|
const lockKey = `hnsw/${nounId}`
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
|
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
|
||||||
// Problem: Without mutex, concurrent operations can:
|
// Problem: Without mutex, concurrent operations can:
|
||||||
// 1. Thread A reads noun (connections: [1,2,3])
|
// 1. Thread A reads noun (connections: [1,2,3])
|
||||||
// 2. Thread B reads noun (connections: [1,2,3])
|
// 2. Thread B reads noun (connections: [1,2,3])
|
||||||
|
|
@ -1650,7 +1643,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
this.hnswLocks.set(lockKey, lockPromise)
|
this.hnswLocks.set(lockKey, lockPromise)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
// Use BaseStorage's getNoun (type-first paths)
|
||||||
// Read existing noun data (if exists)
|
// Read existing noun data (if exists)
|
||||||
const existingNoun = await this.getNoun(nounId)
|
const existingNoun = await this.getNoun(nounId)
|
||||||
|
|
||||||
|
|
@ -1672,7 +1665,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
connections: connectionsMap
|
connections: connectionsMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
|
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
|
||||||
await this.saveNoun(updatedNoun)
|
await this.saveNoun(updatedNoun)
|
||||||
} finally {
|
} finally {
|
||||||
// Release lock (ALWAYS runs, even if error thrown)
|
// Release lock (ALWAYS runs, even if error thrown)
|
||||||
|
|
@ -1683,7 +1676,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get HNSW graph data for a noun
|
* Get HNSW graph data for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getHNSWData(nounId: string): Promise<{
|
public async getHNSWData(nounId: string): Promise<{
|
||||||
level: number
|
level: number
|
||||||
|
|
@ -1713,7 +1706,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
* Save HNSW system data (entry point, max level)
|
* Save HNSW system data (entry point, max level)
|
||||||
* Storage path: system/hnsw-system.json
|
* Storage path: system/hnsw-system.json
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Optimistic locking with generation numbers to prevent race conditions
|
* CRITICAL FIX: Optimistic locking with generation numbers to prevent race conditions
|
||||||
*/
|
*/
|
||||||
public async saveHNSWSystem(systemData: {
|
public async saveHNSWSystem(systemData: {
|
||||||
entryPointId: string | null
|
entryPointId: string | null
|
||||||
|
|
@ -1807,7 +1800,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// GCS Lifecycle Management & Autoclass (v4.0.0)
|
// GCS Lifecycle Management & Autoclass
|
||||||
// Cost optimization through automatic tier transitions and Autoclass
|
// Cost optimization through automatic tier transitions and Autoclass
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@
|
||||||
* - Lazy loading: only loads entities when accessed
|
* - Lazy loading: only loads entities when accessed
|
||||||
* - No eager-loading of entire commit state
|
* - No eager-loading of entire commit state
|
||||||
*
|
*
|
||||||
* v5.4.0: Production-ready, billion-scale historical queries
|
* Production-ready, billion-scale historical queries
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { BaseStorage } from '../baseStorage.js'
|
import { BaseStorage } from '../baseStorage.js'
|
||||||
|
|
@ -144,7 +144,7 @@ export class HistoricalStorageAdapter extends BaseStorage {
|
||||||
*/
|
*/
|
||||||
public async init(): Promise<void> {
|
public async init(): Promise<void> {
|
||||||
// Get COW components from underlying storage
|
// Get COW components from underlying storage
|
||||||
// v6.2.4: Fixed property names - use public properties without underscore prefix
|
// Fixed property names - use public properties without underscore prefix
|
||||||
this.commitLog = this.underlyingStorage.commitLog
|
this.commitLog = this.underlyingStorage.commitLog
|
||||||
this.blobStorage = this.underlyingStorage.blobStorage
|
this.blobStorage = this.underlyingStorage.blobStorage
|
||||||
|
|
||||||
|
|
@ -161,7 +161,7 @@ export class HistoricalStorageAdapter extends BaseStorage {
|
||||||
throw new Error(`Commit not found: ${this.commitId}`)
|
throw new Error(`Commit not found: ${this.commitId}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -310,12 +310,12 @@ export class HistoricalStorageAdapter extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
|
* No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
|
||||||
* @returns Always false (read-only adapter doesn't manage COW state)
|
* @returns Always false (read-only adapter doesn't manage COW state)
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ import { PaginatedResult } from '../../types/paginationTypes.js'
|
||||||
* Uses Maps to store data in memory
|
* Uses Maps to store data in memory
|
||||||
*/
|
*/
|
||||||
export class MemoryStorage extends BaseStorage {
|
export class MemoryStorage extends BaseStorage {
|
||||||
// v5.4.0: Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths
|
// Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths
|
||||||
private statistics: StatisticsData | null = null
|
private statistics: StatisticsData | null = null
|
||||||
|
|
||||||
// Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata)
|
// Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata)
|
||||||
|
|
@ -55,7 +55,6 @@ export class MemoryStorage extends BaseStorage {
|
||||||
* - Parallel processing maximizes throughput
|
* - Parallel processing maximizes throughput
|
||||||
*
|
*
|
||||||
* @returns Memory-optimized batch configuration
|
* @returns Memory-optimized batch configuration
|
||||||
* @since v4.11.0
|
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -72,27 +71,27 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the storage adapter
|
* Initialize the storage adapter
|
||||||
* v6.0.0: Calls super.init() to initialize GraphAdjacencyIndex and type statistics
|
* Calls super.init() to initialize GraphAdjacencyIndex and type statistics
|
||||||
*/
|
*/
|
||||||
public async init(): Promise<void> {
|
public async init(): Promise<void> {
|
||||||
await super.init()
|
await super.init()
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation
|
// Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get nouns with pagination and filtering
|
* Get nouns with pagination and filtering
|
||||||
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
* Returns HNSWNounWithMetadata[] (includes metadata field)
|
||||||
* @param options Pagination and filtering options
|
* @param options Pagination and filtering options
|
||||||
* @returns Promise that resolves to a paginated result of nouns with metadata
|
* @returns Promise that resolves to a paginated result of nouns with metadata
|
||||||
*/
|
*/
|
||||||
// v5.4.0: Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation
|
// Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation
|
// Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation
|
// Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation
|
||||||
|
|
||||||
// v5.4.0: Removed verb *_internal method overrides - using BaseStorage's type-first implementation
|
// Removed verb *_internal method overrides - using BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Primitive operation: Write object to path
|
* Primitive operation: Write object to path
|
||||||
|
|
@ -159,8 +158,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear all data from storage
|
* Clear all data from storage
|
||||||
* v5.4.0: Clears objectStore (type-first paths)
|
* Clears objectStore (type-first paths)
|
||||||
* v7.3.1: Also clears writeCache to prevent stale data after clear
|
* Also clears writeCache to prevent stale data after clear
|
||||||
*/
|
*/
|
||||||
public async clear(): Promise<void> {
|
public async clear(): Promise<void> {
|
||||||
this.objectStore.clear()
|
this.objectStore.clear()
|
||||||
|
|
@ -174,7 +173,7 @@ export class MemoryStorage extends BaseStorage {
|
||||||
this.statisticsCache = null
|
this.statisticsCache = null
|
||||||
this.statisticsModified = false
|
this.statisticsModified = false
|
||||||
|
|
||||||
// v7.3.1: Clear write-through cache (inherited from BaseStorage)
|
// Clear write-through cache (inherited from BaseStorage)
|
||||||
// Without this, readWithInheritance() would return stale cached data
|
// Without this, readWithInheritance() would return stale cached data
|
||||||
// after clear(), causing "ghost" entities to appear
|
// after clear(), causing "ghost" entities to appear
|
||||||
this.clearWriteCache()
|
this.clearWriteCache()
|
||||||
|
|
@ -182,7 +181,7 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get information about storage usage and capacity
|
* Get information about storage usage and capacity
|
||||||
* v5.4.0: Uses BaseStorage counts
|
* Uses BaseStorage counts
|
||||||
*/
|
*/
|
||||||
public async getStorageStatus(): Promise<{
|
public async getStorageStatus(): Promise<{
|
||||||
type: string
|
type: string
|
||||||
|
|
@ -204,12 +203,12 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: No-op for MemoryStorage (doesn't persist)
|
* No-op for MemoryStorage (doesn't persist)
|
||||||
* @returns Always false (marker doesn't persist in memory)
|
* @returns Always false (marker doesn't persist in memory)
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -252,7 +251,7 @@ export class MemoryStorage extends BaseStorage {
|
||||||
*/
|
*/
|
||||||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||||||
if (!this.statistics) {
|
if (!this.statistics) {
|
||||||
// CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init)
|
// CRITICAL FIX: Statistics don't exist yet (first init)
|
||||||
// Return minimal stats with counts instead of null
|
// Return minimal stats with counts instead of null
|
||||||
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
||||||
return {
|
return {
|
||||||
|
|
@ -299,10 +298,10 @@ export class MemoryStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize counts from in-memory storage - O(1) operation (v4.0.0)
|
* Initialize counts from in-memory storage - O(1) operation
|
||||||
*/
|
*/
|
||||||
protected async initializeCounts(): Promise<void> {
|
protected async initializeCounts(): Promise<void> {
|
||||||
// v6.0.0: Scan objectStore paths (ID-first structure) to count entities
|
// Scan objectStore paths (ID-first structure) to count entities
|
||||||
this.entityCounts.clear()
|
this.entityCounts.clear()
|
||||||
this.verbCounts.clear()
|
this.verbCounts.clear()
|
||||||
|
|
||||||
|
|
@ -314,14 +313,14 @@ export class MemoryStorage extends BaseStorage {
|
||||||
// Count nouns (entities/nouns/{shard}/{id}/vectors.json)
|
// Count nouns (entities/nouns/{shard}/{id}/vectors.json)
|
||||||
const nounMatch = path.match(/^entities\/nouns\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
|
const nounMatch = path.match(/^entities\/nouns\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
|
||||||
if (nounMatch) {
|
if (nounMatch) {
|
||||||
// v6.0.0: Type is in metadata, not path - just count total
|
// Type is in metadata, not path - just count total
|
||||||
totalNouns++
|
totalNouns++
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
||||||
const verbMatch = path.match(/^entities\/verbs\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
|
const verbMatch = path.match(/^entities\/verbs\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
|
||||||
if (verbMatch) {
|
if (verbMatch) {
|
||||||
// v6.0.0: Type is in metadata, not path - just count total
|
// Type is in metadata, not path - just count total
|
||||||
totalVerbs++
|
totalVerbs++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -339,26 +338,26 @@ export class MemoryStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// HNSW Index Persistence (v3.35.0+)
|
// HNSW Index Persistence
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get vector for a noun
|
* Get vector for a noun
|
||||||
* v5.4.0: Uses BaseStorage's type-first implementation
|
* Uses BaseStorage's type-first implementation
|
||||||
*/
|
*/
|
||||||
public async getNounVector(id: string): Promise<number[] | null> {
|
public async getNounVector(id: string): Promise<number[] | null> {
|
||||||
const noun = await this.getNoun(id)
|
const noun = await this.getNoun(id)
|
||||||
return noun ? [...noun.vector] : null
|
return noun ? [...noun.vector] : null
|
||||||
}
|
}
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
|
// CRITICAL FIX: Mutex locks for HNSW concurrency control
|
||||||
// Even in-memory operations need serialization to prevent async race conditions
|
// Even in-memory operations need serialization to prevent async race conditions
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save HNSW graph data for a noun
|
* Save HNSW graph data for a noun
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions during concurrent HNSW updates
|
* CRITICAL FIX: Mutex locking to prevent race conditions during concurrent HNSW updates
|
||||||
* Even in-memory operations can race due to async/await interleaving
|
* Even in-memory operations can race due to async/await interleaving
|
||||||
* Prevents data corruption when multiple entities connect to same neighbor simultaneously
|
* Prevents data corruption when multiple entities connect to same neighbor simultaneously
|
||||||
*/
|
*/
|
||||||
|
|
@ -417,7 +416,7 @@ export class MemoryStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW system data (entry point, max level)
|
* Save HNSW system data (entry point, max level)
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions
|
* CRITICAL FIX: Mutex locking to prevent race conditions
|
||||||
*/
|
*/
|
||||||
public async saveHNSWSystem(systemData: {
|
public async saveHNSWSystem(systemData: {
|
||||||
entryPointId: string | null
|
entryPointId: string | null
|
||||||
|
|
|
||||||
|
|
@ -57,7 +57,7 @@ const ROOT_DIR = 'opfs-vector-db'
|
||||||
* OPFS storage adapter for browser environments
|
* OPFS storage adapter for browser environments
|
||||||
* Uses the Origin Private File System API to store data persistently
|
* Uses the Origin Private File System API to store data persistently
|
||||||
*
|
*
|
||||||
* v5.4.0: Type-aware storage now built into BaseStorage
|
* Type-aware storage now built into BaseStorage
|
||||||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||||||
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
||||||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
|
|
@ -97,7 +97,6 @@ export class OPFSStorage extends BaseStorage {
|
||||||
* - Sequential processing preferred for stability
|
* - Sequential processing preferred for stability
|
||||||
*
|
*
|
||||||
* @returns OPFS-optimized batch configuration
|
* @returns OPFS-optimized batch configuration
|
||||||
* @since v4.11.0
|
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -172,7 +171,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
// Initialize counts from storage
|
// Initialize counts from storage
|
||||||
await this.initializeCounts()
|
await this.initializeCounts()
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to initialize OPFS storage:', error)
|
console.error('Failed to initialize OPFS storage:', error)
|
||||||
|
|
@ -232,7 +231,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete an edge from storage
|
* Delete an edge from storage
|
||||||
|
|
@ -482,14 +481,14 @@ export class OPFSStorage extends BaseStorage {
|
||||||
// Remove all files in the index directory
|
// Remove all files in the index directory
|
||||||
await removeDirectoryContents(this.indexDir!)
|
await removeDirectoryContents(this.indexDir!)
|
||||||
|
|
||||||
// v5.6.1: Remove COW (copy-on-write) version control data
|
// Remove COW (copy-on-write) version control data
|
||||||
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
|
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
|
||||||
// Must be deleted to fully clear all data including version history
|
// Must be deleted to fully clear all data including version history
|
||||||
try {
|
try {
|
||||||
// Delete the entire _cow/ directory (not just contents)
|
// Delete the entire _cow/ directory (not just contents)
|
||||||
await this.rootDir!.removeEntry('_cow', { recursive: true })
|
await this.rootDir!.removeEntry('_cow', { recursive: true })
|
||||||
|
|
||||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
// Reset COW managers (but don't disable COW - it's always enabled)
|
||||||
// COW will re-initialize automatically on next use
|
// COW will re-initialize automatically on next use
|
||||||
this.refManager = undefined
|
this.refManager = undefined
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
|
|
@ -505,7 +504,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
this.statisticsCache = null
|
this.statisticsCache = null
|
||||||
this.statisticsModified = false
|
this.statisticsModified = false
|
||||||
|
|
||||||
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
|
// Reset entity counters (inherited from BaseStorageAdapter)
|
||||||
// These in-memory counters must be reset to 0 after clearing all data
|
// These in-memory counters must be reset to 0 after clearing all data
|
||||||
;(this as any).totalNounCount = 0
|
;(this as any).totalNounCount = 0
|
||||||
;(this as any).totalVerbCount = 0
|
;(this as any).totalVerbCount = 0
|
||||||
|
|
@ -517,16 +516,16 @@ export class OPFSStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
* Fixes bug where clear() doesn't persist across instance restarts
|
||||||
* @returns true if marker file exists, false otherwise
|
* @returns true if marker file exists, false otherwise
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Quota monitoring configuration (v4.0.0)
|
// Quota monitoring configuration
|
||||||
private quotaWarningThreshold = 0.8 // Warn at 80% usage
|
private quotaWarningThreshold = 0.8 // Warn at 80% usage
|
||||||
private quotaCriticalThreshold = 0.95 // Critical at 95% usage
|
private quotaCriticalThreshold = 0.95 // Critical at 95% usage
|
||||||
private lastQuotaCheck: number = 0
|
private lastQuotaCheck: number = 0
|
||||||
|
|
@ -675,7 +674,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get detailed quota status with warnings (v4.0.0)
|
* Get detailed quota status with warnings
|
||||||
* Monitors storage usage and warns when approaching quota limits
|
* Monitors storage usage and warns when approaching quota limits
|
||||||
*
|
*
|
||||||
* @returns Promise that resolves to quota status with warning levels
|
* @returns Promise that resolves to quota status with warning levels
|
||||||
|
|
@ -769,7 +768,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Monitor quota during operations (v4.0.0)
|
* Monitor quota during operations
|
||||||
* Automatically checks quota at regular intervals and warns if approaching limits
|
* Automatically checks quota at regular intervals and warns if approaching limits
|
||||||
* Call this before write operations to ensure quota is available
|
* Call this before write operations to ensure quota is available
|
||||||
*
|
*
|
||||||
|
|
@ -1185,7 +1184,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// CRITICAL FIX (v3.37.4): No statistics files exist (first init)
|
// CRITICAL FIX: No statistics files exist (first init)
|
||||||
// Return minimal stats with counts instead of null
|
// Return minimal stats with counts instead of null
|
||||||
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
||||||
return {
|
return {
|
||||||
|
|
@ -1228,7 +1227,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
* @param options Pagination and filter options
|
* @param options Pagination and filter options
|
||||||
* @returns Promise that resolves to a paginated result of nouns
|
* @returns Promise that resolves to a paginated result of nouns
|
||||||
*/
|
*/
|
||||||
// v5.4.0: Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
|
// Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize counts from OPFS storage
|
* Initialize counts from OPFS storage
|
||||||
|
|
@ -1313,28 +1312,28 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// HNSW Index Persistence (v3.35.0+)
|
// HNSW Index Persistence
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun's vector for HNSW rebuild
|
* Get a noun's vector for HNSW rebuild
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* Get vector for a noun
|
* Get vector for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getNounVector(id: string): Promise<number[] | null> {
|
public async getNounVector(id: string): Promise<number[] | null> {
|
||||||
const noun = await this.getNoun(id)
|
const noun = await this.getNoun(id)
|
||||||
return noun ? noun.vector : null
|
return noun ? noun.vector : null
|
||||||
}
|
}
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
|
// CRITICAL FIX: Mutex locks for HNSW concurrency control
|
||||||
// Browser environments are single-threaded but async operations can still interleave
|
// Browser environments are single-threaded but async operations can still interleave
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save HNSW graph data for a noun
|
* Save HNSW graph data for a noun
|
||||||
*
|
*
|
||||||
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
||||||
*/
|
*/
|
||||||
public async saveHNSWData(nounId: string, hnswData: {
|
public async saveHNSWData(nounId: string, hnswData: {
|
||||||
|
|
@ -1354,7 +1353,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
this.hnswLocks.set(lockKey, lockPromise)
|
this.hnswLocks.set(lockKey, lockPromise)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
// Use BaseStorage's getNoun (type-first paths)
|
||||||
const existingNoun = await this.getNoun(nounId)
|
const existingNoun = await this.getNoun(nounId)
|
||||||
|
|
||||||
if (!existingNoun) {
|
if (!existingNoun) {
|
||||||
|
|
@ -1374,7 +1373,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
connections: connectionsMap
|
connections: connectionsMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Use BaseStorage's saveNoun (type-first paths)
|
// Use BaseStorage's saveNoun (type-first paths)
|
||||||
await this.saveNoun(updatedNoun)
|
await this.saveNoun(updatedNoun)
|
||||||
} finally {
|
} finally {
|
||||||
// Release lock
|
// Release lock
|
||||||
|
|
@ -1385,7 +1384,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get HNSW graph data for a noun
|
* Get HNSW graph data for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getHNSWData(nounId: string): Promise<{
|
public async getHNSWData(nounId: string): Promise<{
|
||||||
level: number
|
level: number
|
||||||
|
|
@ -1415,7 +1414,7 @@ export class OPFSStorage extends BaseStorage {
|
||||||
* Save HNSW system data (entry point, max level)
|
* Save HNSW system data (entry point, max level)
|
||||||
* Storage path: index/hnsw-system.json
|
* Storage path: index/hnsw-system.json
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions
|
* CRITICAL FIX: Mutex locking to prevent race conditions
|
||||||
*/
|
*/
|
||||||
public async saveHNSWSystem(systemData: {
|
public async saveHNSWSystem(systemData: {
|
||||||
entryPointId: string | null
|
entryPointId: string | null
|
||||||
|
|
|
||||||
|
|
@ -107,7 +107,7 @@ export class OptimizedS3Search {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine if there are more items
|
// Determine if there are more items
|
||||||
const hasMore = listResult.hasMore || nouns.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop)
|
const hasMore = listResult.hasMore || nouns.length > limit // Fixed >= to > (was causing infinite loop)
|
||||||
|
|
||||||
// Set next cursor
|
// Set next cursor
|
||||||
let nextCursor: string | undefined
|
let nextCursor: string | undefined
|
||||||
|
|
@ -190,7 +190,7 @@ export class OptimizedS3Search {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine if there are more items
|
// Determine if there are more items
|
||||||
const hasMore = listResult.hasMore || verbs.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop)
|
const hasMore = listResult.hasMore || verbs.length > limit // Fixed >= to > (was causing infinite loop)
|
||||||
|
|
||||||
// Set next cursor
|
// Set next cursor
|
||||||
let nextCursor: string | undefined
|
let nextCursor: string | undefined
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ const MAX_R2_PAGE_SIZE = 1000
|
||||||
* Dedicated Cloudflare R2 storage adapter
|
* Dedicated Cloudflare R2 storage adapter
|
||||||
* Optimized for R2's unique characteristics and global edge network
|
* Optimized for R2's unique characteristics and global edge network
|
||||||
*
|
*
|
||||||
* v5.4.0: Type-aware storage now built into BaseStorage
|
* Type-aware storage now built into BaseStorage
|
||||||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||||||
* - Removed getNounsWithPagination override
|
* - Removed getNounsWithPagination override
|
||||||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
|
|
@ -112,7 +112,7 @@ export class R2Storage extends BaseStorage {
|
||||||
// Request coalescer for deduplication
|
// Request coalescer for deduplication
|
||||||
private requestCoalescer: RequestCoalescer | null = null
|
private requestCoalescer: RequestCoalescer | null = null
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled for consistent performance
|
// Write buffering always enabled for consistent performance
|
||||||
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
||||||
|
|
||||||
// Multi-level cache manager for efficient data access
|
// Multi-level cache manager for efficient data access
|
||||||
|
|
@ -122,7 +122,7 @@ export class R2Storage extends BaseStorage {
|
||||||
// Module logger
|
// Module logger
|
||||||
private logger = createModuleLogger('R2Storage')
|
private logger = createModuleLogger('R2Storage')
|
||||||
|
|
||||||
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
// HNSW mutex locks to prevent read-modify-write races
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -168,7 +168,7 @@ export class R2Storage extends BaseStorage {
|
||||||
})
|
})
|
||||||
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
|
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled - no env var check needed
|
// Write buffering always enabled - no env var check needed
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -183,7 +183,7 @@ export class R2Storage extends BaseStorage {
|
||||||
* Zero egress fees enable aggressive caching and parallel downloads
|
* Zero egress fees enable aggressive caching and parallel downloads
|
||||||
*
|
*
|
||||||
* @returns R2-optimized batch configuration
|
* @returns R2-optimized batch configuration
|
||||||
* @since v5.12.0 - Updated for native batch API
|
* Updated for native batch API
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -208,7 +208,6 @@ export class R2Storage extends BaseStorage {
|
||||||
*
|
*
|
||||||
* @param paths - Array of R2 object keys to read
|
* @param paths - Array of R2 object keys to read
|
||||||
* @returns Map of path -> parsed JSON data (only successful reads)
|
* @returns Map of path -> parsed JSON data (only successful reads)
|
||||||
* @since v5.12.0
|
|
||||||
*/
|
*/
|
||||||
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -336,7 +335,7 @@ export class R2Storage extends BaseStorage {
|
||||||
this.nounCacheManager.clear()
|
this.nounCacheManager.clear()
|
||||||
this.verbCacheManager.clear()
|
this.verbCacheManager.clear()
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error('Failed to initialize R2 storage:', error)
|
this.logger.error('Failed to initialize R2 storage:', error)
|
||||||
|
|
@ -409,7 +408,7 @@ export class R2Storage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush noun buffer to R2
|
* Flush noun buffer to R2
|
||||||
|
|
@ -444,16 +443,16 @@ export class R2Storage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save a node to storage
|
* Save a node to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.nounWriteBuffer) {
|
if (this.nounWriteBuffer) {
|
||||||
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
||||||
|
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
||||||
this.nounCacheManager.set(node.id, node)
|
this.nounCacheManager.set(node.id, node)
|
||||||
}
|
}
|
||||||
|
|
@ -728,14 +727,14 @@ export class R2Storage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save an edge to storage
|
* Save an edge to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveEdge(edge: Edge): Promise<void> {
|
protected async saveEdge(edge: Edge): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.verbWriteBuffer) {
|
if (this.verbWriteBuffer) {
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
this.verbCacheManager.set(edge.id, edge)
|
this.verbCacheManager.set(edge.id, edge)
|
||||||
|
|
||||||
await this.verbWriteBuffer.add(edge.id, edge)
|
await this.verbWriteBuffer.add(edge.id, edge)
|
||||||
|
|
@ -750,7 +749,7 @@ export class R2Storage extends BaseStorage {
|
||||||
const requestId = await this.applyBackpressure()
|
const requestId = await this.applyBackpressure()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
|
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
|
||||||
// These fields are essential for 90% of operations - no metadata lookup needed
|
// These fields are essential for 90% of operations - no metadata lookup needed
|
||||||
const serializableEdge = {
|
const serializableEdge = {
|
||||||
id: edge.id,
|
id: edge.id,
|
||||||
|
|
@ -762,7 +761,7 @@ export class R2Storage extends BaseStorage {
|
||||||
])
|
])
|
||||||
),
|
),
|
||||||
|
|
||||||
// CORE RELATIONAL DATA (v3.50.1+)
|
// CORE RELATIONAL DATA
|
||||||
verb: edge.verb,
|
verb: edge.verb,
|
||||||
sourceId: edge.sourceId,
|
sourceId: edge.sourceId,
|
||||||
targetId: edge.targetId,
|
targetId: edge.targetId,
|
||||||
|
|
@ -785,7 +784,7 @@ export class R2Storage extends BaseStorage {
|
||||||
|
|
||||||
this.verbCacheManager.set(edge.id, edge)
|
this.verbCacheManager.set(edge.id, edge)
|
||||||
|
|
||||||
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
|
// Count tracking happens in baseStorage.saveVerbMetadata_internal
|
||||||
// This fixes the race condition where metadata didn't exist yet
|
// This fixes the race condition where metadata didn't exist yet
|
||||||
|
|
||||||
this.releaseBackpressure(true, requestId)
|
this.releaseBackpressure(true, requestId)
|
||||||
|
|
@ -831,7 +830,7 @@ export class R2Storage extends BaseStorage {
|
||||||
connections.set(Number(level), new Set(verbIds as string[]))
|
connections.set(Number(level), new Set(verbIds as string[]))
|
||||||
}
|
}
|
||||||
|
|
||||||
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
// Return HNSWVerb with core relational fields (NO metadata field)
|
||||||
const edge: Edge = {
|
const edge: Edge = {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
|
|
@ -842,7 +841,7 @@ export class R2Storage extends BaseStorage {
|
||||||
sourceId: data.sourceId,
|
sourceId: data.sourceId,
|
||||||
targetId: data.targetId
|
targetId: data.targetId
|
||||||
|
|
||||||
// ✅ NO metadata field in v4.0.0
|
// ✅ NO metadata field
|
||||||
// User metadata retrieved separately via getVerbMetadata()
|
// User metadata retrieved separately via getVerbMetadata()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1081,7 +1080,7 @@ export class R2Storage extends BaseStorage {
|
||||||
|
|
||||||
prodLog.info('🧹 R2: Clearing all data from bucket...')
|
prodLog.info('🧹 R2: Clearing all data from bucket...')
|
||||||
|
|
||||||
// v5.11.0: Clear ALL data using correct paths
|
// Clear ALL data using correct paths
|
||||||
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
||||||
const branchObjects = await this.listObjectsUnderPath('branches/')
|
const branchObjects = await this.listObjectsUnderPath('branches/')
|
||||||
for (const key of branchObjects) {
|
for (const key of branchObjects) {
|
||||||
|
|
@ -1100,7 +1099,7 @@ export class R2Storage extends BaseStorage {
|
||||||
await this.deleteObjectFromPath(key)
|
await this.deleteObjectFromPath(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
// Reset COW managers (but don't disable COW - it's always enabled)
|
||||||
// COW will re-initialize automatically on next use
|
// COW will re-initialize automatically on next use
|
||||||
this.refManager = undefined
|
this.refManager = undefined
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
|
|
@ -1143,17 +1142,17 @@ export class R2Storage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
* Fixes bug where clear() doesn't persist across instance restarts
|
||||||
* @returns true if marker object exists, false otherwise
|
* @returns true if marker object exists, false otherwise
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
// Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
||||||
|
|
||||||
|
|
||||||
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -85,7 +85,7 @@ type S3Command = any
|
||||||
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
||||||
* - bucketName: GCS bucket name
|
* - bucketName: GCS bucket name
|
||||||
*
|
*
|
||||||
* v5.4.0: Type-aware storage now built into BaseStorage
|
* Type-aware storage now built into BaseStorage
|
||||||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||||||
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
||||||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
|
|
@ -152,7 +152,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
// Request coalescer for deduplication
|
// Request coalescer for deduplication
|
||||||
private requestCoalescer: RequestCoalescer | null = null
|
private requestCoalescer: RequestCoalescer | null = null
|
||||||
|
|
||||||
// v6.2.7: Write buffering always enabled for consistent performance
|
// Write buffering always enabled for consistent performance
|
||||||
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
|
||||||
|
|
||||||
// Operation executors for timeout and retry handling
|
// Operation executors for timeout and retry handling
|
||||||
|
|
@ -165,7 +165,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
// Module logger
|
// Module logger
|
||||||
private logger = createModuleLogger('S3Storage')
|
private logger = createModuleLogger('S3Storage')
|
||||||
|
|
||||||
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
// HNSW mutex locks to prevent read-modify-write races
|
||||||
private hnswLocks = new Map<string, Promise<void>>()
|
private hnswLocks = new Map<string, Promise<void>>()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -210,7 +210,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialization mode for fast cold starts (v7.3.0+)
|
* Initialization mode for fast cold starts
|
||||||
*
|
*
|
||||||
* - `'auto'` (default): Progressive in cloud environments (Lambda, Cloud Run),
|
* - `'auto'` (default): Progressive in cloud environments (Lambda, Cloud Run),
|
||||||
* strict locally. Zero-config optimization.
|
* strict locally. Zero-config optimization.
|
||||||
|
|
@ -219,7 +219,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
||||||
* before init() returns.
|
* before init() returns.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
initMode?: InitMode
|
initMode?: InitMode
|
||||||
|
|
||||||
|
|
@ -236,7 +235,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
this.serviceType = options.serviceType || 's3'
|
this.serviceType = options.serviceType || 's3'
|
||||||
this.readOnly = options.readOnly || false
|
this.readOnly = options.readOnly || false
|
||||||
|
|
||||||
// v7.3.0: Handle initMode
|
// Handle initMode
|
||||||
if (options.initMode) {
|
if (options.initMode) {
|
||||||
this.initMode = options.initMode
|
this.initMode = options.initMode
|
||||||
}
|
}
|
||||||
|
|
@ -270,7 +269,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* S3 supports ~5000 operations/second with burst capacity up to 10,000
|
* S3 supports ~5000 operations/second with burst capacity up to 10,000
|
||||||
*
|
*
|
||||||
* @returns S3-optimized batch configuration
|
* @returns S3-optimized batch configuration
|
||||||
* @since v5.12.0 - Updated for native batch API
|
* Updated for native batch API
|
||||||
*/
|
*/
|
||||||
public getBatchConfig(): StorageBatchConfig {
|
public getBatchConfig(): StorageBatchConfig {
|
||||||
return {
|
return {
|
||||||
|
|
@ -295,7 +294,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
*
|
*
|
||||||
* @param paths - Array of S3 object keys to read
|
* @param paths - Array of S3 object keys to read
|
||||||
* @returns Map of path -> parsed JSON data (only successful reads)
|
* @returns Map of path -> parsed JSON data (only successful reads)
|
||||||
* @since v5.12.0
|
|
||||||
*/
|
*/
|
||||||
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
public async readBatch(paths: string[]): Promise<Map<string, any>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
@ -358,7 +356,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Initialize the storage adapter
|
* Initialize the storage adapter
|
||||||
*
|
*
|
||||||
* v7.3.0: Supports progressive initialization for fast cold starts
|
* Supports progressive initialization for fast cold starts
|
||||||
*
|
*
|
||||||
* | Mode | Init Time | When |
|
* | Mode | Init Time | When |
|
||||||
* |------|-----------|------|
|
* |------|-----------|------|
|
||||||
|
|
@ -514,7 +512,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
// Initialize request coalescer
|
// Initialize request coalescer
|
||||||
this.initializeCoalescer()
|
this.initializeCoalescer()
|
||||||
|
|
||||||
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
|
// CRITICAL FIX: Clear any stale cache entries from previous runs
|
||||||
// This prevents cache poisoning from causing silent failures on container restart
|
// This prevents cache poisoning from causing silent failures on container restart
|
||||||
const nodeCacheSize = this.nodeCache?.size || 0
|
const nodeCacheSize = this.nodeCache?.size || 0
|
||||||
if (nodeCacheSize > 0) {
|
if (nodeCacheSize > 0) {
|
||||||
|
|
@ -524,7 +522,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
prodLog.info('🧹 Node cache is empty - starting fresh')
|
prodLog.info('🧹 Node cache is empty - starting fresh')
|
||||||
}
|
}
|
||||||
|
|
||||||
// v7.3.0: Progressive vs Strict initialization
|
// Progressive vs Strict initialization
|
||||||
if (effectiveMode === 'progressive') {
|
if (effectiveMode === 'progressive') {
|
||||||
// PROGRESSIVE MODE: Fast init, background validation
|
// PROGRESSIVE MODE: Fast init, background validation
|
||||||
// Mark as initialized immediately - ready to accept operations
|
// Mark as initialized immediately - ready to accept operations
|
||||||
|
|
@ -533,7 +531,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
prodLog.info(`✅ S3 progressive init complete: ${this.bucketName} (validation deferred)`)
|
prodLog.info(`✅ S3 progressive init complete: ${this.bucketName} (validation deferred)`)
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
|
|
||||||
// Schedule background tasks (non-blocking)
|
// Schedule background tasks (non-blocking)
|
||||||
|
|
@ -556,7 +554,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
await this.initializeCounts()
|
await this.initializeCounts()
|
||||||
this.countsLoaded = true
|
this.countsLoaded = true
|
||||||
|
|
||||||
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
|
// Initialize GraphAdjacencyIndex and type statistics
|
||||||
await super.init()
|
await super.init()
|
||||||
|
|
||||||
// Mark background tasks as complete (nothing to do in background)
|
// Mark background tasks as complete (nothing to do in background)
|
||||||
|
|
@ -573,7 +571,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================
|
// =============================================
|
||||||
// Progressive Initialization (v7.3.0+)
|
// Progressive Initialization
|
||||||
// =============================================
|
// =============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -588,7 +586,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
*
|
*
|
||||||
* @protected
|
* @protected
|
||||||
* @override
|
* @override
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async runBackgroundInit(): Promise<void> {
|
protected async runBackgroundInit(): Promise<void> {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
@ -614,7 +611,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* Stores result in bucketValidated/bucketValidationError for lazy use.
|
* Stores result in bucketValidated/bucketValidationError for lazy use.
|
||||||
*
|
*
|
||||||
* @private
|
* @private
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
private async validateBucketInBackground(): Promise<void> {
|
private async validateBucketInBackground(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|
@ -638,7 +634,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* Load counts from storage in background.
|
* Load counts from storage in background.
|
||||||
*
|
*
|
||||||
* @private
|
* @private
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
private async loadCountsInBackground(): Promise<void> {
|
private async loadCountsInBackground(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|
@ -662,7 +657,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* @throws Error if bucket does not exist or is not accessible
|
* @throws Error if bucket does not exist or is not accessible
|
||||||
* @protected
|
* @protected
|
||||||
* @override
|
* @override
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
protected async ensureValidatedForWrite(): Promise<void> {
|
protected async ensureValidatedForWrite(): Promise<void> {
|
||||||
// If already validated, nothing to do
|
// If already validated, nothing to do
|
||||||
|
|
@ -918,7 +912,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Bulk write nouns to S3
|
* Bulk write nouns to S3
|
||||||
|
|
@ -1179,20 +1173,20 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
return this.socketManager.getBatchSize()
|
return this.socketManager.getBatchSize()
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
|
// Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save a node to storage
|
* Save a node to storage
|
||||||
* v6.2.7: Always uses write buffer for consistent performance
|
* Always uses write buffer for consistent performance
|
||||||
*/
|
*/
|
||||||
protected async saveNode(node: HNSWNode): Promise<void> {
|
protected async saveNode(node: HNSWNode): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
// Always use write buffer - cloud storage benefits from batching
|
||||||
if (this.nounWriteBuffer) {
|
if (this.nounWriteBuffer) {
|
||||||
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
|
||||||
|
|
||||||
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
// Populate cache BEFORE buffering for read-after-write consistency
|
||||||
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
||||||
this.nounCacheManager.set(node.id, node)
|
this.nounCacheManager.set(node.id, node)
|
||||||
}
|
}
|
||||||
|
|
@ -1242,7 +1236,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
this.logger.debug(`Node ${node.id} saved successfully`)
|
this.logger.debug(`Node ${node.id} saved successfully`)
|
||||||
|
|
||||||
// Log the change for efficient synchronization (v4.0.0: no metadata on node)
|
// Log the change for efficient synchronization (no metadata on node)
|
||||||
await this.appendToChangeLog({
|
await this.appendToChangeLog({
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
operation: 'add', // Could be 'update' if we track existing nodes
|
operation: 'add', // Could be 'update' if we track existing nodes
|
||||||
|
|
@ -1250,7 +1244,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
entityId: node.id,
|
entityId: node.id,
|
||||||
data: {
|
data: {
|
||||||
vector: node.vector
|
vector: node.vector
|
||||||
// ✅ NO metadata field in v4.0.0 - stored separately
|
// ✅ NO metadata field - stored separately
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1296,7 +1290,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getNoun_internal override - uses BaseStorage type-first implementation
|
// Removed getNoun_internal override - uses BaseStorage type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a node from storage
|
* Get a node from storage
|
||||||
|
|
@ -1307,7 +1301,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
// Check cache first
|
// Check cache first
|
||||||
const cached = this.nodeCache.get(id)
|
const cached = this.nodeCache.get(id)
|
||||||
|
|
||||||
// Validate cached object before returning (v3.37.8+)
|
// Validate cached object before returning
|
||||||
if (cached !== undefined && cached !== null) {
|
if (cached !== undefined && cached !== null) {
|
||||||
// Validate cached object has required fields (including non-empty vector!)
|
// Validate cached object has required fields (including non-empty vector!)
|
||||||
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
|
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
|
||||||
|
|
@ -1607,7 +1601,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
return nodes
|
return nodes
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal)
|
// Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal)
|
||||||
// Now inherit from BaseStorage's type-first implementation
|
// Now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1788,23 +1782,23 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
return true // Return all edges since filtering requires metadata
|
return true // Return all edges since filtering requires metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getVerbsWithPagination override - use BaseStorage's type-first implementation
|
// Removed getVerbsWithPagination override - use BaseStorage's type-first implementation
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// v5.4.0: Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb)
|
// Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb)
|
||||||
// Total: 8 *_internal methods removed - all now inherit from BaseStorage's type-first implementation
|
// Total: 8 *_internal methods removed - all now inherit from BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Primitive operation: Write object to path
|
* Primitive operation: Write object to path
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
|
* Performs lazy bucket validation on first write in progressive mode.
|
||||||
*/
|
*/
|
||||||
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
protected async writeObjectToPath(path: string, data: any): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
// v7.3.0: Lazy bucket validation for progressive init
|
// Lazy bucket validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
// Apply backpressure before starting operation
|
// Apply backpressure before starting operation
|
||||||
|
|
@ -1897,11 +1891,11 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* Primitive operation: Delete object from path
|
* Primitive operation: Delete object from path
|
||||||
* All metadata operations use this internally via base class routing
|
* All metadata operations use this internally via base class routing
|
||||||
*
|
*
|
||||||
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
|
* Performs lazy bucket validation on first delete in progressive mode.
|
||||||
*/
|
*/
|
||||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
// v7.3.0: Lazy bucket validation for progressive init
|
// Lazy bucket validation for progressive init
|
||||||
await this.ensureValidatedForWrite()
|
await this.ensureValidatedForWrite()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -2315,7 +2309,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.11.0: Clear ALL data using correct paths
|
// Clear ALL data using correct paths
|
||||||
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
||||||
await deleteObjectsWithPrefix('branches/')
|
await deleteObjectsWithPrefix('branches/')
|
||||||
|
|
||||||
|
|
@ -2325,7 +2319,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
// Delete system metadata
|
// Delete system metadata
|
||||||
await deleteObjectsWithPrefix('_system/')
|
await deleteObjectsWithPrefix('_system/')
|
||||||
|
|
||||||
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
// Reset COW managers (but don't disable COW - it's always enabled)
|
||||||
// COW will re-initialize automatically on next use
|
// COW will re-initialize automatically on next use
|
||||||
this.refManager = undefined
|
this.refManager = undefined
|
||||||
this.blobStorage = undefined
|
this.blobStorage = undefined
|
||||||
|
|
@ -2335,7 +2329,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
this.statisticsCache = null
|
this.statisticsCache = null
|
||||||
this.statisticsModified = false
|
this.statisticsModified = false
|
||||||
|
|
||||||
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
|
// Reset entity counters (inherited from BaseStorageAdapter)
|
||||||
// These in-memory counters must be reset to 0 after clearing all data
|
// These in-memory counters must be reset to 0 after clearing all data
|
||||||
;(this as any).totalNounCount = 0
|
;(this as any).totalNounCount = 0
|
||||||
;(this as any).totalVerbCount = 0
|
;(this as any).totalVerbCount = 0
|
||||||
|
|
@ -2457,12 +2451,12 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if COW has been explicitly disabled via clear()
|
* Check if COW has been explicitly disabled via clear()
|
||||||
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
* Fixes bug where clear() doesn't persist across instance restarts
|
||||||
* @returns true if marker object exists, false otherwise
|
* @returns true if marker object exists, false otherwise
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
* Removed checkClearMarker() and createClearMarker() methods
|
||||||
* COW is now always enabled - marker files are no longer used
|
* COW is now always enabled - marker files are no longer used
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
@ -2851,7 +2845,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
totalEdges: this.totalVerbCount
|
totalEdges: this.totalVerbCount
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
|
// CRITICAL FIX: Statistics file doesn't exist yet (first restart)
|
||||||
// Return minimal stats with counts instead of null
|
// Return minimal stats with counts instead of null
|
||||||
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
// This prevents HNSW from seeing entityCount=0 during index rebuild
|
||||||
return {
|
return {
|
||||||
|
|
@ -3415,7 +3409,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
// Removed getNounsWithPagination override - use BaseStorage's type-first implementation
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Estimate total noun count by listing objects across all shards
|
* Estimate total noun count by listing objects across all shards
|
||||||
|
|
@ -3549,7 +3543,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Override base class to enable smart batching for cloud storage (v3.32.3+)
|
* Override base class to enable smart batching for cloud storage
|
||||||
*
|
*
|
||||||
* S3 is cloud storage with network latency (~50ms per write).
|
* S3 is cloud storage with network latency (~50ms per write).
|
||||||
* Smart batching reduces writes from 1000 ops → 100 batches.
|
* Smart batching reduces writes from 1000 ops → 100 batches.
|
||||||
|
|
@ -3560,11 +3554,11 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
return true // S3 benefits from batching
|
return true // S3 benefits from batching
|
||||||
}
|
}
|
||||||
|
|
||||||
// HNSW Index Persistence (v3.35.0+)
|
// HNSW Index Persistence
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun's vector for HNSW rebuild
|
* Get a noun's vector for HNSW rebuild
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getNounVector(id: string): Promise<number[] | null> {
|
public async getNounVector(id: string): Promise<number[] | null> {
|
||||||
const noun = await this.getNoun(id)
|
const noun = await this.getNoun(id)
|
||||||
|
|
@ -3574,7 +3568,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
/**
|
/**
|
||||||
* Save HNSW graph data for a noun
|
* Save HNSW graph data for a noun
|
||||||
*
|
*
|
||||||
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
||||||
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
||||||
*/
|
*/
|
||||||
public async saveHNSWData(nounId: string, hnswData: {
|
public async saveHNSWData(nounId: string, hnswData: {
|
||||||
|
|
@ -3583,7 +3577,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const lockKey = `hnsw/${nounId}`
|
const lockKey = `hnsw/${nounId}`
|
||||||
|
|
||||||
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
|
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
|
||||||
// Problem: Without mutex, concurrent operations can:
|
// Problem: Without mutex, concurrent operations can:
|
||||||
// 1. Thread A reads noun (connections: [1,2,3])
|
// 1. Thread A reads noun (connections: [1,2,3])
|
||||||
// 2. Thread B reads noun (connections: [1,2,3])
|
// 2. Thread B reads noun (connections: [1,2,3])
|
||||||
|
|
@ -3603,7 +3597,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
this.hnswLocks.set(lockKey, lockPromise)
|
this.hnswLocks.set(lockKey, lockPromise)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
// Use BaseStorage's getNoun (type-first paths)
|
||||||
// Read existing noun data (if exists)
|
// Read existing noun data (if exists)
|
||||||
const existingNoun = await this.getNoun(nounId)
|
const existingNoun = await this.getNoun(nounId)
|
||||||
|
|
||||||
|
|
@ -3625,7 +3619,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
connections: connectionsMap
|
connections: connectionsMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
|
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
|
||||||
await this.saveNoun(updatedNoun)
|
await this.saveNoun(updatedNoun)
|
||||||
} finally {
|
} finally {
|
||||||
// Release lock (ALWAYS runs, even if error thrown)
|
// Release lock (ALWAYS runs, even if error thrown)
|
||||||
|
|
@ -3636,7 +3630,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get HNSW graph data for a noun
|
* Get HNSW graph data for a noun
|
||||||
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
* Uses BaseStorage's getNoun (type-first paths)
|
||||||
*/
|
*/
|
||||||
public async getHNSWData(nounId: string): Promise<{
|
public async getHNSWData(nounId: string): Promise<{
|
||||||
level: number
|
level: number
|
||||||
|
|
@ -3666,7 +3660,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
* Save HNSW system data (entry point, max level)
|
* Save HNSW system data (entry point, max level)
|
||||||
* Storage path: system/hnsw-system.json
|
* Storage path: system/hnsw-system.json
|
||||||
*
|
*
|
||||||
* CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
|
* CRITICAL FIX: Optimistic locking with ETags to prevent race conditions
|
||||||
*/
|
*/
|
||||||
public async saveHNSWSystem(systemData: {
|
public async saveHNSWSystem(systemData: {
|
||||||
entryPointId: string | null
|
entryPointId: string | null
|
||||||
|
|
@ -3781,7 +3775,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set S3 lifecycle policy for automatic tier transitions and deletions (v4.0.0)
|
* Set S3 lifecycle policy for automatic tier transitions and deletions
|
||||||
* Automates cost optimization by moving old data to cheaper storage classes
|
* Automates cost optimization by moving old data to cheaper storage classes
|
||||||
*
|
*
|
||||||
* S3 Storage Classes:
|
* S3 Storage Classes:
|
||||||
|
|
@ -3976,7 +3970,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Enable S3 Intelligent-Tiering for automatic cost optimization (v4.0.0)
|
* Enable S3 Intelligent-Tiering for automatic cost optimization
|
||||||
* Automatically moves objects between access tiers based on usage patterns
|
* Automatically moves objects between access tiers based on usage patterns
|
||||||
*
|
*
|
||||||
* Intelligent-Tiering automatically saves up to 95% on storage costs:
|
* Intelligent-Tiering automatically saves up to 95% on storage costs:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
/**
|
/**
|
||||||
* DEPRECATED (v4.7.2): Backward compatibility stubs
|
* DEPRECATED: Backward compatibility stubs
|
||||||
* TODO: Remove in v4.7.3 after migrating s3CompatibleStorage
|
* TODO: Remove after migrating s3CompatibleStorage
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export class StorageCompatibilityLayer {
|
export class StorageCompatibilityLayer {
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -161,7 +161,7 @@ export class BlobStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* v5.7.5: Ensure compression is ready before write operations
|
* Ensure compression is ready before write operations
|
||||||
* Fixes race condition where write happens before async compression init completes
|
* Fixes race condition where write happens before async compression init completes
|
||||||
*/
|
*/
|
||||||
private async ensureCompressionReady(): Promise<void> {
|
private async ensureCompressionReady(): Promise<void> {
|
||||||
|
|
@ -206,7 +206,7 @@ export class BlobStorage {
|
||||||
return hash
|
return hash
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.7.5: Ensure compression is initialized before writing
|
// Ensure compression is initialized before writing
|
||||||
// Fixes race condition where write happens before async init completes
|
// Fixes race condition where write happens before async init completes
|
||||||
await this.ensureCompressionReady()
|
await this.ensureCompressionReady()
|
||||||
|
|
||||||
|
|
@ -223,7 +223,7 @@ export class BlobStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create metadata
|
// Create metadata
|
||||||
// v5.7.5: Store ACTUAL compression state, not intended
|
// Store ACTUAL compression state, not intended
|
||||||
// Prevents corruption if compression failed to initialize
|
// Prevents corruption if compression failed to initialize
|
||||||
const actualCompression = finalData === data ? 'none' : compression
|
const actualCompression = finalData === data ? 'none' : compression
|
||||||
const metadata: BlobMetadata = {
|
const metadata: BlobMetadata = {
|
||||||
|
|
@ -277,7 +277,7 @@ export class BlobStorage {
|
||||||
* @returns Blob data
|
* @returns Blob data
|
||||||
*/
|
*/
|
||||||
async read(hash: string, options: BlobReadOptions = {}): Promise<Buffer> {
|
async read(hash: string, options: BlobReadOptions = {}): Promise<Buffer> {
|
||||||
// v5.3.4 fix: Guard against NULL hash (sentinel value)
|
// Guard against NULL hash (sentinel value)
|
||||||
// NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree"
|
// NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree"
|
||||||
// It should NEVER be read as actual blob data
|
// It should NEVER be read as actual blob data
|
||||||
if (isNullHash(hash)) {
|
if (isNullHash(hash)) {
|
||||||
|
|
@ -309,7 +309,7 @@ export class BlobStorage {
|
||||||
metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`)
|
metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`)
|
||||||
if (metadataBuffer) {
|
if (metadataBuffer) {
|
||||||
prefix = tryPrefix
|
prefix = tryPrefix
|
||||||
// v5.10.1: Unwrap metadata before parsing (defense-in-depth)
|
// Unwrap metadata before parsing (defense-in-depth)
|
||||||
// Metadata should be JSON, but adapter might return wrapped format
|
// Metadata should be JSON, but adapter might return wrapped format
|
||||||
const unwrappedMetadata = unwrapBinaryData(metadataBuffer)
|
const unwrappedMetadata = unwrapBinaryData(metadataBuffer)
|
||||||
metadata = JSON.parse(unwrappedMetadata.toString())
|
metadata = JSON.parse(unwrappedMetadata.toString())
|
||||||
|
|
@ -338,8 +338,8 @@ export class BlobStorage {
|
||||||
finalData = await this.zstdDecompress(data)
|
finalData = await this.zstdDecompress(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// v5.10.1: Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
|
// Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
|
||||||
// Even though COW adapter should unwrap (v5.7.5), verify it happened and re-unwrap if needed
|
// Even though COW adapter should unwrap, verify it happened and re-unwrap if needed
|
||||||
// This prevents "Blob integrity check failed" errors if adapter returns wrapped data
|
// This prevents "Blob integrity check failed" errors if adapter returns wrapped data
|
||||||
// Uses shared binaryDataCodec utility (single source of truth for unwrap logic)
|
// Uses shared binaryDataCodec utility (single source of truth for unwrap logic)
|
||||||
const unwrappedData = unwrapBinaryData(finalData)
|
const unwrappedData = unwrapBinaryData(finalData)
|
||||||
|
|
|
||||||
|
|
@ -43,7 +43,7 @@ export interface CommitLogStats {
|
||||||
/**
|
/**
|
||||||
* CommitLog: Efficient commit history traversal and querying
|
* CommitLog: Efficient commit history traversal and querying
|
||||||
*
|
*
|
||||||
* Pure v5.0.0 implementation - modern, clean, fast
|
* Pure implementation - modern, clean, fast
|
||||||
*/
|
*/
|
||||||
export class CommitLog {
|
export class CommitLog {
|
||||||
private blobStorage: BlobStorage
|
private blobStorage: BlobStorage
|
||||||
|
|
|
||||||
|
|
@ -337,7 +337,7 @@ export class CommitObject {
|
||||||
let currentHash: string | null = startHash
|
let currentHash: string | null = startHash
|
||||||
let depth = 0
|
let depth = 0
|
||||||
|
|
||||||
// v5.3.4 fix: Guard against NULL hash (sentinel for "no parent")
|
// Guard against NULL hash (sentinel for "no parent")
|
||||||
// The initial commit has parent = null or NULL_HASH ('0000...0000')
|
// The initial commit has parent = null or NULL_HASH ('0000...0000')
|
||||||
// We must stop walking when we reach it, not try to read it
|
// We must stop walking when we reach it, not try to read it
|
||||||
while (currentHash && !isNullHash(currentHash)) {
|
while (currentHash && !isNullHash(currentHash)) {
|
||||||
|
|
|
||||||
|
|
@ -55,7 +55,7 @@ export interface RefUpdateOptions {
|
||||||
/**
|
/**
|
||||||
* RefManager: Manages branches, tags, and HEAD pointer
|
* RefManager: Manages branches, tags, and HEAD pointer
|
||||||
*
|
*
|
||||||
* Pure implementation for v5.0.0 - no backward compatibility
|
* Pure implementation - no backward compatibility
|
||||||
*/
|
*/
|
||||||
export class RefManager {
|
export class RefManager {
|
||||||
private adapter: COWStorageAdapter
|
private adapter: COWStorageAdapter
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,7 @@ export function unwrapBinaryData(data: any): Buffer {
|
||||||
/**
|
/**
|
||||||
* Wrap binary data for JSON storage
|
* Wrap binary data for JSON storage
|
||||||
*
|
*
|
||||||
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
|
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH!
|
||||||
* ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
|
* ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
|
||||||
* ⚠️ This function exists for legacy/compatibility only.
|
* ⚠️ This function exists for legacy/compatibility only.
|
||||||
*
|
*
|
||||||
|
|
@ -94,7 +94,7 @@ export function unwrapBinaryData(data: any): Buffer {
|
||||||
* This is FRAGILE because compressed binary can accidentally parse as valid JSON,
|
* This is FRAGILE because compressed binary can accidentally parse as valid JSON,
|
||||||
* causing blob integrity failures.
|
* causing blob integrity failures.
|
||||||
*
|
*
|
||||||
* v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
|
* SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
|
||||||
* - Keys with '-meta:' or 'ref:' prefix → Always JSON
|
* - Keys with '-meta:' or 'ref:' prefix → Always JSON
|
||||||
* - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
|
* - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
|
||||||
* No guessing needed!
|
* No guessing needed!
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js'
|
||||||
import { R2Storage } from './adapters/r2Storage.js'
|
import { R2Storage } from './adapters/r2Storage.js'
|
||||||
import { GcsStorage } from './adapters/gcsStorage.js'
|
import { GcsStorage } from './adapters/gcsStorage.js'
|
||||||
import { AzureBlobStorage } from './adapters/azureBlobStorage.js'
|
import { AzureBlobStorage } from './adapters/azureBlobStorage.js'
|
||||||
// TypeAwareStorageAdapter removed in v5.4.0 - type-aware now built into BaseStorage
|
// TypeAwareStorageAdapter removed - type-aware now built into BaseStorage
|
||||||
// FileSystemStorage is dynamically imported to avoid issues in browser environments
|
// FileSystemStorage is dynamically imported to avoid issues in browser environments
|
||||||
import { isBrowser } from '../utils/environment.js'
|
import { isBrowser } from '../utils/environment.js'
|
||||||
import { OperationConfig } from '../utils/operationUtils.js'
|
import { OperationConfig } from '../utils/operationUtils.js'
|
||||||
|
|
@ -94,7 +94,7 @@ export interface StorageOptions {
|
||||||
sessionToken?: string
|
sessionToken?: string
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialization mode for fast cold starts (v7.3.0+)
|
* Initialization mode for fast cold starts
|
||||||
*
|
*
|
||||||
* - `'auto'` (default): Progressive in cloud environments (Lambda),
|
* - `'auto'` (default): Progressive in cloud environments (Lambda),
|
||||||
* strict locally. Zero-config optimization.
|
* strict locally. Zero-config optimization.
|
||||||
|
|
@ -103,7 +103,6 @@ export interface StorageOptions {
|
||||||
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
||||||
* before init() returns.
|
* before init() returns.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
initMode?: 'progressive' | 'strict' | 'auto'
|
initMode?: 'progressive' | 'strict' | 'auto'
|
||||||
}
|
}
|
||||||
|
|
@ -215,7 +214,7 @@ export interface StorageOptions {
|
||||||
skipCountsFile?: boolean
|
skipCountsFile?: boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialization mode for fast cold starts (v7.3.0+)
|
* Initialization mode for fast cold starts
|
||||||
*
|
*
|
||||||
* - `'auto'` (default): Progressive in cloud environments (Cloud Run),
|
* - `'auto'` (default): Progressive in cloud environments (Cloud Run),
|
||||||
* strict locally. Zero-config optimization.
|
* strict locally. Zero-config optimization.
|
||||||
|
|
@ -224,7 +223,6 @@ export interface StorageOptions {
|
||||||
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
|
||||||
* before init() returns.
|
* before init() returns.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
initMode?: 'progressive' | 'strict' | 'auto'
|
initMode?: 'progressive' | 'strict' | 'auto'
|
||||||
}
|
}
|
||||||
|
|
@ -259,7 +257,7 @@ export interface StorageOptions {
|
||||||
sasToken?: string
|
sasToken?: string
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialization mode for fast cold starts (v7.3.0+)
|
* Initialization mode for fast cold starts
|
||||||
*
|
*
|
||||||
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
|
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
|
||||||
* strict locally. Zero-config optimization.
|
* strict locally. Zero-config optimization.
|
||||||
|
|
@ -268,7 +266,6 @@ export interface StorageOptions {
|
||||||
* - `'strict'`: Traditional blocking init. Validates container and loads counts
|
* - `'strict'`: Traditional blocking init. Validates container and loads counts
|
||||||
* before init() returns.
|
* before init() returns.
|
||||||
*
|
*
|
||||||
* @since v7.3.0
|
|
||||||
*/
|
*/
|
||||||
initMode?: 'progressive' | 'strict' | 'auto'
|
initMode?: 'progressive' | 'strict' | 'auto'
|
||||||
}
|
}
|
||||||
|
|
@ -388,7 +385,7 @@ export interface StorageOptions {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* COW (Copy-on-Write) configuration for instant fork() capability
|
* COW (Copy-on-Write) configuration for instant fork() capability
|
||||||
* v5.0.1: COW is now always enabled (automatic, zero-config)
|
* COW is now always enabled (automatic, zero-config)
|
||||||
*/
|
*/
|
||||||
branch?: string // Current branch name (default: 'main')
|
branch?: string // Current branch name (default: 'main')
|
||||||
enableCompression?: boolean // Enable zstd compression for COW blobs (default: true)
|
enableCompression?: boolean // Enable zstd compression for COW blobs (default: true)
|
||||||
|
|
@ -410,14 +407,14 @@ function getFileSystemPath(options: StorageOptions): string {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Configure COW (Copy-on-Write) options on a storage adapter (v5.4.0)
|
* Configure COW (Copy-on-Write) options on a storage adapter
|
||||||
* TypeAware is now built-in to all adapters, no wrapper needed!
|
* TypeAware is now built-in to all adapters, no wrapper needed!
|
||||||
*
|
*
|
||||||
* @param storage - The storage adapter
|
* @param storage - The storage adapter
|
||||||
* @param options - Storage options (for COW configuration)
|
* @param options - Storage options (for COW configuration)
|
||||||
*/
|
*/
|
||||||
function configureCOW(storage: any, options?: StorageOptions): void {
|
function configureCOW(storage: any, options?: StorageOptions): void {
|
||||||
// v5.0.1: COW will be initialized AFTER storage.init() in Brainy
|
// COW will be initialized AFTER storage.init() in Brainy
|
||||||
// Store COW options for later initialization
|
// Store COW options for later initialization
|
||||||
if (typeof storage.initializeCOW === 'function') {
|
if (typeof storage.initializeCOW === 'function') {
|
||||||
storage._cowOptions = {
|
storage._cowOptions = {
|
||||||
|
|
@ -672,10 +669,10 @@ export async function createStorage(
|
||||||
}
|
}
|
||||||
|
|
||||||
case 'type-aware':
|
case 'type-aware':
|
||||||
// v5.0.0: TypeAware is now the default for ALL adapters
|
// TypeAware is now the default for ALL adapters
|
||||||
// Redirect to the underlying type instead
|
// Redirect to the underlying type instead
|
||||||
console.warn(
|
console.warn(
|
||||||
'⚠️ type-aware is deprecated in v5.0.0 - TypeAware is now always enabled.'
|
'⚠️ type-aware is deprecated - TypeAware is now always enabled.'
|
||||||
)
|
)
|
||||||
console.warn(
|
console.warn(
|
||||||
' Just use the underlying storage type (e.g., "filesystem", "s3", etc.)'
|
' Just use the underlying storage type (e.g., "filesystem", "s3", etc.)'
|
||||||
|
|
@ -866,7 +863,7 @@ export async function createStorage(
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Export storage adapters (v5.4.0: TypeAware is now built-in, no separate export)
|
* Export storage adapters (TypeAware is now built-in, no separate export)
|
||||||
*/
|
*/
|
||||||
export {
|
export {
|
||||||
MemoryStorage,
|
MemoryStorage,
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue