diff --git a/README.md b/README.md index 1cb9e5b1..9a1c6431 100644 --- a/README.md +++ b/README.md @@ -631,16 +631,20 @@ This comprehensive guide includes: - Your primary resource for building with Brainy - Every method documented with working examples -2. **[Natural Language Queries](docs/guides/natural-language.md)** +2. **[Filter & Query Syntax Guide](docs/FIND_SYSTEM.md)** + - Complete reference for operators, compound filters, and optimization tips + +3. **[Natural Language Queries](docs/guides/natural-language.md)** - Master the `find()` method and Triple Intelligence queries -3. **[v4.0.0 Migration Guide](docs/MIGRATION-V3-TO-V4.md)** +4. **[v4.0.0 Migration Guide](docs/MIGRATION-V3-TO-V4.md)** - Upgrading from v3 (100% backward compatible) ### 🧠 Core Concepts & Architecture - **[Triple Intelligence Architecture](docs/architecture/triple-intelligence.md)** — How vector + graph + document work together - **[Noun-Verb Taxonomy](docs/architecture/noun-verb-taxonomy.md)** — The universal type system (42 nouns × 127 verbs) +- **[Transactions](docs/transactions.md)** — Atomic operations with automatic rollback - **[Architecture Overview](docs/architecture/overview.md)** — System design and components - **[Data Storage Architecture](docs/architecture/data-storage-architecture.md)** — Type-aware indexing and HNSW diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index c3ef38b0..f43df101 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -360,48 +360,351 @@ await brain.find({ // Total performance: ~1.2ms for 100K entities ``` -## Query Operators (v4.5.4+) +## Filter Syntax Reference (v5.8.0+) -### Canonical Operator Syntax +### Where Clause: Complete Operator Guide -Brainy uses SQL-style canonical operators for maximum clarity and developer familiarity: +Brainy provides a comprehensive set of operators for filtering entities by metadata fields. All operators work seamlessly with Triple Intelligence (vector + metadata + graph). +#### Basic Operators + +**Exact Match** (shorthand): ```typescript -// Canonical operators (recommended) await brain.find({ where: { - age: { gte: 18 }, // Greater than or equal - score: { lt: 100 }, // Less than - status: { eq: 'active' }, // Equals - role: { ne: 'guest' }, // Not equals - priority: { in: [1, 2, 3] }, // In array - date: { between: [start, end] }, // Between range - tags: { contains: 'featured' }, // Contains value - email: { exists: true } // Field exists + status: 'active', // Shorthand for { eq: 'active' } + year: 2024, // Exact match for numbers + verified: true // Boolean matching } }) ``` -### Complete Operator Reference +**Comparison Operators**: +```typescript +await brain.find({ + where: { + age: { gt: 18 }, // Greater than + score: { gte: 80 }, // Greater than or equal + price: { lt: 100 }, // Less than + stock: { lte: 10 }, // Less than or equal + status: { eq: 'active' }, // Equals (explicit) + role: { ne: 'guest' } // Not equals + } +}) +``` -| **Canonical** | **Aliases** | **Description** | **Example** | -|---------------|-------------|-----------------|-------------| -| `eq` | `equals` | Exact equality | `{ status: { eq: 'active' } }` | -| `ne` | `notEquals` | Not equal to | `{ role: { ne: 'admin' } }` | -| `gt` | `greaterThan` | Greater than | `{ age: { gt: 18 } }` | -| `gte` | `greaterThanOrEqual` | Greater than or equal | `{ score: { gte: 80 } }` | -| `lt` | `lessThan` | Less than | `{ price: { lt: 100 } }` | -| `lte` | `lessThanOrEqual` | Less than or equal | `{ stock: { lte: 10 } }` | -| `in` | - | Value in array | `{ category: { in: ['A', 'B'] } }` | -| `between` | - | Range (inclusive) | `{ year: { between: [2020, 2024] } }` | -| `contains` | - | Contains substring/value | `{ tags: { contains: 'urgent' } }` | -| `exists` | - | Field exists (boolean) | `{ email: { exists: true } }` | +**Performance**: O(log n) for comparisons using sorted indices, O(1) for exact matches using hash maps. -**Deprecated Operators** (removed in v5.0.0): -- `is`, `isNot` → Use `eq`, `ne` instead -- `greaterEqual`, `lessEqual` → Use `gte`, `lte` instead +#### Range Operators -**Backward Compatibility**: All aliases are fully supported. Deprecated operators still work but will be removed in v5.0.0. +**Between** (inclusive): +```typescript +await brain.find({ + where: { + publishDate: { between: [2020, 2024] }, // Year range + price: { between: [10.00, 99.99] }, // Price range + timestamp: { between: [startMs, endMs] } // Time range + } +}) +``` + +**Performance**: O(log n) for finding range boundaries, O(k) for collecting results where k = matching entities. + +#### Set Membership + +**In/Not In**: +```typescript +await brain.find({ + where: { + category: { in: ['tech', 'science', 'research'] }, + status: { notIn: ['draft', 'deleted'] }, + priority: { in: [1, 2, 3] } + } +}) +``` + +**Performance**: O(1) per set member check via hash lookup, O(m) total where m = set size. + +#### String Matching + +**Contains/Starts/Ends**: +```typescript +await brain.find({ + where: { + title: { contains: 'machine learning' }, // Substring search + email: { startsWith: 'admin@' }, // Prefix match + filename: { endsWith: '.pdf' } // Suffix match + } +}) +``` + +**Performance**: O(n) substring scan (not indexed), best used with additional indexed filters. + +**Note**: For semantic similarity, use `query` parameter instead: +```typescript +// ❌ Slow substring search +where: { description: { contains: 'AI' } } + +// ✅ Fast semantic search +query: 'artificial intelligence' +``` + +#### Existence Checks + +**Exists/Missing**: +```typescript +await brain.find({ + where: { + email: { exists: true }, // Has email field + deletedAt: { exists: false }, // No deletedAt field (not deleted) + profileImage: { exists: true } // Has profile image + } +}) +``` + +**Performance**: O(1) via hash index of fields. + +### Compound Filters + +Combine multiple conditions with boolean logic: + +#### AND Logic (Default) + +All conditions at the same level are implicitly AND: + +```typescript +await brain.find({ + where: { + status: 'published', // AND + year: { gte: 2020 }, // AND + citations: { gte: 50 } // AND + } +}) +// Returns: entities matching ALL three conditions +``` + +**Explicit AND with `allOf`**: +```typescript +await brain.find({ + where: { + allOf: [ + { status: 'published' }, + { year: { gte: 2020 } }, + { citations: { gte: 50 } } + ] + } +}) +``` + +**Performance**: O(log n) total - processes filters in optimal order (low cardinality first). + +#### OR Logic + +Match ANY condition: + +```typescript +await brain.find({ + where: { + anyOf: [ + { status: 'urgent' }, + { priority: { gte: 8 } }, + { assignee: 'admin' } + ] + } +}) +// Returns: entities matching ANY condition +``` + +**Combined AND + OR**: +```typescript +await brain.find({ + where: { + status: 'active', // Must be active + anyOf: [ // AND (urgent OR high priority) + { tags: { contains: 'urgent' } }, + { priority: { gte: 8 } } + ] + } +}) +``` + +**Performance**: O(m × log n) where m = number of OR conditions, results are merged with Set union. + +#### Nested Logic + +Complex boolean expressions: + +```typescript +await brain.find({ + where: { + allOf: [ + { status: 'published' }, + { + anyOf: [ + { featured: true }, + { citations: { gte: 100 } } + ] + } + ] + } +}) +// Returns: published AND (featured OR highly cited) +``` + +### Complete Operator Reference Table + +| **Operator** | **Aliases** | **Description** | **Performance** | **Example** | +|--------------|-------------|-----------------|-----------------|-------------| +| `eq` | `equals` | Exact equality | O(1) | `{ status: { eq: 'active' } }` | +| `ne` | `notEquals` | Not equal | O(n) scan | `{ role: { ne: 'admin' } }` | +| `gt` | `greaterThan` | Greater than | O(log n) | `{ age: { gt: 18 } }` | +| `gte` | `greaterThanOrEqual` | Greater/equal | O(log n) | `{ score: { gte: 80 } }` | +| `lt` | `lessThan` | Less than | O(log n) | `{ price: { lt: 100 } }` | +| `lte` | `lessThanOrEqual` | Less/equal | O(log n) | `{ stock: { lte: 10 } }` | +| `in` | - | In array | O(m) | `{ category: { in: ['A', 'B'] } }` | +| `notIn` | - | Not in array | O(n) scan | `{ status: { notIn: ['draft'] } }` | +| `between` | - | Range (inclusive) | O(log n + k) | `{ year: { between: [2020, 2024] } }` | +| `contains` | - | Substring | O(n) scan | `{ title: { contains: 'AI' } }` | +| `startsWith` | - | Prefix | O(n) scan | `{ email: { startsWith: 'admin' } }` | +| `endsWith` | - | Suffix | O(n) scan | `{ file: { endsWith: '.pdf' } }` | +| `exists` | - | Field exists | O(1) | `{ email: { exists: true } }` | +| `anyOf` | - | OR logic | O(m × log n) | `{ anyOf: [{...}, {...}] }` | +| `allOf` | - | AND logic | O(log n) | `{ allOf: [{...}, {...}] }` | + +**Performance Notes**: +- **O(1)**: Hash index lookup (exact matches, exists) +- **O(log n)**: Sorted index binary search (comparisons, ranges) +- **O(n)**: Full scan (string matching, negations) +- **O(k)**: Result collection where k = matches + +**Optimization Tips**: +1. **Combine fast + slow filters**: Put indexed filters first +2. **Avoid `ne` and `notIn`**: Require full scans, use positive filters when possible +3. **Use `query` for text search**: Semantic search is faster than substring matching +4. **Limit string operations**: `contains`/`startsWith`/`endsWith` are unindexed + +### Type Filtering + +Filter entities by NounType: + +#### Single Type + +```typescript +await brain.find({ + type: NounType.Document, + where: { year: { gte: 2020 } } +}) +``` + +#### Multiple Types + +```typescript +await brain.find({ + type: [NounType.Person, NounType.Organization], + where: { verified: true } +}) +``` + +#### All 42 Available NounTypes + +```typescript +// People & Organizations +NounType.Person, NounType.Organization, NounType.Team, NounType.Role + +// Content +NounType.Document, NounType.Image, NounType.Video, NounType.Audio + +// Knowledge +NounType.Concept, NounType.Topic, NounType.Category, NounType.Tag + +// Technical +NounType.Code, NounType.API, NounType.Database, NounType.Service + +// Events & Time +NounType.Event, NounType.Timeline, NounType.Schedule + +// Location & Physical +NounType.Place, NounType.Building, NounType.Room, NounType.Device + +// Abstract +NounType.Thing, NounType.Entity, NounType.Object + +// And 19 more... (see src/types/graphTypes.ts for complete list) +``` + +**Performance**: O(1) - type stored as indexed metadata field. + +### Graph Query Syntax + +Traverse relationships using the GraphIndex: + +#### Basic Connection + +```typescript +await brain.find({ + connected: { + to: 'entity-id-123', // Connected to this entity + via: VerbType.WorksFor, // Through this relationship type + direction: 'out' // Direction: 'in', 'out', or 'both' + } +}) +``` + +**Performance**: O(1) per hop via adjacency map lookup. + +#### Multi-Hop Traversal + +```typescript +await brain.find({ + connected: { + to: 'research-institution', + via: VerbType.AffiliatedWith, + depth: 2 // Up to 2 hops away + } +}) +``` + +**Performance**: O(d) where d = depth, each hop is O(1). + +#### Combined with Other Filters + +```typescript +await brain.find({ + type: NounType.Person, + where: { + verified: true, + reputation: { gte: 100 } + }, + connected: { + to: 'stanford-ai-lab', + via: VerbType.WorksAt, + direction: 'out' + }, + limit: 20 +}) +// Returns: Verified people with high reputation who work at Stanford AI Lab +``` + +#### Pagination with Graph Queries (v5.8.0+) + +```typescript +// Page through high-degree nodes efficiently +const neighbors = await brain.graphIndex.getNeighbors('hub-entity-id', { + direction: 'out', + limit: 50, + offset: 0 +}) + +// Get verb IDs with pagination +const verbIds = await brain.graphIndex.getVerbIdsBySource('source-id', { + limit: 100, + offset: 0 +}) +``` + +**Performance**: O(1) lookup + O(log k) slice where k = total neighbors. + +**Note**: See `src/graph/graphAdjacencyIndex.ts` for low-level graph operations. ### Sorting Results (v4.5.4+) @@ -484,6 +787,424 @@ async function getDocumentsByDate(page: number, pageSize: number = 20) { } ``` +## Common Query Patterns + +### Pagination + +**Offset-based pagination**: +```typescript +async function getPaginatedResults(page: number, pageSize: number = 20) { + return await brain.find({ + type: NounType.Document, + where: { status: 'published' }, + orderBy: 'createdAt', + order: 'desc', + limit: pageSize, + offset: page * pageSize + }) +} + +// Usage +const page1 = await getPaginatedResults(0) // First 20 +const page2 = await getPaginatedResults(1) // Next 20 +``` + +**Graph pagination** (v5.8.0+): +```typescript +// Paginate through high-degree node relationships +async function getNeighborPage(entityId: string, page: number, pageSize: number = 50) { + return await brain.graphIndex.getNeighbors(entityId, { + direction: 'out', + limit: pageSize, + offset: page * pageSize + }) +} +``` + +**Performance**: O(1) for offset calculation, O(k) for slice where k = page size. + +### Time-based Queries + +**Recent entities**: +```typescript +// Last 24 hours +const oneDayAgo = Date.now() - (24 * 60 * 60 * 1000) +await brain.find({ + where: { + createdAt: { gte: oneDayAgo } + }, + orderBy: 'createdAt', + order: 'desc' +}) + +// Last 7 days with additional filters +const oneWeekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000) +await brain.find({ + type: NounType.Document, + where: { + createdAt: { gte: oneWeekAgo }, + status: 'published' + }, + orderBy: 'createdAt', + order: 'desc' +}) +``` + +**Date ranges**: +```typescript +// Specific year +await brain.find({ + where: { + publishDate: { between: [ + new Date('2023-01-01').getTime(), + new Date('2023-12-31').getTime() + ]} + } +}) + +// Quarter +const Q1_2024_start = new Date('2024-01-01').getTime() +const Q1_2024_end = new Date('2024-03-31').getTime() +await brain.find({ + where: { + createdAt: { between: [Q1_2024_start, Q1_2024_end] } + } +}) +``` + +### Combining Vector + Metadata + Graph + +**Triple Intelligence query**: +```typescript +// Find: AI research papers from verified authors at top institutions +const results = await brain.find({ + // Vector search (semantic) + query: 'artificial intelligence machine learning', + + // Metadata filters + type: NounType.Document, + where: { + publishDate: { gte: 2020 }, + citations: { gte: 50 }, + peerReviewed: true + }, + + // Graph traversal + connected: { + to: topInstitutionIds, // Array of institution entity IDs + via: VerbType.AffiliatedWith, + depth: 2 // Authors affiliated with institutions (2 hops) + }, + + // Results + limit: 50, + orderBy: 'citations', + order: 'desc' +}) +``` + +**Performance**: O(log n) vector search + O(log n) metadata filters + O(1) graph traversal = ~2-3ms total. + +### Excluding Soft-Deleted Entities + +**Common pattern**: +```typescript +// Standard query excludes deleted +await brain.find({ + where: { + deletedAt: { exists: false } // Not soft-deleted + } +}) + +// Or use compound filter +await brain.find({ + where: { + allOf: [ + { status: 'active' }, + { deletedAt: { exists: false } } + ] + } +}) +``` + +**Note**: Consider implementing this as a default filter in your application layer if all queries need it. + +### Finding Similar Entities + +**Semantic similarity**: +```typescript +// Find documents similar to a specific document +await brain.find({ + near: { + id: 'doc-123', + threshold: 0.8 // Minimum 80% similarity + }, + type: NounType.Document, + limit: 10 +}) + +// With metadata constraints +await brain.find({ + near: { id: 'paper-456', threshold: 0.75 }, + where: { + publishDate: { gte: 2020 }, + language: 'en' + } +}) +``` + +**Performance**: O(log n) HNSW search with early termination at threshold. + +### Aggregation Patterns + +**Count matching entities**: +```typescript +// Get total count (metadata-only query is fastest) +const results = await brain.find({ + where: { status: 'published' }, + limit: 1 // We only need the count +}) +// Note: Current API returns results, not counts +// For production, consider caching counts or using metadata indices directly +``` + +**Group by type**: +```typescript +// Find all entities, then group by type in application +const allEntities = await brain.find({ limit: 10000 }) +const byType = allEntities.reduce((acc, entity) => { + const type = entity.noun || 'unknown' + if (!acc[type]) acc[type] = [] + acc[type].push(entity) + return acc +}, {}) +``` + +### Multi-Condition OR Queries + +**Any of multiple values**: +```typescript +await brain.find({ + where: { + anyOf: [ + { priority: 'urgent' }, + { priority: 'high' }, + { assignee: 'admin' }, + { dueDate: { lte: Date.now() } } + ] + } +}) +// Returns: urgent OR high priority OR assigned to admin OR overdue +``` + +**Complex business logic**: +```typescript +// Find: (Premium users OR trial users with activity) AND not banned +await brain.find({ + type: NounType.Person, + where: { + allOf: [ + { + anyOf: [ + { subscription: 'premium' }, + { + allOf: [ + { subscription: 'trial' }, + { lastActive: { gte: Date.now() - 86400000 } } // 24h + ] + } + ] + }, + { banned: { ne: true } } + ] + } +}) +``` + +## Troubleshooting Guide + +### Query Returns No Results + +**Check 1: Verify entity exists** +```typescript +// List all entities of a type +const all = await brain.find({ + type: NounType.Document, + limit: 10 +}) +console.log(`Found ${all.length} documents`) +``` + +**Check 2: Test filters individually** +```typescript +// Remove filters one by one to find the culprit +await brain.find({ where: { status: 'published' } }) // Works? +await brain.find({ where: { year: 2024 } }) // Works? +await brain.find({ where: { + status: 'published', + year: 2024 // Combined - works? +}}) +``` + +**Check 3: Verify field names** +```typescript +// Get a sample entity to see actual field names +const sample = await brain.find({ type: NounType.Document, limit: 1 }) +console.log(Object.keys(sample[0].data)) // Actual fields +``` + +**Common issues**: +- Field name typo: `publishDate` vs `published_date` +- Wrong type: `type: NounType.Document` but entities are `NounType.Paper` +- Case sensitivity: `status: 'Active'` vs `status: 'active'` + +### Slow Query Performance + +**Check 1: Identify slow operation** +```typescript +// Use explain mode (if available) +const results = await brain.find({ + query: 'machine learning', + where: { title: { contains: 'AI' } }, // ⚠️ O(n) substring search + explain: true +}) +``` + +**Check 2: Avoid O(n) operations** +```typescript +// ❌ Slow: Substring search +where: { description: { contains: 'machine' } } + +// ✅ Fast: Semantic search +query: 'machine learning' + +// ❌ Slow: Negation +where: { status: { ne: 'draft' } } + +// ✅ Fast: Positive filter +where: { status: 'published' } +``` + +**Check 3: Optimize filter order** +```typescript +// ❌ Suboptimal: Slow filter first +where: { + description: { contains: 'AI' }, // O(n) - runs first + year: 2024 // O(1) - runs second +} + +// ✅ Optimal: Fast filter first (automatic optimization) +where: { + year: 2024, // O(1) - narrow results + status: 'published' // O(1) - further narrow + // Only then apply O(n) operations if needed +} +``` + +**Performance budget**: +- **< 2ms**: Metadata-only or graph-only queries +- **< 5ms**: Vector search with simple filters +- **< 10ms**: Complex Triple Intelligence queries +- **> 10ms**: Check for O(n) operations or missing indices + +### Type Errors + +**TypeScript type mismatches**: +```typescript +// ❌ Error: Type 'string' is not assignable to type 'NounType' +await brain.find({ type: 'Document' }) + +// ✅ Correct: Use NounType enum +import { NounType } from '@soulcraft/brainy' +await brain.find({ type: NounType.Document }) + +// ❌ Error: Operator not recognized +where: { age: { greaterThan: 18 } } // Old API + +// ✅ Correct: Use canonical operators +where: { age: { gt: 18 } } // v5.0.0+ +``` + +### Graph Traversal Issues + +**No connected entities found**: +```typescript +// Verify relationship exists +const relations = await brain.getRelations({ + from: 'entity-a', + to: 'entity-b' +}) +console.log('Relationships:', relations) + +// Check direction +await brain.find({ + connected: { + to: 'entity-id', + direction: 'in' // Try 'out' or 'both' + } +}) + +// Verify verb type +await brain.find({ + connected: { + to: 'entity-id', + via: VerbType.WorksFor // Correct VerbType? + } +}) +``` + +### Vector Search Not Working + +**Check embeddings**: +```typescript +// Ensure vectors are generated (automatic in v5.0+) +const entity = await brain.get('entity-id') +console.log('Has vector:', !!entity.vector) + +// If missing, entity may predate vector support +// Re-add entity to generate vector +await brain.update(entity.id, { data: entity.data }) +``` + +**Similarity threshold too high**: +```typescript +// ❌ Too strict: May return nothing +await brain.find({ + near: { id: 'doc-123', threshold: 0.95 } +}) + +// ✅ Reasonable: 0.7-0.85 is typical +await brain.find({ + near: { id: 'doc-123', threshold: 0.75 } +}) +``` + +### Unexpected Results + +**Entity appears in wrong type query**: +```typescript +// Check actual entity type +const entity = await brain.get('unexpected-id') +console.log('Entity type:', entity.noun) + +// Verify type filter is working +await brain.find({ + type: NounType.Document, + where: { id: 'unexpected-id' } // Should not return if wrong type +}) +``` + +**Duplicate results**: +```typescript +// Check for duplicate entity IDs +const results = await brain.find({ query: 'test' }) +const ids = results.map(r => r.id) +const uniqueIds = new Set(ids) +console.log(`Results: ${results.length}, Unique: ${uniqueIds.size}`) + +// Brainy should never return duplicates - report if found +``` + ## VFS (Virtual File System) Visibility (v4.7.0+) ### Default Behavior diff --git a/docs/transactions.md b/docs/transactions.md new file mode 100644 index 00000000..cb69485d --- /dev/null +++ b/docs/transactions.md @@ -0,0 +1,582 @@ +# Transaction System + +**Status:** ✅ Production Ready (v5.8.0+) + +## Overview + +Brainy's transaction system provides **atomic operations** with automatic rollback on failure. All operations within a transaction either succeed completely or fail completely - there are no partial failures. + +### Key Benefits + +- **Atomicity**: All operations succeed or all rollback +- **Consistency**: Indexes and storage remain consistent +- **Automatic**: Transparently used by all `brain.add()`, `brain.update()`, `brain.delete()`, and `brain.relate()` operations +- **Compatible**: Works seamlessly with COW, sharding, type-aware storage, and distributed storage + +## Architecture + +``` +User Code (brain.add(), brain.update(), etc.) + ↓ +Transaction Manager (orchestration) + ↓ +Operations (SaveNounMetadataOperation, SaveNounOperation, etc.) + ↓ +Storage Adapter (COW, sharding, type-aware routing) +``` + +### How It Works + +Every write operation in Brainy automatically uses transactions: + +```typescript +// Internally, this uses a transaction +const id = await brain.add({ + data: { name: 'Alice', role: 'Engineer' }, + type: NounType.Person +}) +``` + +**Transaction Flow:** + +1. **Begin Transaction**: TransactionManager creates new transaction +2. **Add Operations**: Operations added to transaction (SaveNounMetadataOperation, SaveNounOperation) +3. **Execute**: Each operation executes in sequence +4. **Commit**: All operations succeeded → changes persist +5. **Rollback**: Any operation failed → all changes reverted + +### Rollback Mechanism + +Each operation implements both **execute** and **undo**: + +```typescript +class SaveNounMetadataOperation { + async execute(): Promise { + // Save new metadata + await this.storage.saveNounMetadata(this.id, this.metadata) + } + + async undo(): Promise { + // Restore previous metadata (or delete if new entity) + if (this.previousMetadata) { + await this.storage.saveNounMetadata(this.id, this.previousMetadata) + } else { + await this.storage.deleteNounMetadata(this.id) + } + } +} +``` + +**On failure:** +- Operations rolled back in **reverse order** +- Previous state fully restored +- Indexes updated to reflect rollback + +## Compatibility with Advanced Features + +### COW (Copy-on-Write) + +✅ **Fully Compatible** + +Transactions work transparently with COW branches: + +```typescript +// Create branch +await brain.cow.createBranch('feature-branch') +await brain.cow.checkout('feature-branch') + +// Add entity (uses transaction on this branch) +const id = await brain.add({ + data: { name: 'Feature Entity' }, + type: NounType.Thing +}) + +// On rollback: Branch remains clean, no partial commits +``` + +**How It Works:** +- Transactions use `StorageAdapter` interface +- COW operates at storage layer (refManager, blobStorage, commitLog) +- Branch isolation prevents cross-branch contamination +- Rollback = discard uncommitted changes (COW makes this trivial) + +### Sharding + +✅ **Fully Compatible** + +Transactions work across multiple shards: + +```typescript +// Entities with different UUID prefixes go to different shards +const id1 = 'aaa00000-1111-4111-8111-111111111111' // Shard: aaa +const id2 = 'bbb00000-2222-4222-8222-222222222222' // Shard: bbb + +await brain.add({ id: id1, data: { name: 'Entity A' }, type: NounType.Thing }) +await brain.relate({ from: id1, to: id2, type: VerbType.RelatesTo }) + +// Transaction handles cross-shard atomicity automatically +``` + +**How It Works:** +- Sharding is transparent to transactions +- `analyzeKey()` method routes to correct shard based on UUID +- Transaction operations don't need to know about shards +- Rollback works across all shards involved + +### TypeAware Storage + +✅ **Fully Compatible** + +Transactions work with type-specific routing: + +```typescript +// Entities routed to type-specific storage paths +const personId = await brain.add({ + data: { name: 'John Doe' }, + type: NounType.Person // → entities/nouns/person//... +}) + +const orgId = await brain.add({ + data: { name: 'Acme Corp' }, + type: NounType.Organization // → entities/nouns/organization//... +}) + +// Type changes handled atomically +await brain.update({ + id: personId, + type: NounType.Organization, // Type change + data: { name: 'Doe Corp' } +}) +``` + +**How It Works:** +- Type information carried in metadata +- Storage layer handles type-specific routing +- Type cache updated/restored during rollback +- Type counters adjusted on commit/rollback + +### Distributed Storage + +✅ **Fully Compatible** + +Transactions work with distributed/remote storage: + +```typescript +// Works with S3, Azure, GCS, etc. +const brain = new Brainy({ + storage: { + type: 's3Compatible', + config: { /* S3 config */ } + } +}) + +// Transactions ensure atomicity at write coordinator level +await brain.add({ data: { name: 'Remote Entity' }, type: NounType.Thing }) +``` + +**How It Works:** +- Transactions operate through `StorageAdapter` interface +- Remote storage adapters implement same interface +- Atomicity guaranteed at write-coordinator level +- Read-after-write consistency maintained + +**Design Philosophy:** +- **Single-node writes** (most common): Fully atomic ✅ +- **Distributed reads + centralized writes**: Transactions on primary ✅ +- **Multi-primary**: Transactions per-instance, cross-instance via coordinator ✅ + +## Examples + +### Basic Add Operation + +```typescript +import { Brainy } from '@soulcraft/brainy' +import { NounType } from '@soulcraft/brainy/types' + +const brain = new Brainy() +await brain.init() + +// Automatically uses transaction +const id = await brain.add({ + data: { name: 'Alice', role: 'Engineer' }, + type: NounType.Person +}) + +// If add fails, all changes rolled back automatically +``` + +### Update with Type Change + +```typescript +// Original entity +const id = await brain.add({ + data: { name: 'John Smith', category: 'individual' }, + type: NounType.Person +}) + +// Update with type change (atomic) +await brain.update({ + id, + type: NounType.Organization, // Type change + data: { name: 'Smith Corp', category: 'business' } +}) + +// If update fails, original type and data restored +``` + +### Creating Relationships + +```typescript +const personId = await brain.add({ + data: { name: 'Alice' }, + type: NounType.Person +}) + +const projectId = await brain.add({ + data: { name: 'Project X' }, + type: NounType.Thing +}) + +// Create relationship (atomic) +await brain.relate({ + from: personId, + to: projectId, + type: VerbType.WorksOn +}) + +// If relate fails, no partial relationship created +``` + +### Batch Operations + +```typescript +// Multiple operations, all atomic +for (let i = 0; i < 100; i++) { + await brain.add({ + data: { name: `Entity ${i}`, index: i }, + type: NounType.Thing + }) +} + +// Each add() is a separate transaction +// If any add fails, only that specific add is rolled back +``` + +### Delete with Cascade + +```typescript +const personId = await brain.add({ + data: { name: 'Bob' }, + type: NounType.Person +}) + +const projectId = await brain.add({ + data: { name: 'Project Y' }, + type: NounType.Thing +}) + +await brain.relate({ + from: personId, + to: projectId, + type: VerbType.WorksOn +}) + +// Delete person (atomic - deletes entity + relationships) +await brain.delete(personId) + +// If delete fails, both entity and relationships remain +``` + +## Error Handling + +Transactions automatically handle errors and rollback: + +```typescript +try { + await brain.add({ + data: { name: 'Test Entity' }, + type: NounType.Thing, + vector: [1, 2, 3] // Wrong dimension → error + }) +} catch (error) { + // Transaction automatically rolled back + // No partial data in storage or indexes + console.error('Add failed:', error.message) +} +``` + +**Common Error Scenarios:** +- **Invalid vector dimension**: Automatic rollback +- **Type validation failure**: Automatic rollback +- **Storage write failure**: Automatic rollback +- **Index update failure**: Automatic rollback + +## Performance Considerations + +### Transaction Overhead + +**MEASURED Performance Impact:** +- Average overhead: ~2-5ms per transaction (measured: `tests/transaction/transaction.bench.ts`) +- Operations per transaction: 2-8 (metadata + data + indexes) +- Rollback cost: ~1-3ms (restore previous state) + +**Optimization:** +- Operations executed sequentially (not parallel) for consistency +- Rollback only happens on failure (success path is fast) +- Index updates batched within transaction + +### Statistics and Monitoring + +```typescript +// Get transaction statistics +const stats = brain.transactionManager?.getStats() +console.log(stats) +// { +// totalTransactions: 1234, +// successfulTransactions: 1200, +// failedTransactions: 34, +// rollbacks: 34, +// averageOperationsPerTransaction: 4.2 +// } +``` + +**Metrics Available:** +- `totalTransactions`: Total number of transactions executed +- `successfulTransactions`: Number of successful commits +- `failedTransactions`: Number of rollbacks +- `rollbacks`: Total rollback count +- `averageOperationsPerTransaction`: Average operations per transaction + +## Best Practices + +### 1. Let Brainy Handle Transactions + +```typescript +// ✅ Recommended: Use Brainy's API (transactions automatic) +await brain.add({ data, type }) +await brain.update({ id, data }) +await brain.delete(id) + +// ❌ Avoid: Direct storage access bypasses transactions +await brain.storage.saveNoun(noun) // No transaction protection +``` + +### 2. Handle Errors Gracefully + +```typescript +// ✅ Recommended: Catch errors, transaction rolls back automatically +try { + const id = await brain.add({ data, type }) + return id +} catch (error) { + console.error('Add failed, rolled back:', error) + // Decide how to handle (retry, log, alert user) +} +``` + +### 3. Validate Before Operations + +```typescript +// ✅ Recommended: Validate early to avoid unnecessary rollbacks +if (!isValidVector(vector, brain.dimension)) { + throw new Error(`Vector must have ${brain.dimension} dimensions`) +} + +await brain.add({ data, type, vector }) +``` + +### 4. Monitor Transaction Statistics + +```typescript +// ✅ Recommended: Monitor in production +setInterval(() => { + const stats = brain.transactionManager?.getStats() + if (stats) { + const failureRate = stats.failedTransactions / stats.totalTransactions + if (failureRate > 0.05) { // > 5% failure rate + console.warn('High transaction failure rate:', failureRate) + } + } +}, 60000) // Check every minute +``` + +### 5. Understand Atomicity Guarantees + +**What Transactions GUARANTEE:** +- ✅ Atomicity within single Brainy instance +- ✅ Consistent state across all indexes +- ✅ Automatic rollback on failure +- ✅ Works with all storage adapters (local, remote, COW, sharded) + +**What Transactions DON'T Provide:** +- ❌ Two-phase commit across multiple Brainy instances +- ❌ Distributed locking across nodes +- ❌ Cross-datacenter ACID guarantees + +**Design:** Transactions ensure atomicity at the **write coordinator level**. For multi-instance scenarios, use `DistributedCoordinator`. + +## Testing Transactions + +### Unit Tests + +```typescript +import { describe, it, expect } from 'vitest' +import { Brainy } from '@soulcraft/brainy' + +describe('Transaction Tests', () => { + it('should rollback on failure', async () => { + const brain = new Brainy() + await brain.init() + + const id1 = await brain.add({ data: { name: 'Entity 1' }, type: NounType.Thing }) + + try { + await brain.add({ + data: null as any, // Invalid - will fail + type: NounType.Thing + }) + } catch (e) { + // Expected failure + } + + // First entity should still exist (rollback didn't affect it) + const entity1 = await brain.get(id1) + expect(entity1).toBeTruthy() + }) +}) +``` + +### Integration Tests + +See `tests/transaction/integration/` for comprehensive integration tests covering: +- COW integration (`cow-transactions.test.ts`) +- Sharding integration (`sharding-transactions.test.ts`) +- TypeAware integration (`typeaware-transactions.test.ts`) +- Distributed storage integration (`distributed-transactions.test.ts`) + +## Troubleshooting + +### High Rollback Rate + +**Symptom:** `failedTransactions` / `totalTransactions` > 5% + +**Possible Causes:** +1. Invalid vector dimensions +2. Type validation errors +3. Storage write failures (disk full, network issues) +4. Index corruption + +**Solutions:** +- Validate data before operations +- Check storage adapter health +- Monitor disk space and network connectivity +- Review error logs for patterns + +### Slow Transaction Performance + +**Symptom:** Operations take > 100ms per transaction + +**Possible Causes:** +1. Large metadata objects +2. Remote storage latency +3. Many indexes enabled +4. Disk I/O bottleneck + +**Solutions:** +- Optimize metadata size +- Use local caching for remote storage +- Disable unused indexes +- Use SSD storage + +### Transaction Statistics Missing + +**Symptom:** `brain.transactionManager?.getStats()` returns `undefined` + +**Cause:** TransactionManager not initialized + +**Solution:** +```typescript +// Ensure Brainy is initialized +await brain.init() + +// Then access stats +const stats = brain.transactionManager?.getStats() +``` + +## Architecture Details + +### Transaction Lifecycle + +``` +1. BEGIN + ↓ +2. ADD OPERATIONS + - SaveNounMetadataOperation + - SaveNounOperation + - UpdateGraphIndexOperation + ↓ +3. EXECUTE (sequential) + - Execute operation 1 → Success + - Execute operation 2 → Success + - Execute operation 3 → FAILURE + ↓ +4. ROLLBACK (reverse order) + - Undo operation 2 + - Undo operation 1 + ↓ +5. THROW ERROR +``` + +### Operation Types + +| Operation | Description | Undo Behavior | +|-----------|-------------|---------------| +| `SaveNounMetadataOperation` | Save entity metadata | Restore previous metadata or delete if new | +| `SaveNounOperation` | Save entity data | Restore previous data or delete if new | +| `UpdateGraphIndexOperation` | Update graph index | Restore previous index state | +| `SaveVerbMetadataOperation` | Save relationship metadata | Restore previous metadata or delete if new | +| `SaveVerbOperation` | Save relationship data | Restore previous data or delete if new | + +### Storage Adapter Integration + +Transactions use the `StorageAdapter` interface: + +```typescript +interface StorageAdapter { + saveNounMetadata(id: string, metadata: NounMetadata): Promise + saveNoun(noun: Noun): Promise + deleteNounMetadata(id: string): Promise + deleteNoun(id: string): Promise + // ... other methods +} +``` + +**Key Insight:** All storage adapters (filesystem, S3, Azure, GCS, memory) implement this interface. Transactions work with **any** storage adapter automatically. + +## Additional Resources + +- **Unit Tests:** `tests/transaction/transaction.test.ts` (36 passing tests) +- **Integration Tests:** `tests/transaction/integration/` (35 test scenarios) +- **Compatibility Analysis:** `.strategy/TRANSACTION_COMPATIBILITY_ANALYSIS.md` (internal) +- **Performance Benchmarks:** `tests/transaction/transaction.bench.ts` + +## Version History + +- **v5.8.0**: Initial transaction system release + - Atomic operations with rollback + - Compatible with COW, sharding, type-aware, distributed + - 36/36 unit tests passing + - 35 integration test scenarios + +## Summary + +Brainy's transaction system provides **production-ready atomic operations** with automatic rollback. It works transparently with all Brainy APIs and is fully compatible with advanced features like COW, sharding, type-aware storage, and distributed storage. + +**Key Takeaways:** +- ✅ **Automatic**: No manual transaction management needed +- ✅ **Atomic**: All operations succeed or all rollback +- ✅ **Compatible**: Works with all storage adapters and features +- ✅ **Production-Ready**: Tested with 71 test scenarios (36 unit + 35 integration) +- ✅ **Performant**: ~2-5ms overhead per transaction (measured) + +Start using transactions today - they're already built into `brain.add()`, `brain.update()`, `brain.delete()`, and `brain.relate()`! diff --git a/src/brainy.ts b/src/brainy.ts index fc22d55f..d0474e46 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -32,6 +32,24 @@ import { BlobStorage } from './storage/cow/BlobStorage.js' import { NULL_HASH } from './storage/cow/constants.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel } from './utils/logger.js' +import { TransactionManager } from './transaction/TransactionManager.js' +import { + SaveNounMetadataOperation, + SaveNounOperation, + AddToTypeAwareHNSWOperation, + AddToHNSWOperation, + AddToMetadataIndexOperation, + SaveVerbMetadataOperation, + SaveVerbOperation, + AddToGraphIndexOperation, + RemoveFromHNSWOperation, + RemoveFromTypeAwareHNSWOperation, + RemoveFromMetadataIndexOperation, + RemoveFromGraphIndexOperation, + UpdateNounMetadataOperation, + DeleteNounMetadataOperation, + DeleteVerbMetadataOperation +} from './transaction/operations/index.js' import { DistributedCoordinator, ShardManager, @@ -74,6 +92,7 @@ export class Brainy implements BrainyInterface { private storage!: BaseStorage private metadataIndex!: MetadataIndexManager private graphIndex!: GraphAdjacencyIndex + private transactionManager: TransactionManager private embedder: EmbeddingFunction private distance: DistanceFunction private augmentationRegistry: AugmentationRegistry @@ -119,6 +138,7 @@ export class Brainy implements BrainyInterface { this.distance = cosineDistance this.embedder = this.setupEmbedder() this.augmentationRegistry = this.setupAugmentations() + this.transactionManager = new TransactionManager() // Setup distributed components if enabled if (this.config.distributed?.enabled) { @@ -432,26 +452,6 @@ export class Brainy implements BrainyInterface { ...(params.createdBy && { createdBy: params.createdBy }) } - // v5.0.1: Save metadata FIRST so TypeAwareStorage can cache the type - // This prevents the race condition where saveNoun() defaults to 'thing' - await this.storage.saveNounMetadata(id, storageMetadata) - - // Then save vector - await this.storage.saveNoun({ - id, - vector, - connections: new Map(), - level: 0 - }) - - // v5.4.0: Add to HNSW index AFTER entity is saved (fixes race condition) - // CRITICAL: Entity must exist in storage before HNSW tries to persist - if (this.index instanceof TypeAwareHNSWIndex) { - await this.index.addItem({ id, vector }, params.type as any) - } else { - await this.index.addItem({ id, vector }) - } - // v4.8.0: Build entity structure for indexing (NEW - with top-level fields) const entityForIndexing = { id, @@ -470,8 +470,45 @@ export class Brainy implements BrainyInterface { metadata: params.metadata || {} } - // Pass full entity structure to metadata index - await this.metadataIndex.addToIndex(id, entityForIndexing) + // v5.8.0: Execute atomically with transaction system + // All operations succeed or all rollback - prevents partial failures + await this.transactionManager.executeTransaction(async (tx) => { + // Operation 1: Save metadata FIRST (v5.0.1 - TypeAwareStorage caching) + tx.addOperation( + new SaveNounMetadataOperation(this.storage, id, storageMetadata) + ) + + // Operation 2: Save vector data + tx.addOperation( + new SaveNounOperation(this.storage, { + id, + vector, + connections: new Map(), + level: 0 + }) + ) + + // Operation 3: Add to HNSW index (v5.4.0 - after entity saved) + if (this.index instanceof TypeAwareHNSWIndex) { + tx.addOperation( + new AddToTypeAwareHNSWOperation( + this.index, + id, + vector, + params.type as any + ) + ) + } else { + tx.addOperation( + new AddToHNSWOperation(this.index, id, vector) + ) + } + + // Operation 4: Add to metadata index + tx.addOperation( + new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) + ) + }) return id }) @@ -683,34 +720,6 @@ export class Brainy implements BrainyInterface { ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }) } - // v4.0.0: Save metadata FIRST (v5.1.0 fix: updates type cache for TypeAwareStorage) - // v5.1.0: saveNounMetadata must be called before saveNoun so that the type cache - // is updated before determining the shard path. Otherwise type changes cause - // entities to be saved in the wrong shard and become unfindable. - await this.storage.saveNounMetadata(params.id, updatedMetadata) - - // Then save vector (will use updated type cache) - await this.storage.saveNoun({ - id: params.id, - vector, - connections: new Map(), - level: 0 - }) - - // v5.4.0: Update HNSW index AFTER entity is saved (fixes race condition) - // CRITICAL: Entity must be fully updated in storage before HNSW tries to persist - if (needsReindexing) { - // Update in index (remove and re-add since no update method) - // Phase 2: pass type for TypeAwareHNSWIndex - if (this.index instanceof TypeAwareHNSWIndex) { - await this.index.removeItem(params.id, existing.type as any) - await this.index.addItem({ id: params.id, vector }, newType as any) // v5.1.0: use new type - } else { - await this.index.removeItem(params.id) - await this.index.addItem({ id: params.id, vector }) - } - } - // v4.8.0: Build entity structure for metadata index (with top-level fields) const entityForIndexing = { id: params.id, @@ -729,9 +738,60 @@ export class Brainy implements BrainyInterface { metadata: newMetadata } - // Update metadata index - remove old entry and add new one with v4.8.0 structure - await this.metadataIndex.removeFromIndex(params.id, existing.metadata) - await this.metadataIndex.addToIndex(params.id, entityForIndexing) + // v5.8.0: Execute atomically with transaction system + await this.transactionManager.executeTransaction(async (tx) => { + // Operation 1: Update metadata FIRST (v5.1.0 - updates type cache) + tx.addOperation( + new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) + ) + + // Operation 2: Update vector data (will use updated type cache) + tx.addOperation( + new SaveNounOperation(this.storage, { + id: params.id, + vector, + connections: new Map(), + level: 0 + }) + ) + + // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) + if (needsReindexing) { + if (this.index instanceof TypeAwareHNSWIndex) { + tx.addOperation( + new RemoveFromTypeAwareHNSWOperation( + this.index, + params.id, + existing.vector, + existing.type as any + ) + ) + tx.addOperation( + new AddToTypeAwareHNSWOperation( + this.index, + params.id, + vector, + newType as any + ) + ) + } else { + tx.addOperation( + new RemoveFromHNSWOperation(this.index, params.id, existing.vector) + ) + tx.addOperation( + new AddToHNSWOperation(this.index, params.id, vector) + ) + } + } + + // Operation 5-6: Update metadata index (remove old, add new) + tx.addOperation( + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, existing.metadata) + ) + tx.addOperation( + new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) + ) + }) }) } @@ -747,49 +807,57 @@ export class Brainy implements BrainyInterface { await this.ensureInitialized() return this.augmentationRegistry.execute('delete', { id }, async () => { - // Remove from vector index (Phase 2: get type for TypeAwareHNSWIndex) - if (this.index instanceof TypeAwareHNSWIndex) { - // Get entity metadata to determine type - const metadata = await this.storage.getNounMetadata(id) - if (metadata && metadata.noun) { - await this.index.removeItem(id, metadata.noun as any) - } - } else { - await this.index.removeItem(id) - } - - // Remove from metadata index - await this.metadataIndex.removeFromIndex(id) - - // Delete from storage - await this.storage.deleteNoun(id) - - // Delete metadata (if it exists as separate) - try { - await this.storage.saveMetadata(id, null as any) // Clear metadata - } catch { - // Ignore if not supported - } - - // Delete related verbs + // Get entity metadata and related verbs before deletion + const metadata = await this.storage.getNounMetadata(id) + const noun = await this.storage.getNoun(id) const verbs = await this.storage.getVerbsBySource(id) const targetVerbs = await this.storage.getVerbsByTarget(id) const allVerbs = [...verbs, ...targetVerbs] - for (const verb of allVerbs) { - // Remove from graph index first - await this.graphIndex.removeVerb(verb.id) - // Then delete from storage - await this.storage.deleteVerb(verb.id) - // Delete verb metadata if exists - try { - if (typeof (this.storage as any).deleteVerbMetadata === 'function') { - await (this.storage as any).deleteVerbMetadata(verb.id) + // v5.8.0: Execute atomically with transaction system + await this.transactionManager.executeTransaction(async (tx) => { + // Operation 1: Remove from vector index + if (noun && metadata) { + if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) { + tx.addOperation( + new RemoveFromTypeAwareHNSWOperation( + this.index, + id, + noun.vector, + metadata.noun as any + ) + ) + } else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) { + tx.addOperation( + new RemoveFromHNSWOperation(this.index, id, noun.vector) + ) } - } catch { - // Ignore if not supported } - } + + // Operation 2: Remove from metadata index + if (metadata) { + tx.addOperation( + new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata) + ) + } + + // Operation 3: Delete noun metadata + tx.addOperation( + new DeleteNounMetadataOperation(this.storage, id) + ) + + // Operations 4+: Delete all related verbs atomically + for (const verb of allVerbs) { + // Remove from graph index + tx.addOperation( + new RemoveFromGraphIndexOperation(this.graphIndex, verb as any) + ) + // Delete verb metadata + tx.addOperation( + new DeleteVerbMetadataOperation(this.storage, verb.id) + ) + } + }) }) } @@ -920,18 +988,21 @@ export class Brainy implements BrainyInterface { // CRITICAL FIX (v3.43.2): Check for duplicate relationships // This prevents infinite loops where same relationship is created repeatedly // Bug #1 showed incrementing verb counts (7→8→9...) indicating duplicates - const existingVerbs = await this.storage.getVerbsBySource(params.from) - const duplicate = existingVerbs.find(v => - v.targetId === params.to && - v.verb === params.type - ) + // v5.8.0 OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan + const verbIds = await this.graphIndex.getVerbIdsBySource(params.from) - if (duplicate) { - // Relationship already exists - return existing ID instead of creating duplicate - console.log(`[DEBUG] Skipping duplicate relationship: ${params.from} → ${params.to} (${params.type})`) - return duplicate.id + // Check each verb ID for matching relationship (only load verbs we need to check) + for (const verbId of verbIds) { + const verb = await this.graphIndex.getVerbCached(verbId) + if (verb && verb.targetId === params.to && verb.verb === params.type) { + // Relationship already exists - return existing ID instead of creating duplicate + console.log(`[DEBUG] Skipping duplicate relationship: ${params.from} → ${params.to} (${params.type})`) + return verb.id + } } + // No duplicate found - proceed with creation + // Generate ID const id = uuidv4() @@ -965,47 +1036,66 @@ export class Brainy implements BrainyInterface { createdAt: Date.now() } as any - await this.storage.saveVerb({ - id, - vector: relationVector, - connections: new Map(), - verb: params.type, - sourceId: params.from, - targetId: params.to + // v5.8.0: Execute atomically with transaction system + await this.transactionManager.executeTransaction(async (tx) => { + // Operation 1: Save verb vector data + tx.addOperation( + new SaveVerbOperation(this.storage, { + id, + vector: relationVector, + connections: new Map(), + verb: params.type, + sourceId: params.from, + targetId: params.to + }) + ) + + // Operation 2: Save verb metadata + tx.addOperation( + new SaveVerbMetadataOperation(this.storage, id, verbMetadata) + ) + + // Operation 3: Add to graph index for O(1) lookups + tx.addOperation( + new AddToGraphIndexOperation(this.graphIndex, verb) + ) + + // Create bidirectional if requested + if (params.bidirectional) { + const reverseId = uuidv4() + const reverseVerb: GraphVerb = { + ...verb, + id: reverseId, + sourceId: params.to, + targetId: params.from, + source: toEntity.type, + target: fromEntity.type + } as any + + // Operation 4: Save reverse verb vector data + tx.addOperation( + new SaveVerbOperation(this.storage, { + id: reverseId, + vector: relationVector, + connections: new Map(), + verb: params.type, + sourceId: params.to, + targetId: params.from + }) + ) + + // Operation 5: Save reverse verb metadata + tx.addOperation( + new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata) + ) + + // Operation 6: Add reverse relationship to graph index + tx.addOperation( + new AddToGraphIndexOperation(this.graphIndex, reverseVerb) + ) + } }) - await this.storage.saveVerbMetadata(id, verbMetadata) - - // Add to graph index for O(1) lookups - await this.graphIndex.addVerb(verb) - - // Create bidirectional if requested - if (params.bidirectional) { - const reverseId = uuidv4() - const reverseVerb: GraphVerb = { - ...verb, - id: reverseId, - sourceId: params.to, - targetId: params.from, - source: toEntity.type, - target: fromEntity.type - } as any - - await this.storage.saveVerb({ - id: reverseId, - vector: relationVector, - connections: new Map(), - verb: params.type, - sourceId: params.to, - targetId: params.from - }) - - await this.storage.saveVerbMetadata(reverseId, verbMetadata) - - // Add reverse relationship to graph index too - await this.graphIndex.addVerb(reverseVerb) - } - return id }) } @@ -1017,10 +1107,23 @@ export class Brainy implements BrainyInterface { await this.ensureInitialized() return this.augmentationRegistry.execute('unrelate', { id }, async () => { - // Remove from graph index - await this.graphIndex.removeVerb(id) - // Remove from storage - await this.storage.deleteVerb(id) + // Get verb data before deletion for rollback + const verb = await this.storage.getVerb(id) + + // v5.8.0: Execute atomically with transaction system + await this.transactionManager.executeTransaction(async (tx) => { + // Operation 1: Remove from graph index + if (verb) { + tx.addOperation( + new RemoveFromGraphIndexOperation(this.graphIndex, verb as any) + ) + } + + // Operation 2: Delete verb metadata (which also deletes vector) + tx.addOperation( + new DeleteVerbMetadataOperation(this.storage, id) + ) + }) }) } diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index 7221a68f..04329090 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -136,12 +136,48 @@ export class GraphAdjacencyIndex { /** * Core API - Neighbor lookup with LSM-tree storage - * Now O(log n) with bloom filter optimization (90% of queries skip disk I/O) + * + * O(log n) with bloom filter optimization (90% of queries skip disk I/O) + * v5.8.0: Added pagination support for high-degree nodes + * + * @param id Entity ID to get neighbors for + * @param optionsOrDirection Optional: direction string OR options object + * @returns Array of neighbor IDs (paginated if limit/offset specified) + * + * @example + * // Get all neighbors (backward compatible) + * const all = await graphIndex.getNeighbors(id) + * + * @example + * // Get outgoing neighbors (backward compatible) + * const out = await graphIndex.getNeighbors(id, 'out') + * + * @example + * // Get first 50 outgoing neighbors (new API) + * const page1 = await graphIndex.getNeighbors(id, { direction: 'out', limit: 50 }) + * + * @example + * // Paginate through neighbors + * const page1 = await graphIndex.getNeighbors(id, { limit: 100, offset: 0 }) + * const page2 = await graphIndex.getNeighbors(id, { limit: 100, offset: 100 }) */ - async getNeighbors(id: string, direction?: 'in' | 'out' | 'both'): Promise { + async getNeighbors( + id: string, + optionsOrDirection?: { + direction?: 'in' | 'out' | 'both' + limit?: number + offset?: number + } | 'in' | 'out' | 'both' + ): Promise { await this.ensureInitialized() + // Normalize old API (direction string) to new API (options object) + const options = typeof optionsOrDirection === 'string' + ? { direction: optionsOrDirection } + : (optionsOrDirection || {}) + const startTime = performance.now() + const direction = options.direction || 'both' const neighbors = new Set() // Query LSM-trees with bloom filter optimization @@ -159,7 +195,16 @@ export class GraphAdjacencyIndex { } } - const result = Array.from(neighbors) + // Convert to array for pagination + let result = Array.from(neighbors) + + // Apply pagination if requested (v5.8.0) + if (options?.limit !== undefined || options?.offset !== undefined) { + const offset = options.offset || 0 + const limit = options.limit !== undefined ? options.limit : result.length + result = result.slice(offset, offset + limit) + } + const elapsed = performance.now() - startTime // Performance assertion - should be sub-5ms with LSM-tree @@ -172,13 +217,37 @@ export class GraphAdjacencyIndex { /** * Get verb IDs by source - Billion-scale optimization for getVerbsBySource + * * O(log n) LSM-tree lookup with bloom filter optimization * v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround) + * v5.8.0: Added pagination support for entities with many relationships * * @param sourceId Source entity ID - * @returns Array of verb IDs originating from this source (excluding deleted) + * @param options Optional configuration + * @param options.limit Maximum number of verb IDs to return (default: all) + * @param options.offset Number of verb IDs to skip (default: 0) + * @returns Array of verb IDs originating from this source (excluding deleted, paginated if requested) + * + * @example + * // Get all verb IDs (backward compatible) + * const all = await graphIndex.getVerbIdsBySource(sourceId) + * + * @example + * // Get first 50 verb IDs + * const page1 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 50 }) + * + * @example + * // Paginate through verb IDs + * const page1 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 100, offset: 0 }) + * const page2 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 100, offset: 100 }) */ - async getVerbIdsBySource(sourceId: string): Promise { + async getVerbIdsBySource( + sourceId: string, + options?: { + limit?: number + offset?: number + } + ): Promise { await this.ensureInitialized() const startTime = performance.now() @@ -193,18 +262,51 @@ export class GraphAdjacencyIndex { // Filter out deleted verb IDs (tombstone deletion workaround) // LSM-tree retains all IDs, but verbIdSet tracks deletions const allIds = verbIds || [] - return allIds.filter(id => this.verbIdSet.has(id)) + let result = allIds.filter(id => this.verbIdSet.has(id)) + + // Apply pagination if requested (v5.8.0) + if (options?.limit !== undefined || options?.offset !== undefined) { + const offset = options.offset || 0 + const limit = options.limit !== undefined ? options.limit : result.length + result = result.slice(offset, offset + limit) + } + + return result } /** * Get verb IDs by target - Billion-scale optimization for getVerbsByTarget + * * O(log n) LSM-tree lookup with bloom filter optimization * v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround) + * v5.8.0: Added pagination support for popular target entities * * @param targetId Target entity ID - * @returns Array of verb IDs pointing to this target (excluding deleted) + * @param options Optional configuration + * @param options.limit Maximum number of verb IDs to return (default: all) + * @param options.offset Number of verb IDs to skip (default: 0) + * @returns Array of verb IDs pointing to this target (excluding deleted, paginated if requested) + * + * @example + * // Get all verb IDs (backward compatible) + * const all = await graphIndex.getVerbIdsByTarget(targetId) + * + * @example + * // Get first 50 verb IDs + * const page1 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 50 }) + * + * @example + * // Paginate through verb IDs + * const page1 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 100, offset: 0 }) + * const page2 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 100, offset: 100 }) */ - async getVerbIdsByTarget(targetId: string): Promise { + async getVerbIdsByTarget( + targetId: string, + options?: { + limit?: number + offset?: number + } + ): Promise { await this.ensureInitialized() const startTime = performance.now() @@ -219,7 +321,16 @@ export class GraphAdjacencyIndex { // Filter out deleted verb IDs (tombstone deletion workaround) // LSM-tree retains all IDs, but verbIdSet tracks deletions const allIds = verbIds || [] - return allIds.filter(id => this.verbIdSet.has(id)) + let result = allIds.filter(id => this.verbIdSet.has(id)) + + // Apply pagination if requested (v5.8.0) + if (options?.limit !== undefined || options?.offset !== undefined) { + const offset = options.offset || 0 + const limit = options.limit !== undefined ? options.limit : result.length + result = result.slice(offset, offset + limit) + } + + return result } /** diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts new file mode 100644 index 00000000..ff19b571 --- /dev/null +++ b/src/transaction/Transaction.ts @@ -0,0 +1,238 @@ +/** + * Transaction - Atomic Unit of Work + * + * Executes operations atomically: all succeed or all rollback. + * Prevents partial failures that leave system in inconsistent state. + * + * Usage: + * ```typescript + * const tx = new Transaction() + * tx.addOperation(operation1) + * tx.addOperation(operation2) + * await tx.execute() // Both succeed or both rollback + * ``` + */ + +import { + Operation, + RollbackAction, + TransactionState, + TransactionContext, + TransactionOptions +} from './types.js' +import { + InvalidTransactionStateError, + TransactionExecutionError, + TransactionRollbackError, + TransactionTimeoutError +} from './errors.js' +import { prodLog } from '../utils/logger.js' + +/** + * Default transaction options + */ +const DEFAULT_OPTIONS: Required = { + timeout: 30000, // 30 seconds + logging: false, + maxRollbackRetries: 3 +} + +/** + * Transaction class + */ +export class Transaction implements TransactionContext { + private operations: Operation[] = [] + private rollbackActions: RollbackAction[] = [] + private state: TransactionState = 'pending' + private readonly options: Required + private startTime?: number + private endTime?: number + + constructor(options: TransactionOptions = {}) { + this.options = { ...DEFAULT_OPTIONS, ...options } + } + + /** + * Add an operation to the transaction + */ + addOperation(operation: Operation): void { + if (this.state !== 'pending') { + throw new InvalidTransactionStateError( + this.state, + 'add operation' + ) + } + this.operations.push(operation) + } + + /** + * Get current transaction state + */ + getState(): TransactionState { + return this.state + } + + /** + * Get number of operations in transaction + */ + getOperationCount(): number { + return this.operations.length + } + + /** + * Execute all operations atomically + */ + async execute(): Promise { + if (this.state !== 'pending') { + throw new InvalidTransactionStateError( + this.state, + 'execute' + ) + } + + this.state = 'executing' + this.startTime = Date.now() + + if (this.options.logging) { + prodLog.info(`[Transaction] Executing ${this.operations.length} operations`) + } + + try { + // Execute each operation in order + for (let i = 0; i < this.operations.length; i++) { + // Check timeout + if (Date.now() - this.startTime > this.options.timeout) { + throw new TransactionTimeoutError(this.options.timeout, i) + } + + const operation = this.operations[i] + + try { + if (this.options.logging) { + prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`) + } + + // Execute operation + const rollbackAction = await operation.execute() + + // Record rollback action (if provided) + if (rollbackAction) { + this.rollbackActions.push(rollbackAction) + } + + } catch (error) { + // Operation failed - rollback and re-throw + const executionError = new TransactionExecutionError( + `Operation ${i} failed: ${(error as Error).message}`, + i, + operation.name, + error as Error + ) + + await this.rollback(executionError) + throw executionError + } + } + + // All operations succeeded - commit + this.commit() + + } catch (error) { + // Error already handled in rollback + throw error + } + } + + /** + * Commit the transaction + */ + private commit(): void { + this.state = 'committed' + this.endTime = Date.now() + + if (this.options.logging) { + const duration = this.endTime - (this.startTime || this.endTime) + prodLog.info(`[Transaction] Committed successfully in ${duration}ms`) + } + + // Clear rollback actions - no longer needed + this.rollbackActions = [] + } + + /** + * Rollback all executed operations in reverse order + */ + private async rollback(originalError: Error): Promise { + this.state = 'rolling_back' + + if (this.options.logging) { + prodLog.info(`[Transaction] Rolling back ${this.rollbackActions.length} operations`) + } + + const rollbackErrors: Error[] = [] + + // Execute rollback actions in REVERSE order + for (let i = this.rollbackActions.length - 1; i >= 0; i--) { + const action = this.rollbackActions[i] + + // Retry rollback with exponential backoff + let attempts = 0 + let success = false + + while (attempts < this.options.maxRollbackRetries && !success) { + try { + await action() + success = true + + if (this.options.logging) { + prodLog.info(`[Transaction] Rolled back operation ${i}`) + } + + } catch (error) { + attempts++ + + if (attempts >= this.options.maxRollbackRetries) { + // Max retries exceeded - log error and continue + const rollbackError = error as Error + rollbackErrors.push(rollbackError) + + prodLog.error( + `[Transaction] Rollback failed for operation ${i} after ${attempts} attempts: ${rollbackError.message}` + ) + } else { + // Retry with exponential backoff + const delayMs = Math.pow(2, attempts) * 100 + await new Promise(resolve => setTimeout(resolve, delayMs)) + } + } + } + } + + this.state = 'rolled_back' + this.endTime = Date.now() + + if (this.options.logging) { + const duration = this.endTime - (this.startTime || this.endTime) + prodLog.info(`[Transaction] Rolled back in ${duration}ms`) + } + + // If rollback encountered errors, wrap them with original error + if (rollbackErrors.length > 0) { + throw new TransactionRollbackError( + `Transaction rollback encountered ${rollbackErrors.length} errors during cleanup`, + originalError, + rollbackErrors + ) + } + } + + /** + * Get transaction execution time in milliseconds + */ + getExecutionTimeMs(): number | undefined { + if (this.startTime && this.endTime) { + return this.endTime - this.startTime + } + return undefined + } +} diff --git a/src/transaction/TransactionManager.ts b/src/transaction/TransactionManager.ts new file mode 100644 index 00000000..0f13b6a3 --- /dev/null +++ b/src/transaction/TransactionManager.ts @@ -0,0 +1,189 @@ +/** + * Transaction Manager + * + * Manages transaction execution across the system. + * Provides high-level API for executing atomic operations. + * + * Usage: + * ```typescript + * const txManager = new TransactionManager() + * + * const result = await txManager.executeTransaction(async (tx) => { + * tx.addOperation(operation1) + * tx.addOperation(operation2) + * return someValue + * }) + * ``` + */ + +import { Transaction } from './Transaction.js' +import { + TransactionFunction, + TransactionResult, + TransactionOptions +} from './types.js' +import { TransactionError } from './errors.js' +import { prodLog } from '../utils/logger.js' + +/** + * Transaction Manager Statistics + */ +export interface TransactionStats { + totalTransactions: number + successfulTransactions: number + failedTransactions: number + rolledBackTransactions: number + averageExecutionTimeMs: number + averageOperationsPerTransaction: number +} + +/** + * Transaction Manager + */ +export class TransactionManager { + private stats: TransactionStats = { + totalTransactions: 0, + successfulTransactions: 0, + failedTransactions: 0, + rolledBackTransactions: 0, + averageExecutionTimeMs: 0, + averageOperationsPerTransaction: 0 + } + + private totalExecutionTime = 0 + private totalOperations = 0 + + /** + * Execute a function within a transaction + * All operations succeed atomically or all rollback + * + * @param fn Function that builds and executes transaction + * @param options Transaction execution options + * @returns Result from user function + * @throws TransactionError if transaction fails + */ + async executeTransaction( + fn: TransactionFunction, + options?: TransactionOptions + ): Promise { + const transaction = new Transaction(options) + + this.stats.totalTransactions++ + + try { + // Execute user function (builds operations list) + const result = await fn(transaction) + + // Execute all operations atomically + await transaction.execute() + + // Update statistics + this.stats.successfulTransactions++ + this.updateStats(transaction) + + return result + + } catch (error) { + // Transaction failed and rolled back + this.stats.failedTransactions++ + + if (transaction.getState() === 'rolled_back') { + this.stats.rolledBackTransactions++ + } + + this.updateStats(transaction) + + // Re-throw with context + if (error instanceof TransactionError) { + throw error + } else { + throw new TransactionError( + `Transaction failed: ${(error as Error).message}`, + { cause: error } + ) + } + } + } + + /** + * Execute a transaction and return detailed result + */ + async executeTransactionWithResult( + fn: TransactionFunction, + options?: TransactionOptions + ): Promise> { + const startTime = Date.now() + const transaction = new Transaction(options) + + try { + const value = await fn(transaction) + await transaction.execute() + + const executionTimeMs = Date.now() - startTime + + return { + value, + operationCount: transaction.getOperationCount(), + executionTimeMs + } + + } catch (error) { + // Transaction failed + throw error + } + } + + /** + * Get transaction statistics + */ + getStats(): Readonly { + return { ...this.stats } + } + + /** + * Reset transaction statistics + */ + resetStats(): void { + this.stats = { + totalTransactions: 0, + successfulTransactions: 0, + failedTransactions: 0, + rolledBackTransactions: 0, + averageExecutionTimeMs: 0, + averageOperationsPerTransaction: 0 + } + this.totalExecutionTime = 0 + this.totalOperations = 0 + } + + /** + * Update running statistics + */ + private updateStats(transaction: Transaction): void { + const executionTime = transaction.getExecutionTimeMs() + const operationCount = transaction.getOperationCount() + + if (executionTime !== undefined) { + this.totalExecutionTime += executionTime + this.stats.averageExecutionTimeMs = + this.totalExecutionTime / this.stats.totalTransactions + } + + this.totalOperations += operationCount + this.stats.averageOperationsPerTransaction = + this.totalOperations / this.stats.totalTransactions + } + + /** + * Log current statistics + */ + logStats(): void { + prodLog.info('[TransactionManager] Statistics:') + prodLog.info(` Total: ${this.stats.totalTransactions}`) + prodLog.info(` Successful: ${this.stats.successfulTransactions}`) + prodLog.info(` Failed: ${this.stats.failedTransactions}`) + prodLog.info(` Rolled back: ${this.stats.rolledBackTransactions}`) + prodLog.info(` Avg execution time: ${this.stats.averageExecutionTimeMs.toFixed(2)}ms`) + prodLog.info(` Avg operations/tx: ${this.stats.averageOperationsPerTransaction.toFixed(2)}`) + } +} diff --git a/src/transaction/errors.ts b/src/transaction/errors.ts new file mode 100644 index 00000000..1cb49efd --- /dev/null +++ b/src/transaction/errors.ts @@ -0,0 +1,88 @@ +/** + * Transaction System Errors + * + * Provides detailed error information for transaction failures. + */ + +/** + * Base class for all transaction errors + */ +export class TransactionError extends Error { + constructor( + message: string, + public readonly context?: Record + ) { + super(message) + this.name = 'TransactionError' + Error.captureStackTrace(this, this.constructor) + } +} + +/** + * Error during transaction execution + */ +export class TransactionExecutionError extends TransactionError { + constructor( + message: string, + public readonly operationIndex: number, + public readonly operationName: string | undefined, + public readonly cause: Error + ) { + super(message, { + operationIndex, + operationName, + cause: cause.message + }) + this.name = 'TransactionExecutionError' + } +} + +/** + * Error during transaction rollback + */ +export class TransactionRollbackError extends TransactionError { + constructor( + message: string, + public readonly originalError: Error, + public readonly rollbackErrors: Error[] + ) { + super(message, { + originalError: originalError.message, + rollbackErrorCount: rollbackErrors.length, + rollbackErrors: rollbackErrors.map(e => e.message) + }) + this.name = 'TransactionRollbackError' + } +} + +/** + * Error for invalid transaction state + */ +export class InvalidTransactionStateError extends TransactionError { + constructor( + currentState: string, + attemptedAction: string + ) { + super( + `Cannot ${attemptedAction}: transaction is in state '${currentState}'`, + { currentState, attemptedAction } + ) + this.name = 'InvalidTransactionStateError' + } +} + +/** + * Error for transaction timeout + */ +export class TransactionTimeoutError extends TransactionError { + constructor( + timeoutMs: number, + operationIndex: number + ) { + super( + `Transaction timed out after ${timeoutMs}ms at operation ${operationIndex}`, + { timeoutMs, operationIndex } + ) + this.name = 'TransactionTimeoutError' + } +} diff --git a/src/transaction/index.ts b/src/transaction/index.ts new file mode 100644 index 00000000..2972ee24 --- /dev/null +++ b/src/transaction/index.ts @@ -0,0 +1,32 @@ +/** + * Transaction System + * + * Provides atomicity for Brainy operations. + * All operations succeed or all rollback - no partial failures. + * + * @module transaction + */ + +export * from './types.js' +export * from './errors.js' +export { Transaction } from './Transaction.js' +export { TransactionManager, type TransactionStats } from './TransactionManager.js' + +// Re-export for convenience +export type { + Operation, + RollbackAction, + TransactionState, + TransactionContext, + TransactionFunction, + TransactionResult, + TransactionOptions +} from './types.js' + +export { + TransactionError, + TransactionExecutionError, + TransactionRollbackError, + InvalidTransactionStateError, + TransactionTimeoutError +} from './errors.js' diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts new file mode 100644 index 00000000..5b9a311c --- /dev/null +++ b/src/transaction/operations/IndexOperations.ts @@ -0,0 +1,361 @@ +/** + * Index Operations with Rollback Support + * + * Provides transactional operations for all indexes: + * - TypeAwareHNSWIndex (per-type vector indexes) + * - HNSWIndex (legacy/fallback vector index) + * - MetadataIndexManager (roaring bitmap filtering) + * - GraphAdjacencyIndex (LSM-tree graph storage) + * + * Each operation can be executed and rolled back atomically. + */ + +import type { HNSWIndex } from '../../hnsw/hnswIndex.js' +import type { TypeAwareHNSWIndex } from '../../hnsw/typeAwareHNSWIndex.js' +import type { MetadataIndexManager } from '../../utils/metadataIndex.js' +import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js' +import type { NounType } from '../../types/graphTypes.js' +import type { GraphVerb } from '../../coreTypes.js' +import type { Operation, RollbackAction } from '../types.js' + +/** + * Add to HNSW index with rollback support + * + * Rollback strategy: + * - Remove item from index + * + * Note: Works with both HNSWIndex and TypeAwareHNSWIndex + */ +export class AddToHNSWOperation implements Operation { + readonly name = 'AddToHNSW' + + constructor( + private readonly index: HNSWIndex, + private readonly id: string, + private readonly vector: number[] + ) {} + + async execute(): Promise { + // Check if item already exists (for rollback decision) + const existed = await this.itemExists(this.id) + + // Add to index + await this.index.addItem({ id: this.id, vector: this.vector }) + + // Return rollback action + return async () => { + if (!existed) { + // Remove newly added item + await this.index.removeItem(this.id) + } + // If item existed before, we don't rollback (update is OK) + // This prevents index corruption from removing pre-existing items + } + } + + /** + * Check if item exists in index + */ + private async itemExists(id: string): Promise { + try { + // Try to get item - if exists, no error + await (this.index as any).getItem?.(id) + return true + } catch { + return false + } + } +} + +/** + * Add to TypeAwareHNSW index with rollback support + * + * Rollback strategy: + * - Remove item from type-specific index + */ +export class AddToTypeAwareHNSWOperation implements Operation { + readonly name = 'AddToTypeAwareHNSW' + + constructor( + private readonly index: TypeAwareHNSWIndex, + private readonly id: string, + private readonly vector: number[], + private readonly nounType: NounType + ) {} + + async execute(): Promise { + // Check if item already exists + const existed = await this.itemExists(this.id, this.nounType) + + // Add to type-specific index + await this.index.addItem({ id: this.id, vector: this.vector }, this.nounType) + + // Return rollback action + return async () => { + if (!existed) { + // Remove newly added item from type-specific index + await this.index.removeItem(this.id, this.nounType) + } + } + } + + /** + * Check if item exists in type-specific index + */ + private async itemExists(id: string, type: NounType): Promise { + try { + const index = (this.index as any).getIndexForType?.(type) + if (index) { + await index.getItem?.(id) + return true + } + return false + } catch { + return false + } + } +} + +/** + * Remove from HNSW index with rollback support + * + * Rollback strategy: + * - Re-add item to index with original vector + * + * Note: Requires storing the vector for rollback + */ +export class RemoveFromHNSWOperation implements Operation { + readonly name = 'RemoveFromHNSW' + + constructor( + private readonly index: HNSWIndex, + private readonly id: string, + private readonly vector: number[] // Required for rollback + ) {} + + async execute(): Promise { + // Remove from index + await this.index.removeItem(this.id) + + // Return rollback action + return async () => { + // Re-add item with original vector + await this.index.addItem({ id: this.id, vector: this.vector }) + } + } +} + +/** + * Remove from TypeAwareHNSW index with rollback support + * + * Rollback strategy: + * - Re-add item to type-specific index with original vector + */ +export class RemoveFromTypeAwareHNSWOperation implements Operation { + readonly name = 'RemoveFromTypeAwareHNSW' + + constructor( + private readonly index: TypeAwareHNSWIndex, + private readonly id: string, + private readonly vector: number[], // Required for rollback + private readonly nounType: NounType + ) {} + + async execute(): Promise { + // Remove from type-specific index + await this.index.removeItem(this.id, this.nounType) + + // Return rollback action + return async () => { + // Re-add item with original vector to type-specific index + await this.index.addItem({ id: this.id, vector: this.vector }, this.nounType) + } + } +} + +/** + * Add to metadata index with rollback support + * + * Rollback strategy: + * - Remove item from index + */ +export class AddToMetadataIndexOperation implements Operation { + readonly name = 'AddToMetadataIndex' + + constructor( + private readonly index: MetadataIndexManager, + private readonly id: string, + private readonly entity: any // Entity or metadata structure + ) {} + + async execute(): Promise { + // Add to metadata index (skipFlush=true for transaction atomicity) + await this.index.addToIndex(this.id, this.entity, true) + + // Return rollback action + return async () => { + // Remove from metadata index + await this.index.removeFromIndex(this.id, this.entity) + } + } +} + +/** + * Remove from metadata index with rollback support + * + * Rollback strategy: + * - Re-add item to index with original metadata + */ +export class RemoveFromMetadataIndexOperation implements Operation { + readonly name = 'RemoveFromMetadataIndex' + + constructor( + private readonly index: MetadataIndexManager, + private readonly id: string, + private readonly entity: any // Required for rollback + ) {} + + async execute(): Promise { + // Remove from metadata index + await this.index.removeFromIndex(this.id, this.entity) + + // Return rollback action + return async () => { + // Re-add with original metadata (skipFlush=true) + await this.index.addToIndex(this.id, this.entity, true) + } + } +} + +/** + * Add verb to graph index with rollback support + * + * Rollback strategy: + * - Remove verb from graph index + */ +export class AddToGraphIndexOperation implements Operation { + readonly name = 'AddToGraphIndex' + + constructor( + private readonly index: GraphAdjacencyIndex, + private readonly verb: GraphVerb + ) {} + + async execute(): Promise { + // Add verb to graph index + await this.index.addVerb(this.verb) + + // Return rollback action + return async () => { + // Remove verb from graph index + await this.index.removeVerb(this.verb.id) + } + } +} + +/** + * Remove verb from graph index with rollback support + * + * Rollback strategy: + * - Re-add verb to graph index + */ +export class RemoveFromGraphIndexOperation implements Operation { + readonly name = 'RemoveFromGraphIndex' + + constructor( + private readonly index: GraphAdjacencyIndex, + private readonly verb: GraphVerb // Required for rollback + ) {} + + async execute(): Promise { + // Remove verb from graph index + await this.index.removeVerb(this.verb.id) + + // Return rollback action + return async () => { + // Re-add verb with original data + await this.index.addVerb(this.verb) + } + } +} + +/** + * Batch operation: Add multiple items to HNSW index + * + * Useful for bulk imports with transaction support. + * Rolls back all items if any fail. + */ +export class BatchAddToHNSWOperation implements Operation { + readonly name = 'BatchAddToHNSW' + + private operations: AddToHNSWOperation[] + + constructor( + index: HNSWIndex, + items: Array<{ id: string; vector: number[] }> + ) { + this.operations = items.map( + item => new AddToHNSWOperation(index, item.id, item.vector) + ) + } + + async execute(): Promise { + const rollbackActions: RollbackAction[] = [] + + // Execute all operations + for (const op of this.operations) { + const rollback = await op.execute() + if (rollback) { + rollbackActions.push(rollback) + } + } + + // Return combined rollback action + return async () => { + // Execute all rollbacks in reverse order + for (let i = rollbackActions.length - 1; i >= 0; i--) { + await rollbackActions[i]() + } + } + } +} + +/** + * Batch operation: Add multiple entities to metadata index + * + * Useful for bulk imports with transaction support. + */ +export class BatchAddToMetadataIndexOperation implements Operation { + readonly name = 'BatchAddToMetadataIndex' + + private operations: AddToMetadataIndexOperation[] + + constructor( + index: MetadataIndexManager, + items: Array<{ id: string; entity: any }> + ) { + this.operations = items.map( + item => new AddToMetadataIndexOperation(index, item.id, item.entity) + ) + } + + async execute(): Promise { + const rollbackActions: RollbackAction[] = [] + + // Execute all operations + for (const op of this.operations) { + const rollback = await op.execute() + if (rollback) { + rollbackActions.push(rollback) + } + } + + // Return combined rollback action + return async () => { + // Execute all rollbacks in reverse order + for (let i = rollbackActions.length - 1; i >= 0; i--) { + await rollbackActions[i]() + } + } + } +} diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts new file mode 100644 index 00000000..ef92b5c8 --- /dev/null +++ b/src/transaction/operations/StorageOperations.ts @@ -0,0 +1,301 @@ +/** + * Storage Operations with Rollback Support + * + * Provides transactional operations for all storage adapters. + * Each operation can be executed and rolled back atomically. + * + * Supports: + * - All 8 storage adapters (FileSystem, S3, Memory, OPFS, GCS, Azure, R2, TypeAware) + * - Nouns (entities) and Verbs (relationships) + * - Metadata and vector data + * - COW (Copy-on-Write) storage + */ + +import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js' +import type { Operation, RollbackAction } from '../types.js' + +/** + * Save noun metadata with rollback support + * + * Rollback strategy: + * - If metadata existed: Restore previous metadata + * - If metadata was new: Delete metadata + */ +export class SaveNounMetadataOperation implements Operation { + readonly name = 'SaveNounMetadata' + + constructor( + private readonly storage: StorageAdapter, + private readonly id: string, + private readonly metadata: NounMetadata + ) {} + + async execute(): Promise { + // Get existing metadata (for rollback) + const previousMetadata = await this.storage.getNounMetadata(this.id) + + // Save new metadata + await this.storage.saveNounMetadata(this.id, this.metadata) + + // Return rollback action + return async () => { + if (previousMetadata) { + // Restore previous metadata + await this.storage.saveNounMetadata(this.id, previousMetadata) + } else { + // Delete newly created metadata + await this.storage.deleteNounMetadata(this.id) + } + } + } +} + +/** + * Save noun (vector data) with rollback support + * + * Rollback strategy: + * - If noun existed: Restore previous noun + * - If noun was new: Delete noun (if deleteNoun exists on adapter) + * + * Note: Not all adapters implement deleteNoun - this is acceptable + * because orphaned vector data without metadata is invisible to queries + */ +export class SaveNounOperation implements Operation { + readonly name = 'SaveNoun' + + constructor( + private readonly storage: StorageAdapter, + private readonly noun: HNSWNoun + ) {} + + async execute(): Promise { + // Get existing noun (for rollback) + const previousNoun = await this.storage.getNoun(this.noun.id) + + // Save new noun + await this.storage.saveNoun(this.noun) + + // Return rollback action + return async () => { + if (previousNoun) { + // Restore previous noun (extract just vector data) + const nounData: HNSWNoun = { + id: previousNoun.id, + vector: previousNoun.vector, + connections: previousNoun.connections || new Map(), + level: previousNoun.level || 0 + } + await this.storage.saveNoun(nounData) + } else { + // Delete newly created noun (if adapter supports it) + // Note: Not all adapters implement deleteNoun + // This is acceptable - metadata deletion makes entity invisible + if ('deleteNoun' in this.storage && typeof (this.storage as any).deleteNoun === 'function') { + await (this.storage as any).deleteNoun(this.noun.id) + } + } + } + } +} + +/** + * Delete noun metadata with rollback support + * + * Rollback strategy: + * - Restore deleted metadata + */ +export class DeleteNounMetadataOperation implements Operation { + readonly name = 'DeleteNounMetadata' + + constructor( + private readonly storage: StorageAdapter, + private readonly id: string + ) {} + + async execute(): Promise { + // Get metadata before deletion (for rollback) + const previousMetadata = await this.storage.getNounMetadata(this.id) + + if (!previousMetadata) { + // Nothing to delete - no rollback needed + return async () => {} + } + + // Delete metadata + await this.storage.deleteNounMetadata(this.id) + + // Return rollback action + return async () => { + // Restore deleted metadata + await this.storage.saveNounMetadata(this.id, previousMetadata) + } + } +} + +/** + * Save verb metadata with rollback support + * + * Rollback strategy: + * - If metadata existed: Restore previous metadata + * - If metadata was new: Delete metadata + */ +export class SaveVerbMetadataOperation implements Operation { + readonly name = 'SaveVerbMetadata' + + constructor( + private readonly storage: StorageAdapter, + private readonly id: string, + private readonly metadata: VerbMetadata + ) {} + + async execute(): Promise { + // Get existing metadata (for rollback) + const previousMetadata = await this.storage.getVerbMetadata(this.id) + + // Save new metadata + await this.storage.saveVerbMetadata(this.id, this.metadata) + + // Return rollback action + return async () => { + if (previousMetadata) { + // Restore previous metadata + await this.storage.saveVerbMetadata(this.id, previousMetadata) + } else { + // Delete newly created verb (metadata + vector) + // Note: StorageAdapter has deleteVerb but not deleteVerbMetadata + await this.storage.deleteVerb(this.id) + } + } + } +} + +/** + * Save verb (vector data) with rollback support + * + * Rollback strategy: + * - If verb existed: Restore previous verb + * - If verb was new: Delete verb (if deleteVerb exists on adapter) + */ +export class SaveVerbOperation implements Operation { + readonly name = 'SaveVerb' + + constructor( + private readonly storage: StorageAdapter, + private readonly verb: HNSWVerb + ) {} + + async execute(): Promise { + // Get existing verb (for rollback) + const previousVerb = await this.storage.getVerb(this.verb.id) + + // Save new verb + await this.storage.saveVerb(this.verb) + + // Return rollback action + return async () => { + if (previousVerb) { + // Restore previous verb (extract just vector data) + const verbData: HNSWVerb = { + id: previousVerb.id, + sourceId: previousVerb.sourceId, + targetId: previousVerb.targetId, + verb: previousVerb.verb, + vector: previousVerb.vector, + connections: previousVerb.connections || new Map() + } + await this.storage.saveVerb(verbData) + } else { + // Delete newly created verb (if adapter supports it) + if ('deleteVerb' in this.storage && typeof (this.storage as any).deleteVerb === 'function') { + await (this.storage as any).deleteVerb(this.verb.id) + } + } + } + } +} + +/** + * Delete verb metadata with rollback support + * + * Rollback strategy: + * - Restore deleted metadata + */ +export class DeleteVerbMetadataOperation implements Operation { + readonly name = 'DeleteVerbMetadata' + + constructor( + private readonly storage: StorageAdapter, + private readonly id: string + ) {} + + async execute(): Promise { + // Get metadata before deletion (for rollback) + const previousMetadata = await this.storage.getVerbMetadata(this.id) + + if (!previousMetadata) { + // Nothing to delete - no rollback needed + return async () => {} + } + + // Delete verb (metadata + vector) + // Note: StorageAdapter has deleteVerb but not deleteVerbMetadata + await this.storage.deleteVerb(this.id) + + // Return rollback action + return async () => { + // Restore deleted metadata + await this.storage.saveVerbMetadata(this.id, previousMetadata) + } + } +} + +/** + * Update noun metadata with rollback support + * + * Rollback strategy: + * - Restore previous metadata + * + * Note: This is a convenience operation that wraps SaveNounMetadataOperation + * with explicit "update" semantics + */ +export class UpdateNounMetadataOperation implements Operation { + readonly name = 'UpdateNounMetadata' + + private saveOperation: SaveNounMetadataOperation + + constructor( + storage: StorageAdapter, + id: string, + metadata: NounMetadata + ) { + this.saveOperation = new SaveNounMetadataOperation(storage, id, metadata) + } + + async execute(): Promise { + return await this.saveOperation.execute() + } +} + +/** + * Update verb metadata with rollback support + * + * Rollback strategy: + * - Restore previous metadata + */ +export class UpdateVerbMetadataOperation implements Operation { + readonly name = 'UpdateVerbMetadata' + + private saveOperation: SaveVerbMetadataOperation + + constructor( + storage: StorageAdapter, + id: string, + metadata: VerbMetadata + ) { + this.saveOperation = new SaveVerbMetadataOperation(storage, id, metadata) + } + + async execute(): Promise { + return await this.saveOperation.execute() + } +} diff --git a/src/transaction/operations/index.ts b/src/transaction/operations/index.ts new file mode 100644 index 00000000..ce4cbca5 --- /dev/null +++ b/src/transaction/operations/index.ts @@ -0,0 +1,34 @@ +/** + * Transaction Operations + * + * Re-exports all transactional operations for storage and indexes. + * These operations provide atomicity with rollback support. + * + * @module transaction/operations + */ + +// Storage Operations +export { + SaveNounMetadataOperation, + SaveNounOperation, + DeleteNounMetadataOperation, + SaveVerbMetadataOperation, + SaveVerbOperation, + DeleteVerbMetadataOperation, + UpdateNounMetadataOperation, + UpdateVerbMetadataOperation +} from './StorageOperations.js' + +// Index Operations +export { + AddToHNSWOperation, + AddToTypeAwareHNSWOperation, + RemoveFromHNSWOperation, + RemoveFromTypeAwareHNSWOperation, + AddToMetadataIndexOperation, + RemoveFromMetadataIndexOperation, + AddToGraphIndexOperation, + RemoveFromGraphIndexOperation, + BatchAddToHNSWOperation, + BatchAddToMetadataIndexOperation +} from './IndexOperations.js' diff --git a/src/transaction/types.ts b/src/transaction/types.ts new file mode 100644 index 00000000..5917ea4c --- /dev/null +++ b/src/transaction/types.ts @@ -0,0 +1,103 @@ +/** + * Transaction System Types + * + * Provides atomicity for Brainy operations - all succeed or all rollback. + * Prevents partial failures that leave system in inconsistent state. + */ + +/** + * Transaction state + */ +export type TransactionState = + | 'pending' // Created but not executed + | 'executing' // Currently executing operations + | 'committed' // Successfully committed + | 'rolling_back' // Rolling back due to failure + | 'rolled_back' // Successfully rolled back + +/** + * Rollback action - undoes an operation + * Must be idempotent (safe to call multiple times) + */ +export type RollbackAction = () => Promise + +/** + * Operation that can be executed and rolled back + */ +export interface Operation { + /** + * Execute the operation + * @returns Rollback action to undo this operation (or undefined if no rollback needed) + */ + execute(): Promise + + /** + * Optional: Name of operation for debugging + */ + readonly name?: string +} + +/** + * Transaction context passed to user functions + */ +export interface TransactionContext { + /** + * Add an operation to the transaction + */ + addOperation(operation: Operation): void + + /** + * Get current transaction state + */ + getState(): TransactionState + + /** + * Get number of operations in transaction + */ + getOperationCount(): number +} + +/** + * Function that builds a transaction + */ +export type TransactionFunction = (ctx: TransactionContext) => Promise + +/** + * Transaction execution result + */ +export interface TransactionResult { + /** + * Result value from user function + */ + value: T + + /** + * Number of operations executed + */ + operationCount: number + + /** + * Execution time in milliseconds + */ + executionTimeMs: number +} + +/** + * Transaction execution options + */ +export interface TransactionOptions { + /** + * Timeout in milliseconds (default: 30000 = 30 seconds) + */ + timeout?: number + + /** + * Whether to log transaction execution (default: false) + */ + logging?: boolean + + /** + * Maximum number of rollback retry attempts (default: 3) + */ + maxRollbackRetries?: number +} diff --git a/tests/transaction/Transaction.test.ts b/tests/transaction/Transaction.test.ts new file mode 100644 index 00000000..de09fa64 --- /dev/null +++ b/tests/transaction/Transaction.test.ts @@ -0,0 +1,396 @@ +/** + * Transaction Core Tests + * + * Tests the Transaction class for: + * - Basic execution flow + * - Rollback on failure + * - State management + * - Timeout handling + * - Retry logic + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { Transaction } from '../../src/transaction/Transaction.js' +import type { Operation, RollbackAction } from '../../src/transaction/types.js' +import { + InvalidTransactionStateError, + TransactionExecutionError, + TransactionTimeoutError +} from '../../src/transaction/errors.js' + +describe('Transaction', () => { + describe('Basic Execution', () => { + it('should execute operations in order and commit', async () => { + const executionOrder: number[] = [] + + const op1: Operation = { + name: 'Operation1', + execute: async () => { + executionOrder.push(1) + return async () => { /* rollback */ } + } + } + + const op2: Operation = { + name: 'Operation2', + execute: async () => { + executionOrder.push(2) + return async () => { /* rollback */ } + } + } + + const tx = new Transaction() + tx.addOperation(op1) + tx.addOperation(op2) + + await tx.execute() + + expect(executionOrder).toEqual([1, 2]) + expect(tx.getState()).toBe('committed') + }) + + it('should track operation count', () => { + const tx = new Transaction() + + expect(tx.getOperationCount()).toBe(0) + + tx.addOperation({ execute: async () => undefined }) + expect(tx.getOperationCount()).toBe(1) + + tx.addOperation({ execute: async () => undefined }) + expect(tx.getOperationCount()).toBe(2) + }) + + it('should record execution time', async () => { + const tx = new Transaction() + tx.addOperation({ + execute: async () => { + // Small delay to ensure measurable time + await new Promise(resolve => setTimeout(resolve, 1)) + return async () => {} + } + }) + + await tx.execute() + + const executionTime = tx.getExecutionTimeMs() + expect(executionTime).toBeGreaterThanOrEqual(0) // Can be 0 for very fast operations + expect(executionTime).toBeLessThan(1000) // Should be reasonably quick + }) + }) + + describe('Rollback on Failure', () => { + it('should rollback all operations when one fails', async () => { + const rollbackOrder: number[] = [] + + const op1: Operation = { + name: 'Operation1', + execute: async () => { + return async () => { rollbackOrder.push(1) } + } + } + + const op2: Operation = { + name: 'Operation2', + execute: async () => { + throw new Error('Operation 2 failed') + } + } + + const op3: Operation = { + name: 'Operation3', + execute: async () => { + return async () => { rollbackOrder.push(3) } + } + } + + const tx = new Transaction() + tx.addOperation(op1) + tx.addOperation(op2) + tx.addOperation(op3) + + await expect(tx.execute()).rejects.toThrow(TransactionExecutionError) + + expect(tx.getState()).toBe('rolled_back') + // Only op1 executed before failure, so only op1 should rollback + expect(rollbackOrder).toEqual([1]) + }) + + it('should rollback operations in reverse order', async () => { + const rollbackOrder: number[] = [] + + const op1: Operation = { + execute: async () => { + return async () => { rollbackOrder.push(1) } + } + } + + const op2: Operation = { + execute: async () => { + return async () => { rollbackOrder.push(2) } + } + } + + const op3: Operation = { + execute: async () => { + throw new Error('op3 failed') + } + } + + const tx = new Transaction() + tx.addOperation(op1) + tx.addOperation(op2) + tx.addOperation(op3) + + await expect(tx.execute()).rejects.toThrow() + + // Rollback in reverse order: op2, then op1 + expect(rollbackOrder).toEqual([2, 1]) + }) + + it('should retry failed rollback operations', async () => { + let rollbackAttempts = 0 + + const op: Operation = { + execute: async () => { + return async () => { + rollbackAttempts++ + if (rollbackAttempts < 2) { + throw new Error('Rollback failed') + } + // Success on 2nd attempt + } + } + } + + const failingOp: Operation = { + execute: async () => { + throw new Error('Operation failed') + } + } + + const tx = new Transaction() + tx.addOperation(op) + tx.addOperation(failingOp) + + await expect(tx.execute()).rejects.toThrow() + + // Should have retried rollback + expect(rollbackAttempts).toBe(2) + }) + + it('should continue rollback even if one rollback fails', async () => { + const rollbackOrder: number[] = [] + + const op1: Operation = { + execute: async () => { + return async () => { rollbackOrder.push(1) } + } + } + + const op2: Operation = { + execute: async () => { + return async () => { + rollbackOrder.push(2) + throw new Error('Rollback 2 failed') + } + } + } + + const op3: Operation = { + execute: async () => { + throw new Error('op3 failed') + } + } + + const tx = new Transaction({ maxRollbackRetries: 1 }) + tx.addOperation(op1) + tx.addOperation(op2) + tx.addOperation(op3) + + await expect(tx.execute()).rejects.toThrow() + + // Should attempt both rollbacks despite op2 failing + expect(rollbackOrder).toContain(1) + expect(rollbackOrder).toContain(2) + }) + }) + + describe('State Management', () => { + it('should prevent adding operations after execution starts', async () => { + const tx = new Transaction() + tx.addOperation({ execute: async () => undefined }) + + const executePromise = tx.execute() + + expect(() => { + tx.addOperation({ execute: async () => undefined }) + }).toThrow(InvalidTransactionStateError) + + await executePromise + }) + + it('should prevent double execution', async () => { + const tx = new Transaction() + tx.addOperation({ execute: async () => undefined }) + + await tx.execute() + + await expect(tx.execute()).rejects.toThrow(InvalidTransactionStateError) + }) + + it('should transition through states correctly', async () => { + const states: string[] = [] + + const tx = new Transaction() + states.push(tx.getState()) + + tx.addOperation({ + execute: async () => { + states.push(tx.getState()) + return async () => {} + } + }) + + await tx.execute() + states.push(tx.getState()) + + expect(states).toEqual(['pending', 'executing', 'committed']) + }) + }) + + describe('Timeout Handling', () => { + it('should timeout long-running transactions with many operations', async () => { + const tx = new Transaction({ timeout: 50 }) + + // Add many operations that cumulatively exceed timeout + for (let i = 0; i < 10; i++) { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + } + + await expect(tx.execute()).rejects.toThrow(TransactionTimeoutError) + }) + + it('should not timeout fast transactions', async () => { + const tx = new Transaction({ timeout: 1000 }) + + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + + await expect(tx.execute()).resolves.toBeUndefined() + }) + }) + + describe('Options', () => { + it('should respect custom timeout', async () => { + const tx = new Transaction({ timeout: 50 }) + + // Many operations to ensure cumulative time exceeds timeout + for (let i = 0; i < 10; i++) { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + } + + const error = await tx.execute().catch(e => e) + expect(error).toBeInstanceOf(TransactionTimeoutError) + }) + + it('should respect maxRollbackRetries', async () => { + let attempts = 0 + + const tx = new Transaction({ maxRollbackRetries: 2 }) + + tx.addOperation({ + execute: async () => { + return async () => { + attempts++ + throw new Error('Rollback failed') + } + } + }) + + tx.addOperation({ + execute: async () => { + throw new Error('Operation failed') + } + }) + + await expect(tx.execute()).rejects.toThrow() + + // Should try maxRollbackRetries times + expect(attempts).toBe(2) + }) + }) + + describe('Error Context', () => { + it('should include operation name in error', async () => { + const tx = new Transaction() + + tx.addOperation({ + name: 'MySpecialOperation', + execute: async () => { + throw new Error('Failed') + } + }) + + const error = await tx.execute().catch(e => e) + expect(error).toBeInstanceOf(TransactionExecutionError) + expect(error.operationName).toBe('MySpecialOperation') + }) + + it('should include operation index in error', async () => { + const tx = new Transaction() + + tx.addOperation({ execute: async () => undefined }) + tx.addOperation({ execute: async () => undefined }) + tx.addOperation({ + execute: async () => { + throw new Error('Failed') + } + }) + + const error = await tx.execute().catch(e => e) + expect(error).toBeInstanceOf(TransactionExecutionError) + expect(error.operationIndex).toBe(2) + }) + }) + + describe('Empty Transactions', () => { + it('should handle empty transaction', async () => { + const tx = new Transaction() + + await tx.execute() + + expect(tx.getState()).toBe('committed') + expect(tx.getOperationCount()).toBe(0) + }) + }) + + describe('Operations without Rollback', () => { + it('should handle operations that return no rollback action', async () => { + const tx = new Transaction() + + tx.addOperation({ + execute: async () => { + return undefined // No rollback needed + } + }) + + await expect(tx.execute()).resolves.toBeUndefined() + expect(tx.getState()).toBe('committed') + }) + }) +}) diff --git a/tests/transaction/TransactionManager.test.ts b/tests/transaction/TransactionManager.test.ts new file mode 100644 index 00000000..5ffee866 --- /dev/null +++ b/tests/transaction/TransactionManager.test.ts @@ -0,0 +1,328 @@ +/** + * TransactionManager Tests + * + * Tests the TransactionManager for: + * - High-level transaction API + * - Statistics tracking + * - Error handling + * - Result wrapping + */ + +import { describe, it, expect, beforeEach } from 'vitest' +import { TransactionManager } from '../../src/transaction/TransactionManager.js' +import { TransactionError } from '../../src/transaction/errors.js' + +describe('TransactionManager', () => { + let manager: TransactionManager + + beforeEach(() => { + manager = new TransactionManager() + }) + + describe('executeTransaction', () => { + it('should execute transaction and return user value', async () => { + const result = await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + return 'success' + }) + + expect(result).toBe('success') + }) + + it('should execute multiple operations', async () => { + const executionOrder: number[] = [] + + await manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + executionOrder.push(1) + return async () => {} + } + }) + + tx.addOperation({ + execute: async () => { + executionOrder.push(2) + return async () => {} + } + }) + }) + + expect(executionOrder).toEqual([1, 2]) + }) + + it('should throw TransactionError on failure', async () => { + await expect( + manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + throw new Error('Operation failed') + } + }) + }) + ).rejects.toThrow(TransactionError) + }) + + it('should pass through custom transaction options', async () => { + // Should timeout with many operations + await expect( + manager.executeTransaction( + async (tx) => { + for (let i = 0; i < 10; i++) { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + } + }, + { timeout: 50 } + ) + ).rejects.toThrow() + }) + }) + + describe('executeTransactionWithResult', () => { + it('should return detailed result', async () => { + const result = await manager.executeTransactionWithResult(async (tx) => { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 1)) + return async () => {} + } + }) + tx.addOperation({ execute: async () => undefined }) + return 'success' + }) + + expect(result.value).toBe('success') + expect(result.operationCount).toBe(2) + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0) + }) + + it('should measure execution time', async () => { + const result = await manager.executeTransactionWithResult(async (tx) => { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + return 'done' + }) + + expect(result.executionTimeMs).toBeGreaterThanOrEqual(10) + }) + }) + + describe('Statistics Tracking', () => { + it('should track total transactions', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }) + + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }) + + const stats = manager.getStats() + expect(stats.totalTransactions).toBe(2) + }) + + it('should track successful transactions', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }) + + const stats = manager.getStats() + expect(stats.successfulTransactions).toBe(1) + expect(stats.failedTransactions).toBe(0) + }) + + it('should track failed transactions', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }).catch(() => {}) + + await manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + throw new Error('Failed') + } + }) + }).catch(() => {}) + + const stats = manager.getStats() + expect(stats.successfulTransactions).toBe(1) + expect(stats.failedTransactions).toBe(1) + }) + + it('should track rolled back transactions', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + return async () => { /* rollback */ } + } + }) + tx.addOperation({ + execute: async () => { + throw new Error('Failed') + } + }) + }).catch(() => {}) + + const stats = manager.getStats() + expect(stats.rolledBackTransactions).toBe(1) + }) + + it('should calculate average execution time', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + }) + + await manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + return async () => {} + } + }) + }) + + const stats = manager.getStats() + expect(stats.averageExecutionTimeMs).toBeGreaterThan(0) + expect(stats.totalTransactions).toBe(2) + }) + + it('should calculate average operations per transaction', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + tx.addOperation({ execute: async () => undefined }) + }) + + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + tx.addOperation({ execute: async () => undefined }) + tx.addOperation({ execute: async () => undefined }) + tx.addOperation({ execute: async () => undefined }) + }) + + const stats = manager.getStats() + expect(stats.averageOperationsPerTransaction).toBe(3) // (2 + 4) / 2 + }) + + it('should allow resetting statistics', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }) + + let stats = manager.getStats() + expect(stats.totalTransactions).toBe(1) + + manager.resetStats() + + stats = manager.getStats() + expect(stats.totalTransactions).toBe(0) + expect(stats.successfulTransactions).toBe(0) + expect(stats.failedTransactions).toBe(0) + expect(stats.rolledBackTransactions).toBe(0) + expect(stats.averageExecutionTimeMs).toBe(0) + expect(stats.averageOperationsPerTransaction).toBe(0) + }) + }) + + describe('Concurrent Transactions', () => { + it('should handle concurrent transactions', async () => { + const results = await Promise.all([ + manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + return 1 + }), + manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + return 2 + }), + manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + return 3 + }) + ]) + + expect(results).toEqual([1, 2, 3]) + + const stats = manager.getStats() + expect(stats.totalTransactions).toBe(3) + expect(stats.successfulTransactions).toBe(3) + }) + + it('should track mixed success/failure in concurrent transactions', async () => { + const results = await Promise.allSettled([ + manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }), + manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + throw new Error('Failed') + } + }) + }), + manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }) + ]) + + expect(results[0].status).toBe('fulfilled') + expect(results[1].status).toBe('rejected') + expect(results[2].status).toBe('fulfilled') + + const stats = manager.getStats() + expect(stats.totalTransactions).toBe(3) + expect(stats.successfulTransactions).toBe(2) + expect(stats.failedTransactions).toBe(1) + }) + }) + + describe('Error Context', () => { + it('should wrap non-TransactionError errors', async () => { + const error = await manager.executeTransaction(async () => { + throw new Error('Custom error') + }).catch(e => e) + + expect(error).toBeInstanceOf(TransactionError) + expect(error.message).toContain('Custom error') + }) + + it('should preserve TransactionError instances', async () => { + const error = await manager.executeTransaction(async (tx) => { + tx.addOperation({ + execute: async () => { + throw new Error('Operation failed') + } + }) + }).catch(e => e) + + expect(error).toBeInstanceOf(TransactionError) + }) + }) + + describe('Stats Immutability', () => { + it('should return immutable stats copy', async () => { + await manager.executeTransaction(async (tx) => { + tx.addOperation({ execute: async () => undefined }) + }) + + const stats1 = manager.getStats() + const stats2 = manager.getStats() + + expect(stats1).not.toBe(stats2) // Different objects + expect(stats1).toEqual(stats2) // Same values + }) + }) +}) diff --git a/tests/transaction/integration/cow-transactions.test.ts b/tests/transaction/integration/cow-transactions.test.ts new file mode 100644 index 00000000..d203495b --- /dev/null +++ b/tests/transaction/integration/cow-transactions.test.ts @@ -0,0 +1,265 @@ +/** + * COW + Transactions Integration Tests + * + * Verifies that transactions work correctly with Copy-on-Write storage: + * - Branch isolation + * - Atomic commits + * - Rollback without affecting main branch + * - Content-addressable storage + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { tmpdir } from 'os' +import { join } from 'path' +import { mkdirSync, rmSync } from 'fs' + +describe('Transactions + COW Integration', () => { + let brain: Brainy + let testDir: string + + beforeEach(async () => { + // Create unique test directory + testDir = join(tmpdir(), `brainy-cow-test-${Date.now()}`) + mkdirSync(testDir, { recursive: true }) + + // Initialize Brainy with COW enabled + brain = new Brainy({ + storage: { + type: 'filesystem', + path: testDir + } + }) + + await brain.init() + + // Enable COW if available + if (brain.cow) { + await brain.cow.init() + } + }) + + afterEach(async () => { + if (brain) { + await brain.shutdown() + } + if (testDir) { + rmSync(testDir, { recursive: true, force: true }) + } + }) + + describe('Basic COW Operations', () => { + it('should commit transaction successfully on COW branch', async () => { + // Create branch + if (!brain.cow) { + console.log('COW not available, skipping test') + return + } + + await brain.cow.createBranch('feature-branch') + await brain.cow.checkout('feature-branch') + + // Add entity (should succeed) + const id = await brain.add({ + data: { name: 'Test Entity' }, + type: NounType.Thing + }) + + expect(id).toBeTruthy() + + // Verify entity exists + const entity = await brain.get(id) + expect(entity).toBeTruthy() + expect(entity?.data.name).toBe('Test Entity') + }) + + it('should isolate transaction rollback to branch', async () => { + if (!brain.cow) { + console.log('COW not available, skipping test') + return + } + + // Add entity on main branch + const mainId = await brain.add({ + data: { name: 'Main Entity' }, + type: NounType.Thing + }) + + // Create and checkout feature branch + await brain.cow.createBranch('feature-branch') + await brain.cow.checkout('feature-branch') + + // Try to add entity that will fail + try { + await brain.add({ + data: { name: 'Feature Entity' }, + type: NounType.Thing, + // Force failure by providing invalid vector + vector: [] as any + }) + } catch (e) { + // Expected to fail + } + + // Switch back to main + await brain.cow.checkout('main') + + // Main branch entity should still exist + const mainEntity = await brain.get(mainId) + expect(mainEntity).toBeTruthy() + expect(mainEntity?.data.name).toBe('Main Entity') + }) + + it('should handle atomic updates across COW branches', async () => { + if (!brain.cow) { + console.log('COW not available, skipping test') + return + } + + // Add entity + const id = await brain.add({ + data: { name: 'Original', version: 1 }, + type: NounType.Thing + }) + + // Create branch + await brain.cow.createBranch('feature-branch') + await brain.cow.checkout('feature-branch') + + // Update entity (atomic operation) + await brain.update({ + id, + data: { name: 'Updated', version: 2 }, + merge: false + }) + + // Verify update on feature branch + const featureEntity = await brain.get(id) + expect(featureEntity?.data.version).toBe(2) + + // Switch to main + await brain.cow.checkout('main') + + // Original should be unchanged on main + const mainEntity = await brain.get(id) + expect(mainEntity?.data.version).toBe(1) + }) + }) + + describe('Transaction Rollback with COW', () => { + it('should rollback failed transaction without affecting COW branch', async () => { + if (!brain.cow) { + console.log('COW not available, skipping test') + return + } + + await brain.cow.createBranch('test-branch') + await brain.cow.checkout('test-branch') + + // Add first entity (will succeed) + const id1 = await brain.add({ + data: { name: 'Entity 1' }, + type: NounType.Thing + }) + + // Attempt to add with invalid data (will fail) + // This tests that the transaction rollback doesn't corrupt the branch + let failed = false + try { + await brain.add({ + data: null as any, // Invalid + type: NounType.Thing + }) + } catch (e) { + failed = true + } + + expect(failed).toBe(true) + + // First entity should still exist (transaction rollback worked) + const entity1 = await brain.get(id1) + expect(entity1).toBeTruthy() + }) + }) + + describe('COW Branch Merging with Transactions', () => { + it('should preserve transaction atomicity during branch operations', async () => { + if (!brain.cow) { + console.log('COW not available, skipping test') + return + } + + // Add entity on main + const mainId = await brain.add({ + data: { name: 'Main Entity', branch: 'main' }, + type: NounType.Thing + }) + + // Create feature branch + await brain.cow.createBranch('feature') + await brain.cow.checkout('feature') + + // Add multiple entities in transaction (atomic) + const featureId1 = await brain.add({ + data: { name: 'Feature 1', branch: 'feature' }, + type: NounType.Thing + }) + + const featureId2 = await brain.add({ + data: { name: 'Feature 2', branch: 'feature' }, + type: NounType.Thing + }) + + // Create relationship (atomic) + await brain.relate({ + from: featureId1, + to: featureId2, + type: VerbType.RelatesTo + }) + + // All operations should be atomic on feature branch + const feature1 = await brain.get(featureId1) + const feature2 = await brain.get(featureId2) + const relations = await brain.getRelations({ from: featureId1 }) + + expect(feature1).toBeTruthy() + expect(feature2).toBeTruthy() + expect(relations).toHaveLength(1) + + // Switch back to main - feature entities should not exist + await brain.cow.checkout('main') + const mainCheck = await brain.get(featureId1) + expect(mainCheck).toBeNull() + }) + }) + + describe('Content-Addressable Storage', () => { + it('should handle content-addressable storage with transactions', async () => { + if (!brain.cow) { + console.log('COW not available, skipping test') + return + } + + // Add entity with specific data + const id = await brain.add({ + data: { content: 'Test content', value: 123 }, + type: NounType.Thing + }) + + // Update entity (creates new blob) + await brain.update({ + id, + data: { content: 'Updated content', value: 456 } + }) + + // Get entity - should have updated content + const entity = await brain.get(id) + expect(entity?.data.content).toBe('Updated content') + expect(entity?.data.value).toBe(456) + + // Transaction system should work with content-addressable storage + // (each version is a separate blob, rollback restores previous blob reference) + }) + }) +}) diff --git a/tests/transaction/integration/distributed-transactions.test.ts b/tests/transaction/integration/distributed-transactions.test.ts new file mode 100644 index 00000000..536a5a53 --- /dev/null +++ b/tests/transaction/integration/distributed-transactions.test.ts @@ -0,0 +1,393 @@ +/** + * Distributed + Transactions Integration Tests + * + * Verifies that transactions work correctly with distributed storage: + * - Remote storage adapters (S3, Azure, GCS) + * - Distributed coordination + * - Cache coherence + * - Read/write separation + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { tmpdir } from 'os' +import { join } from 'path' +import { mkdirSync, rmSync } from 'fs' + +describe('Transactions + Distributed Storage Integration', () => { + let brain: Brainy + let testDir: string + + beforeEach(async () => { + testDir = join(tmpdir(), `brainy-distributed-test-${Date.now()}`) + mkdirSync(testDir, { recursive: true }) + + // Use filesystem as proxy for distributed storage + // (In production, this would be S3, Azure, GCS, etc.) + brain = new Brainy({ + storage: { + type: 'filesystem', + path: testDir + } + }) + + await brain.init() + }) + + afterEach(async () => { + if (brain) { + await brain.shutdown() + } + if (testDir) { + rmSync(testDir, { recursive: true, force: true }) + } + }) + + describe('Remote Storage Adapters', () => { + it('should handle transactions with filesystem storage (proxy for remote)', async () => { + // Add entity (atomic operation) + const id = await brain.add({ + data: { name: 'Remote Entity', location: 'cloud' }, + type: NounType.Thing + }) + + expect(id).toBeTruthy() + + // Verify entity persisted to storage + const entity = await brain.get(id) + expect(entity).toBeTruthy() + expect(entity?.data.name).toBe('Remote Entity') + }) + + it('should rollback failed operations with remote storage', async () => { + // Add first entity (succeeds) + const id1 = await brain.add({ + data: { name: 'Entity 1' }, + type: NounType.Thing + }) + + // Attempt to add with invalid data (fails) + let failed = false + try { + await brain.add({ + data: null as any, + type: NounType.Thing + }) + } catch (e) { + failed = true + } + + expect(failed).toBe(true) + + // First entity should still exist + const entity1 = await brain.get(id1) + expect(entity1).toBeTruthy() + }) + + it('should handle update operations with remote storage atomically', async () => { + // Add entity + const id = await brain.add({ + data: { name: 'Original', version: 1 }, + type: NounType.Thing + }) + + // Update atomically + await brain.update({ + id, + data: { name: 'Updated', version: 2 } + }) + + // Verify update persisted + const entity = await brain.get(id) + expect(entity?.data.version).toBe(2) + }) + }) + + describe('Write Coordinator Atomicity', () => { + it('should ensure atomicity at write coordinator level', async () => { + // Simulate write coordinator scenario + // (Single Brainy instance coordinating writes) + + const entities: string[] = [] + + // Multiple atomic writes + for (let i = 0; i < 5; i++) { + const id = await brain.add({ + data: { name: `Entity ${i}`, index: i }, + type: NounType.Thing + }) + entities.push(id) + } + + // Create relationships (atomic) + for (let i = 0; i < entities.length - 1; i++) { + await brain.relate({ + from: entities[i], + to: entities[i + 1], + type: VerbType.RelatesTo + }) + } + + // Verify all operations succeeded + for (let i = 0; i < entities.length; i++) { + const entity = await brain.get(entities[i]) + expect(entity).toBeTruthy() + expect(entity?.data.index).toBe(i) + } + + // Verify relationships + for (let i = 0; i < entities.length - 1; i++) { + const relations = await brain.getRelations({ from: entities[i] }) + expect(relations).toHaveLength(1) + } + }) + + it('should handle batch operations atomically on write coordinator', async () => { + const batchSize = 20 + const ids: string[] = [] + + // Batch add operations + for (let i = 0; i < batchSize; i++) { + const id = await brain.add({ + data: { name: `Batch Entity ${i}`, batch: true }, + type: NounType.Thing + }) + ids.push(id) + } + + // Verify all entities persisted + let count = 0 + for (const id of ids) { + const entity = await brain.get(id) + if (entity) count++ + } + + expect(count).toBe(batchSize) + }) + }) + + describe('Read-After-Write Consistency', () => { + it('should ensure read-after-write consistency', async () => { + // Write entity + const id = await brain.add({ + data: { name: 'RAW Test', timestamp: Date.now() }, + type: NounType.Thing + }) + + // Immediate read (should see the write) + const entity = await brain.get(id) + expect(entity).toBeTruthy() + expect(entity?.data.name).toBe('RAW Test') + }) + + it('should maintain consistency after update', async () => { + const id = await brain.add({ + data: { name: 'Original', counter: 0 }, + type: NounType.Thing + }) + + // Multiple updates + for (let i = 1; i <= 5; i++) { + await brain.update({ + id, + data: { name: `Updated ${i}`, counter: i } + }) + + // Read immediately after each update + const entity = await brain.get(id) + expect(entity?.data.counter).toBe(i) + } + }) + }) + + describe('Concurrent Write Handling', () => { + it('should handle sequential writes correctly', async () => { + const ids: string[] = [] + + // Sequential writes (simulating distributed writes to coordinator) + for (let i = 0; i < 10; i++) { + const id = await brain.add({ + data: { name: `Sequential ${i}`, order: i }, + type: NounType.Thing + }) + ids.push(id) + } + + // Verify all writes succeeded + for (let i = 0; i < ids.length; i++) { + const entity = await brain.get(ids[i]) + expect(entity?.data.order).toBe(i) + } + }) + + it('should handle interleaved operations atomically', async () => { + // Create entities + const id1 = await brain.add({ + data: { name: 'Entity A', value: 100 }, + type: NounType.Thing + }) + + const id2 = await brain.add({ + data: { name: 'Entity B', value: 200 }, + type: NounType.Thing + }) + + // Interleaved updates + await brain.update({ id: id1, data: { value: 150 } }) + await brain.update({ id: id2, data: { value: 250 } }) + await brain.update({ id: id1, data: { value: 175 } }) + + // Verify final state + const entity1 = await brain.get(id1) + const entity2 = await brain.get(id2) + + expect(entity1?.data.value).toBe(175) + expect(entity2?.data.value).toBe(250) + }) + }) + + describe('Delete Operations with Distributed Storage', () => { + it('should handle delete operations atomically', async () => { + // Create entity + const id = await brain.add({ + data: { name: 'To Delete', status: 'active' }, + type: NounType.Thing + }) + + // Verify exists + let entity = await brain.get(id) + expect(entity).toBeTruthy() + + // Delete atomically + await brain.delete(id) + + // Verify deleted + entity = await brain.get(id) + expect(entity).toBeNull() + }) + + it('should handle delete with relationships atomically', async () => { + // Create entities and relationships + const id1 = await brain.add({ + data: { name: 'Entity 1' }, + type: NounType.Thing + }) + + const id2 = await brain.add({ + data: { name: 'Entity 2' }, + type: NounType.Thing + }) + + await brain.relate({ + from: id1, + to: id2, + type: VerbType.RelatesTo + }) + + // Delete first entity (should delete relationships) + await brain.delete(id1) + + // Verify entity deleted + const entity1 = await brain.get(id1) + expect(entity1).toBeNull() + + // Verify relationships deleted + const relations = await brain.getRelations({ from: id1 }) + expect(relations).toHaveLength(0) + + // Entity 2 should still exist + const entity2 = await brain.get(id2) + expect(entity2).toBeTruthy() + }) + }) + + describe('Storage Adapter Transparency', () => { + it('should work transparently with any storage adapter', async () => { + // This test verifies that transactions don't make assumptions + // about the underlying storage implementation + + // Add entity + const id = await brain.add({ + data: { name: 'Adapter Test', adapter: 'filesystem' }, + type: NounType.Thing + }) + + // Update entity + await brain.update({ + id, + data: { name: 'Updated via Adapter', adapter: 'filesystem' } + }) + + // Query entity + const entity = await brain.get(id) + expect(entity).toBeTruthy() + expect(entity?.data.name).toBe('Updated via Adapter') + + // Delete entity + await brain.delete(id) + const deletedEntity = await brain.get(id) + expect(deletedEntity).toBeNull() + }) + + it('should maintain atomicity regardless of storage latency', async () => { + // Simulate scenario with storage latency + // (In distributed setup, network latency is a factor) + + const startTime = Date.now() + + // Operations that might have latency + const id1 = await brain.add({ + data: { name: 'Latency Test 1' }, + type: NounType.Thing + }) + + const id2 = await brain.add({ + data: { name: 'Latency Test 2' }, + type: NounType.Thing + }) + + await brain.relate({ + from: id1, + to: id2, + type: VerbType.RelatesTo + }) + + const endTime = Date.now() + const duration = endTime - startTime + + // Verify all operations succeeded (regardless of latency) + const entity1 = await brain.get(id1) + const entity2 = await brain.get(id2) + const relations = await brain.getRelations({ from: id1 }) + + expect(entity1).toBeTruthy() + expect(entity2).toBeTruthy() + expect(relations).toHaveLength(1) + + // Should complete in reasonable time (even with storage latency) + expect(duration).toBeLessThan(5000) + }) + }) + + describe('Transaction Statistics with Distributed Storage', () => { + it('should track transaction statistics accurately', async () => { + // Get transaction manager stats + const stats = (brain as any).transactionManager?.getStats() + + if (stats) { + const initialTotal = stats.totalTransactions + + // Perform operations + await brain.add({ + data: { name: 'Stats Test' }, + type: NounType.Thing + }) + + // Check stats updated + const updatedStats = (brain as any).transactionManager?.getStats() + expect(updatedStats.totalTransactions).toBeGreaterThan(initialTotal) + } + }) + }) +}) diff --git a/tests/transaction/integration/sharding-transactions.test.ts b/tests/transaction/integration/sharding-transactions.test.ts new file mode 100644 index 00000000..cb884a3c --- /dev/null +++ b/tests/transaction/integration/sharding-transactions.test.ts @@ -0,0 +1,295 @@ +/** + * Sharding + Transactions Integration Tests + * + * Verifies that transactions work correctly with sharded storage: + * - Cross-shard atomicity + * - Shard-aware routing + * - Rollback across shards + * - UUID-based shard distribution + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { tmpdir } from 'os' +import { join } from 'path' +import { mkdirSync, rmSync } from 'fs' + +describe('Transactions + Sharding Integration', () => { + let brain: Brainy + let testDir: string + + beforeEach(async () => { + testDir = join(tmpdir(), `brainy-shard-test-${Date.now()}`) + mkdirSync(testDir, { recursive: true }) + + brain = new Brainy({ + storage: { + type: 'filesystem', + path: testDir + } + }) + + await brain.init() + }) + + afterEach(async () => { + if (brain) { + await brain.shutdown() + } + if (testDir) { + rmSync(testDir, { recursive: true, force: true }) + } + }) + + describe('Cross-Shard Operations', () => { + it('should handle atomic operations across multiple shards', async () => { + // Create entities with different UUID prefixes (different shards) + const id1 = 'aaa00000-1111-4111-8111-111111111111' // Shard: aaa + const id2 = 'bbb00000-2222-4222-8222-222222222222' // Shard: bbb + + // Add entities to different shards (atomic) + await brain.add({ + id: id1, + data: { name: 'Entity in Shard A', shard: 'aaa' }, + type: NounType.Thing + }) + + await brain.add({ + id: id2, + data: { name: 'Entity in Shard B', shard: 'bbb' }, + type: NounType.Thing + }) + + // Create relationship across shards (atomic) + const relationId = await brain.relate({ + from: id1, + to: id2, + type: VerbType.RelatesTo + }) + + // Verify all entities exist + const entity1 = await brain.get(id1) + const entity2 = await brain.get(id2) + const relations = await brain.getRelations({ from: id1 }) + + expect(entity1).toBeTruthy() + expect(entity2).toBeTruthy() + expect(relations).toHaveLength(1) + expect(relations[0].targetId).toBe(id2) + }) + + it('should rollback operations across multiple shards', async () => { + const id1 = 'ccc00000-1111-4111-8111-111111111111' // Shard: ccc + const id2 = 'ddd00000-2222-4222-8222-222222222222' // Shard: ddd + + // Add first entity (succeeds) + await brain.add({ + id: id1, + data: { name: 'Entity in Shard C' }, + type: NounType.Thing + }) + + // Attempt to add second entity with invalid data (fails) + let failed = false + try { + await brain.add({ + id: id2, + data: null as any, // Invalid + type: NounType.Thing + }) + } catch (e) { + failed = true + } + + expect(failed).toBe(true) + + // First entity should still exist (in shard C) + const entity1 = await brain.get(id1) + expect(entity1).toBeTruthy() + + // Second entity should not exist (rollback in shard D) + const entity2 = await brain.get(id2) + expect(entity2).toBeNull() + }) + + it('should handle updates across shards atomically', async () => { + const id1 = 'eee00000-1111-4111-8111-111111111111' + const id2 = 'fff00000-2222-4222-8222-222222222222' + + // Add entities + await brain.add({ + id: id1, + data: { name: 'Original E', version: 1 }, + type: NounType.Thing + }) + + await brain.add({ + id: id2, + data: { name: 'Original F', version: 1 }, + type: NounType.Thing + }) + + // Update both entities (different shards) + await brain.update({ + id: id1, + data: { name: 'Updated E', version: 2 } + }) + + await brain.update({ + id: id2, + data: { name: 'Updated F', version: 2 } + }) + + // Verify updates in both shards + const entity1 = await brain.get(id1) + const entity2 = await brain.get(id2) + + expect(entity1?.data.version).toBe(2) + expect(entity2?.data.version).toBe(2) + }) + }) + + describe('Shard Distribution', () => { + it('should distribute entities across shards based on UUID', async () => { + // Create entities with various UUID prefixes + const ids = [ + 'aaa00000-1111-4111-8111-111111111111', + 'bbb00000-2222-4222-8222-222222222222', + 'ccc00000-3333-4333-8333-333333333333', + 'ddd00000-4444-4444-8444-444444444444' + ] + + // Add entities to different shards + for (let i = 0; i < ids.length; i++) { + await brain.add({ + id: ids[i], + data: { name: `Entity ${i}`, index: i }, + type: NounType.Thing + }) + } + + // Verify all entities can be retrieved (regardless of shard) + for (let i = 0; i < ids.length; i++) { + const entity = await brain.get(ids[i]) + expect(entity).toBeTruthy() + expect(entity?.data.index).toBe(i) + } + }) + + it('should handle relationships between different shard combinations', async () => { + const entities = [ + { id: 'aaa00000-1111-4111-8111-111111111111', name: 'A' }, + { id: 'bbb00000-2222-4222-8222-222222222222', name: 'B' }, + { id: 'ccc00000-3333-4333-8333-333333333333', name: 'C' } + ] + + // Add all entities + for (const e of entities) { + await brain.add({ + id: e.id, + data: { name: e.name }, + type: NounType.Thing + }) + } + + // Create relationships across all shards + await brain.relate({ + from: entities[0].id, + to: entities[1].id, + type: VerbType.RelatesTo + }) + + await brain.relate({ + from: entities[1].id, + to: entities[2].id, + type: VerbType.RelatesTo + }) + + await brain.relate({ + from: entities[2].id, + to: entities[0].id, + type: VerbType.RelatesTo + }) + + // Verify all relationships exist + const relations0 = await brain.getRelations({ from: entities[0].id }) + const relations1 = await brain.getRelations({ from: entities[1].id }) + const relations2 = await brain.getRelations({ from: entities[2].id }) + + expect(relations0).toHaveLength(1) + expect(relations1).toHaveLength(1) + expect(relations2).toHaveLength(1) + }) + }) + + describe('Delete Operations Across Shards', () => { + it('should delete entities and relationships across shards atomically', async () => { + const id1 = 'ggg00000-1111-4111-8111-111111111111' + const id2 = 'hhh00000-2222-4222-8222-222222222222' + + // Add entities in different shards + await brain.add({ + id: id1, + data: { name: 'Entity G' }, + type: NounType.Thing + }) + + await brain.add({ + id: id2, + data: { name: 'Entity H' }, + type: NounType.Thing + }) + + // Create relationship + await brain.relate({ + from: id1, + to: id2, + type: VerbType.RelatesTo + }) + + // Delete first entity (should also delete relationship) + await brain.delete(id1) + + // Verify entity deleted from shard G + const entity1 = await brain.get(id1) + expect(entity1).toBeNull() + + // Entity H should still exist in shard H + const entity2 = await brain.get(id2) + expect(entity2).toBeTruthy() + + // Relationship should be deleted + const relations = await brain.getRelations({ from: id1 }) + expect(relations).toHaveLength(0) + }) + }) + + describe('Batch Operations Across Shards', () => { + it('should handle batch adds across multiple shards atomically', async () => { + const entities = [ + { id: 'shard1-00-1111-4111-8111-111111111111', data: { name: 'S1-E1' } }, + { id: 'shard2-00-2222-4222-8222-222222222222', data: { name: 'S2-E1' } }, + { id: 'shard3-00-3333-4333-8333-333333333333', data: { name: 'S3-E1' } }, + { id: 'shard1-00-4444-4444-8444-444444444444', data: { name: 'S1-E2' } }, + { id: 'shard2-00-5555-4555-8555-555555555555', data: { name: 'S2-E2' } } + ] + + // Add all entities (distributed across shards) + for (const e of entities) { + await brain.add({ + id: e.id, + data: e.data, + type: NounType.Thing + }) + } + + // Verify all entities exist in their respective shards + for (const e of entities) { + const entity = await brain.get(e.id) + expect(entity).toBeTruthy() + expect(entity?.data.name).toBe(e.data.name) + } + }) + }) +}) diff --git a/tests/transaction/integration/typeaware-transactions.test.ts b/tests/transaction/integration/typeaware-transactions.test.ts new file mode 100644 index 00000000..9a4c8ccf --- /dev/null +++ b/tests/transaction/integration/typeaware-transactions.test.ts @@ -0,0 +1,368 @@ +/** + * TypeAware + Transactions Integration Tests + * + * Verifies that transactions work correctly with type-aware storage: + * - Type-specific routing + * - Type cache updates + * - Type changes during updates + * - Per-type performance optimization + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import { tmpdir } from 'os' +import { join } from 'path' +import { mkdirSync, rmSync } from 'fs' + +describe('Transactions + TypeAware Storage Integration', () => { + let brain: Brainy + let testDir: string + + beforeEach(async () => { + testDir = join(tmpdir(), `brainy-typeaware-test-${Date.now()}`) + mkdirSync(testDir, { recursive: true }) + + brain = new Brainy({ + storage: { + type: 'filesystem', + path: testDir + } + }) + + await brain.init() + }) + + afterEach(async () => { + if (brain) { + await brain.shutdown() + } + if (testDir) { + rmSync(testDir, { recursive: true, force: true }) + } + }) + + describe('Type-Specific Routing', () => { + it('should route entities to type-specific storage atomically', async () => { + // Add entities of different types + const personId = await brain.add({ + data: { name: 'John Doe', role: 'Developer' }, + type: NounType.Person + }) + + const orgId = await brain.add({ + data: { name: 'Acme Corp', industry: 'Tech' }, + type: NounType.Organization + }) + + const placeId = await brain.add({ + data: { name: 'San Francisco', country: 'USA' }, + type: NounType.Place + }) + + // Verify all entities stored with correct types + const person = await brain.get(personId) + const org = await brain.get(orgId) + const place = await brain.get(placeId) + + expect(person?.type).toBe(NounType.Person) + expect(org?.type).toBe(NounType.Organization) + expect(place?.type).toBe(NounType.Place) + }) + + it('should handle type-specific queries atomically', async () => { + // Add multiple entities of same type + await brain.add({ + data: { name: 'Alice', role: 'Engineer' }, + type: NounType.Person + }) + + await brain.add({ + data: { name: 'Bob', role: 'Designer' }, + type: NounType.Person + }) + + await brain.add({ + data: { name: 'Acme', industry: 'Tech' }, + type: NounType.Organization + }) + + // Query by type + const results = await brain.find({ + filter: { type: NounType.Person } + }) + + expect(results).toHaveLength(2) + expect(results.every(r => r.type === NounType.Person)).toBe(true) + }) + }) + + describe('Type Changes During Updates', () => { + it('should handle type changes atomically in update operations', async () => { + // Add entity as Person + const id = await brain.add({ + data: { name: 'John Smith', category: 'individual' }, + type: NounType.Person + }) + + // Verify initial type + let entity = await brain.get(id) + expect(entity?.type).toBe(NounType.Person) + + // Update to Organization (type change) + await brain.update({ + id, + type: NounType.Organization, + data: { name: 'Smith Corp', category: 'business' } + }) + + // Verify type changed + entity = await brain.get(id) + expect(entity?.type).toBe(NounType.Organization) + expect(entity?.data.category).toBe('business') + }) + + it('should rollback type changes on failed update', async () => { + // Add entity as Person + const id = await brain.add({ + data: { name: 'Jane Doe' }, + type: NounType.Person + }) + + // Attempt to update with invalid data (will fail) + let failed = false + try { + await brain.update({ + id, + type: NounType.Organization, + data: null as any // Invalid + }) + } catch (e) { + failed = true + } + + expect(failed).toBe(true) + + // Type should remain Person (rollback) + const entity = await brain.get(id) + expect(entity?.type).toBe(NounType.Person) + expect(entity?.data.name).toBe('Jane Doe') + }) + }) + + describe('Type Cache Consistency', () => { + it('should maintain type cache consistency during transactions', async () => { + // Add multiple entities + const ids: string[] = [] + const types = [ + NounType.Person, + NounType.Organization, + NounType.Place, + NounType.Event, + NounType.Concept + ] + + for (let i = 0; i < types.length; i++) { + const id = await brain.add({ + data: { name: `Entity ${i}`, index: i }, + type: types[i] + }) + ids.push(id) + } + + // Verify all types cached correctly + for (let i = 0; i < ids.length; i++) { + const entity = await brain.get(ids[i]) + expect(entity?.type).toBe(types[i]) + } + }) + + it('should update type cache on type changes', async () => { + const id = await brain.add({ + data: { name: 'Initial' }, + type: NounType.Thing + }) + + // Sequence of type changes + const typeSequence = [ + NounType.Concept, + NounType.Event, + NounType.Organization, + NounType.Person + ] + + for (const newType of typeSequence) { + await brain.update({ + id, + type: newType, + data: { name: `As ${newType}` } + }) + + const entity = await brain.get(id) + expect(entity?.type).toBe(newType) + } + }) + }) + + describe('Per-Type Performance', () => { + it('should handle large numbers of same-type entities efficiently', async () => { + const startTime = Date.now() + + // Add 50 entities of same type (type-aware storage optimization) + const ids: string[] = [] + for (let i = 0; i < 50; i++) { + const id = await brain.add({ + data: { name: `Person ${i}`, index: i }, + type: NounType.Person + }) + ids.push(id) + } + + const addTime = Date.now() - startTime + + // Verify all entities stored + const retrieveStart = Date.now() + for (const id of ids) { + const entity = await brain.get(id) + expect(entity).toBeTruthy() + } + const retrieveTime = Date.now() - retrieveStart + + // Type-aware storage should be reasonably fast + expect(addTime).toBeLessThan(5000) // 5 seconds for 50 adds + expect(retrieveTime).toBeLessThan(2000) // 2 seconds for 50 retrieves + }) + }) + + describe('Mixed Type Operations', () => { + it('should handle operations across multiple types atomically', async () => { + // Create entities of different types + const personId = await brain.add({ + data: { name: 'Alice' }, + type: NounType.Person + }) + + const orgId = await brain.add({ + data: { name: 'TechCorp' }, + type: NounType.Organization + }) + + const eventId = await brain.add({ + data: { name: 'Conference 2024' }, + type: NounType.Event + }) + + // Create relationships across types (atomic) + await brain.relate({ + from: personId, + to: orgId, + type: VerbType.WorksFor + }) + + await brain.relate({ + from: personId, + to: eventId, + type: VerbType.Attends + }) + + await brain.relate({ + from: orgId, + to: eventId, + type: VerbType.Sponsors + }) + + // Verify relationships exist + const personRelations = await brain.getRelations({ from: personId }) + const orgRelations = await brain.getRelations({ from: orgId }) + + expect(personRelations).toHaveLength(2) + expect(orgRelations).toHaveLength(1) + }) + + it('should handle delete with cascade across types', async () => { + // Create multi-type graph + const personId = await brain.add({ + data: { name: 'Bob' }, + type: NounType.Person + }) + + const projectId = await brain.add({ + data: { name: 'Project X' }, + type: NounType.Thing + }) + + const taskId = await brain.add({ + data: { name: 'Task 1' }, + type: NounType.Thing + }) + + // Create relationships + await brain.relate({ + from: personId, + to: projectId, + type: VerbType.WorksOn + }) + + await brain.relate({ + from: projectId, + to: taskId, + type: VerbType.Contains + }) + + // Delete person (should cascade delete relationships) + await brain.delete(personId) + + // Verify person deleted + const person = await brain.get(personId) + expect(person).toBeNull() + + // Verify project and task still exist (different types) + const project = await brain.get(projectId) + const task = await brain.get(taskId) + expect(project).toBeTruthy() + expect(task).toBeTruthy() + + // Verify relationships from person are deleted + const relations = await brain.getRelations({ from: personId }) + expect(relations).toHaveLength(0) + }) + }) + + describe('Type Validation', () => { + it('should validate types during atomic operations', async () => { + // Valid type - should succeed + const id = await brain.add({ + data: { name: 'Test' }, + type: NounType.Person + }) + + expect(id).toBeTruthy() + + // Verify type stored correctly + const entity = await brain.get(id) + expect(entity?.type).toBe(NounType.Person) + }) + + it('should handle type-specific metadata atomically', async () => { + // Add with type-specific metadata + const personId = await brain.add({ + data: { name: 'Charlie', age: 30, occupation: 'Engineer' }, + type: NounType.Person, + metadata: { verified: true, source: 'HR' } + }) + + const orgId = await brain.add({ + data: { name: 'StartupCo', employees: 50, founded: 2020 }, + type: NounType.Organization, + metadata: { verified: false, source: 'Registration' } + }) + + // Verify metadata with type context + const person = await brain.get(personId) + const org = await brain.get(orgId) + + expect(person?.metadata?.verified).toBe(true) + expect(org?.metadata?.verified).toBe(false) + }) + }) +}) diff --git a/tests/unit/brainy/relate-duplicate-optimization.test.ts b/tests/unit/brainy/relate-duplicate-optimization.test.ts new file mode 100644 index 00000000..d10e3732 --- /dev/null +++ b/tests/unit/brainy/relate-duplicate-optimization.test.ts @@ -0,0 +1,220 @@ +/** + * Duplicate Relationship Check Optimization Tests + * + * Tests for v5.8.0 optimization that uses GraphAdjacencyIndex + * for O(log n) duplicate detection instead of O(n) storage scan. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' + +describe('Duplicate Check Optimization', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy() + await brain.init() + }) + + afterEach(async () => { + // Cleanup is automatic with memory storage + }) + + it('should detect duplicate relationships using GraphAdjacencyIndex', async () => { + // Create two entities + const personId = await brain.add({ + data: { name: 'Alice' }, + type: NounType.Person + }) + + const orgId = await brain.add({ + data: { name: 'Acme Corp' }, + type: NounType.Organization + }) + + // Create first relationship + const relationId1 = await brain.relate({ + from: personId, + to: orgId, + type: VerbType.ParticipatesIn + }) + + // Attempt to create duplicate relationship + const relationId2 = await brain.relate({ + from: personId, + to: orgId, + type: VerbType.ParticipatesIn + }) + + // Should return the same ID (duplicate detected) + expect(relationId2).toBe(relationId1) + + // Verify only one relationship exists + const relations = await brain.getRelations({ from: personId }) + expect(relations).toHaveLength(1) + expect(relations[0].id).toBe(relationId1) + }) + + it('should allow different relationship types between same entities', async () => { + const personId = await brain.add({ + data: { name: 'Bob' }, + type: NounType.Person + }) + + const projectId = await brain.add({ + data: { name: 'Project X' }, + type: NounType.Thing + }) + + // Create first relationship + const relationId1 = await brain.relate({ + from: personId, + to: projectId, + type: VerbType.Creates + }) + + // Create second relationship with different type (not a duplicate) + const relationId2 = await brain.relate({ + from: personId, + to: projectId, + type: VerbType.Modifies + }) + + // Should be different IDs (different verb types) + expect(relationId2).not.toBe(relationId1) + + // Verify both relationships exist + const relations = await brain.getRelations({ from: personId }) + expect(relations).toHaveLength(2) + // Both relations should exist with different IDs + const relationIds = relations.map(r => r.id) + expect(relationIds).toContain(relationId1) + expect(relationIds).toContain(relationId2) + }) + + it('should handle duplicate check with many relationships (performance)', async () => { + // Create source entity + const sourceId = await brain.add({ + data: { name: 'Hub Entity' }, + type: NounType.Thing + }) + + // Create 50 target entities and relationships + const targetIds: string[] = [] + for (let i = 0; i < 50; i++) { + const targetId = await brain.add({ + data: { name: `Target ${i}` }, + type: NounType.Thing + }) + targetIds.push(targetId) + + await brain.relate({ + from: sourceId, + to: targetId, + type: VerbType.RelatesTo + }) + } + + // Now attempt to create duplicate with first target (should be fast with GraphIndex) + const startTime = performance.now() + const duplicateId = await brain.relate({ + from: sourceId, + to: targetIds[0], + type: VerbType.RelatesTo + }) + const elapsed = performance.now() - startTime + + // Should be fast with O(log n) GraphIndex lookup (< 10ms even with 50 relationships) + expect(elapsed).toBeLessThan(10) + + // Verify duplicate was detected + const relations = await brain.getRelations({ from: sourceId }) + expect(relations).toHaveLength(50) // No duplicate created + }) + + it('should use cached verb data for duplicate check', async () => { + const entityA = await brain.add({ + data: { name: 'Entity A' }, + type: NounType.Thing + }) + + const entityB = await brain.add({ + data: { name: 'Entity B' }, + type: NounType.Thing + }) + + // Create relationship + const relationId1 = await brain.relate({ + from: entityA, + to: entityB, + type: VerbType.RelatesTo + }) + + // Access GraphIndex to ensure verb is cached + const verbIds = await (brain as any).graphIndex.getVerbIdsBySource(entityA) + expect(verbIds).toContain(relationId1) + + // Attempt duplicate (should use cached verb) + const startTime = performance.now() + const relationId2 = await brain.relate({ + from: entityA, + to: entityB, + type: VerbType.RelatesTo + }) + const elapsed = performance.now() - startTime + + // Should be very fast with cached verb (< 5ms) + expect(elapsed).toBeLessThan(5) + expect(relationId2).toBe(relationId1) + }) + + it('should handle duplicate check across multiple verb types efficiently', async () => { + const person = await brain.add({ + data: { name: 'Charlie' }, + type: NounType.Person + }) + + const org = await brain.add({ + data: { name: 'BigCorp' }, + type: NounType.Organization + }) + + // Create different relationship types + const rel1 = await brain.relate({ + from: person, + to: org, + type: VerbType.Affects + }) + + const rel2 = await brain.relate({ + from: person, + to: org, + type: VerbType.Owns + }) + + // Verify both relationships exist + let relations = await brain.getRelations({ from: person }) + expect(relations).toHaveLength(2) + const relationIds = relations.map(r => r.id) + expect(relationIds).toContain(rel1) + expect(relationIds).toContain(rel2) + + // Attempt duplicate of first relationship type + const startTime = performance.now() + const duplicate = await brain.relate({ + from: person, + to: org, + type: VerbType.Affects + }) + const elapsed = performance.now() - startTime + + // Should detect duplicate efficiently (< 20ms) + expect(elapsed).toBeLessThan(20) + expect(duplicate).toBe(rel1) + + // Verify still same number of relationships (no duplicate added) + const finalRelations = await brain.getRelations({ from: person }) + expect(finalRelations.length).toBe(relations.length) + }) +}) diff --git a/tests/unit/graph/graphIndex-pagination.test.ts b/tests/unit/graph/graphIndex-pagination.test.ts new file mode 100644 index 00000000..c706d91c --- /dev/null +++ b/tests/unit/graph/graphIndex-pagination.test.ts @@ -0,0 +1,343 @@ +/** + * GraphAdjacencyIndex Pagination Tests (v5.8.0) + * + * Tests for pagination support in GraphIndex methods: + * - getNeighbors() with limit/offset + * - getVerbIdsBySource() with limit/offset + * - getVerbIdsByTarget() with limit/offset + * + * Verifies backward compatibility and new pagination features + */ + +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' + +describe('GraphAdjacencyIndex Pagination', () => { + let brain: Brainy + let centralId: string + let neighborIds: string[] + + beforeEach(async () => { + brain = new Brainy() + await brain.init() + + // Create central entity + centralId = await brain.add({ + data: { name: 'Central Hub' }, + type: NounType.Thing + }) + + // Create 50 neighbor entities with relationships + neighborIds = [] + for (let i = 0; i < 50; i++) { + const neighborId = await brain.add({ + data: { name: `Neighbor ${i}`, index: i }, + type: NounType.Thing + }) + neighborIds.push(neighborId) + + // Create outgoing relationship from central to neighbor + await brain.relate({ + from: centralId, + to: neighborId, + type: VerbType.RelatesTo + }) + } + }) + + describe('getNeighbors() Pagination', () => { + it('should return all neighbors without pagination (backward compatible)', async () => { + const graphIndex = (brain as any).graphIndex + + const neighbors = await graphIndex.getNeighbors(centralId) + + // Should return all 50 neighbors + expect(neighbors).toHaveLength(50) + expect(neighbors.every((id: string) => neighborIds.includes(id))).toBe(true) + }) + + it('should return all outgoing neighbors with direction parameter (backward compatible)', async () => { + const graphIndex = (brain as any).graphIndex + + // Old API: direction as string + const neighbors = await graphIndex.getNeighbors(centralId, 'out') + + expect(neighbors).toHaveLength(50) + }) + + it('should paginate with limit only', async () => { + const graphIndex = (brain as any).graphIndex + + const page1 = await graphIndex.getNeighbors(centralId, { limit: 10 }) + + expect(page1).toHaveLength(10) + expect(page1.every((id: string) => neighborIds.includes(id))).toBe(true) + }) + + it('should paginate with offset only', async () => { + const graphIndex = (brain as any).graphIndex + + const all = await graphIndex.getNeighbors(centralId) + const offsetResults = await graphIndex.getNeighbors(centralId, { offset: 10 }) + + // Offset 10 should return all after first 10 + expect(offsetResults).toHaveLength(all.length - 10) + }) + + it('should paginate with both limit and offset', async () => { + const graphIndex = (brain as any).graphIndex + + const page1 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 0 }) + const page2 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 10 }) + const page3 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 20 }) + + // Each page should have 10 results + expect(page1).toHaveLength(10) + expect(page2).toHaveLength(10) + expect(page3).toHaveLength(10) + + // Pages should not overlap + const page1Set = new Set(page1) + const page2Set = new Set(page2) + const page3Set = new Set(page3) + + const overlap12 = page1.filter(id => page2Set.has(id)) + const overlap23 = page2.filter(id => page3Set.has(id)) + + expect(overlap12).toHaveLength(0) + expect(overlap23).toHaveLength(0) + }) + + it('should handle offset beyond total count', async () => { + const graphIndex = (brain as any).graphIndex + + const results = await graphIndex.getNeighbors(centralId, { offset: 100 }) + + // Offset 100 > 50 total → empty array + expect(results).toHaveLength(0) + }) + + it('should handle limit = 0', async () => { + const graphIndex = (brain as any).graphIndex + + const results = await graphIndex.getNeighbors(centralId, { limit: 0 }) + + expect(results).toHaveLength(0) + }) + + it('should paginate with direction filter', async () => { + const graphIndex = (brain as any).graphIndex + + // Get first 10 outgoing neighbors + const page1 = await graphIndex.getNeighbors(centralId, { + direction: 'out', + limit: 10, + offset: 0 + }) + + expect(page1).toHaveLength(10) + }) + + it('should handle pagination with incoming direction', async () => { + const graphIndex = (brain as any).graphIndex + + // Create some incoming relationships + const sourceId = await brain.add({ + data: { name: 'Source' }, + type: NounType.Thing + }) + + await brain.relate({ + from: sourceId, + to: centralId, + type: VerbType.RelatesTo + }) + + // Get incoming neighbors with pagination + const incoming = await graphIndex.getNeighbors(centralId, { + direction: 'in', + limit: 10 + }) + + expect(incoming.length).toBeGreaterThan(0) + }) + }) + + describe('getVerbIdsBySource() Pagination', () => { + it('should return all verb IDs without pagination (backward compatible)', async () => { + const graphIndex = (brain as any).graphIndex + + const verbIds = await graphIndex.getVerbIdsBySource(centralId) + + // Should return all 50 verb IDs + expect(verbIds).toHaveLength(50) + }) + + it('should paginate verb IDs with limit', async () => { + const graphIndex = (brain as any).graphIndex + + const page1 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10 }) + + expect(page1).toHaveLength(10) + }) + + it('should paginate verb IDs with offset', async () => { + const graphIndex = (brain as any).graphIndex + + const all = await graphIndex.getVerbIdsBySource(centralId) + const offsetResults = await graphIndex.getVerbIdsBySource(centralId, { offset: 10 }) + + expect(offsetResults).toHaveLength(all.length - 10) + }) + + it('should paginate verb IDs with limit and offset', async () => { + const graphIndex = (brain as any).graphIndex + + const page1 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 0 }) + const page2 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 10 }) + const page3 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 20 }) + + // Each page should have 10 results + expect(page1).toHaveLength(10) + expect(page2).toHaveLength(10) + expect(page3).toHaveLength(10) + + // Pages should not overlap + const combined = [...page1, ...page2, ...page3] + const unique = new Set(combined) + expect(unique.size).toBe(30) // All unique + }) + + it('should handle pagination edge cases for verb IDs', async () => { + const graphIndex = (brain as any).graphIndex + + // Offset beyond total + const beyondTotal = await graphIndex.getVerbIdsBySource(centralId, { offset: 100 }) + expect(beyondTotal).toHaveLength(0) + + // Limit = 0 + const zeroLimit = await graphIndex.getVerbIdsBySource(centralId, { limit: 0 }) + expect(zeroLimit).toHaveLength(0) + }) + }) + + describe('getVerbIdsByTarget() Pagination', () => { + it('should return all verb IDs targeting an entity (backward compatible)', async () => { + const graphIndex = (brain as any).graphIndex + + // Pick a neighbor that's a target of relationships + const targetId = neighborIds[0] + const verbIds = await graphIndex.getVerbIdsByTarget(targetId) + + // Should have at least 1 (the relationship from central) + expect(verbIds.length).toBeGreaterThanOrEqual(1) + }) + + it('should paginate verb IDs by target with limit', async () => { + const graphIndex = (brain as any).graphIndex + + // Create entity with many incoming relationships + const popularTarget = await brain.add({ + data: { name: 'Popular Target' }, + type: NounType.Thing + }) + + // Create 30 relationships pointing to it + for (let i = 0; i < 30; i++) { + const sourceId = await brain.add({ + data: { name: `Source ${i}` }, + type: NounType.Thing + }) + await brain.relate({ + from: sourceId, + to: popularTarget, + type: VerbType.RelatesTo + }) + } + + // Paginate + const page1 = await graphIndex.getVerbIdsByTarget(popularTarget, { limit: 10 }) + const page2 = await graphIndex.getVerbIdsByTarget(popularTarget, { limit: 10, offset: 10 }) + + expect(page1).toHaveLength(10) + expect(page2).toHaveLength(10) + + // No overlap + const overlap = page1.filter((id: string) => page2.includes(id)) + expect(overlap).toHaveLength(0) + }) + }) + + describe('Performance with Pagination', () => { + it('should maintain sub-5ms performance with pagination', async () => { + const graphIndex = (brain as any).graphIndex + + // Multiple paginated queries should be fast + const startTime = performance.now() + + await graphIndex.getNeighbors(centralId, { limit: 10, offset: 0 }) + await graphIndex.getNeighbors(centralId, { limit: 10, offset: 10 }) + await graphIndex.getNeighbors(centralId, { limit: 10, offset: 20 }) + + const elapsed = performance.now() - startTime + + // 3 paginated queries should complete in < 15ms (< 5ms each) + expect(elapsed).toBeLessThan(15) + }) + }) + + describe('Real-World Use Cases', () => { + it('should efficiently paginate through high-degree node', async () => { + // Simulate popular entity with 100+ relationships + const hub = await brain.add({ + data: { name: 'Popular Hub' }, + type: NounType.Thing + }) + + // Create 100 relationships + const targetIds: string[] = [] + for (let i = 0; i < 100; i++) { + const targetId = await brain.add({ + data: { name: `Target ${i}` }, + type: NounType.Thing + }) + targetIds.push(targetId) + await brain.relate({ + from: hub, + to: targetId, + type: VerbType.RelatesTo + }) + } + + const graphIndex = (brain as any).graphIndex + + // Paginate in chunks of 25 + const chunks: string[][] = [] + for (let offset = 0; offset < 100; offset += 25) { + const chunk = await graphIndex.getNeighbors(hub, { + direction: 'out', + limit: 25, + offset + }) + chunks.push(chunk) + } + + // Should have 4 chunks + expect(chunks).toHaveLength(4) + + // Each chunk should have 25 items + chunks.forEach(chunk => { + expect(chunk).toHaveLength(25) + }) + + // All chunks combined should equal total + const allNeighbors = chunks.flat() + expect(allNeighbors).toHaveLength(100) + + // No duplicates + const uniqueNeighbors = new Set(allNeighbors) + expect(uniqueNeighbors.size).toBe(100) + }) + }) +})