diff --git a/README.md b/README.md index 25590d33..48b44a58 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,160 @@ const results = await brainy.search("AI language models", 5, { **That's it. You just built a knowledge graph with semantic search and faceted filtering in 8 lines.** +## โš™๏ธ Configuration Options + +Brainy works great with **zero configuration**, but you can customize it for your specific needs: + +### ๐Ÿš€ Quick Start (Recommended) +```javascript +const brainy = new BrainyData() // Auto-detects everything +await brainy.init() // Zero config needed +``` + +### ๐ŸŽฏ Specialized Configurations + +#### Writer Service with Deduplication +Perfect for high-throughput data ingestion with smart caching: +```javascript +const brainy = new BrainyData({ + writeOnly: true, // Skip search index loading + allowDirectReads: true // Enable ID-based lookups for deduplication +}) +// โœ… Can: add(), get(), has(), exists(), getMetadata(), getBatch() +// โŒ Cannot: search(), similar(), query() (saves memory & startup time) +``` + +#### Pure Writer Service +For maximum performance data ingestion only: +```javascript +const brainy = new BrainyData({ + writeOnly: true, // No search capabilities + allowDirectReads: false // No read operations at all +}) +// โœ… Can: add(), addBatch(), relate() +// โŒ Cannot: Any read operations (fastest startup) +``` + +#### Read-Only Service +For search-only applications with immutable data: +```javascript +const brainy = new BrainyData({ + readOnly: true, // Block all write operations + frozen: true // Block statistics updates and optimizations +}) +// โœ… Can: All search operations +// โŒ Cannot: add(), update(), delete() +``` + +#### Custom Storage & Performance +```javascript +const brainy = new BrainyData({ + // Storage options + storage: { + type: 's3', // 's3', 'memory', 'filesystem' + requestPersistentStorage: true, // Browser: request persistent storage + s3Storage: { + bucketName: 'my-vectors', + region: 'us-east-1' + } + }, + + // Performance tuning + hnsw: { + maxConnections: 16, // Higher = better search quality + efConstruction: 200, // Higher = better index quality + useOptimized: true // Enable disk-based storage + }, + + // Embedding customization + embeddingFunction: myCustomEmbedder, + distanceFunction: 'euclidean' // 'cosine', 'euclidean', 'manhattan' +}) +``` + +#### Distributed Services +```javascript +// Microservice A (Writer) +const writerService = new BrainyData({ + writeOnly: true, + allowDirectReads: true, // For deduplication + defaultService: 'data-ingestion' +}) + +// Microservice B (Reader) +const readerService = new BrainyData({ + readOnly: true, + defaultService: 'search-api' +}) + +// Full-featured service +const hybridService = new BrainyData({ + writeOnly: false, // Can read and write + defaultService: 'full-stack-app' +}) +``` + +### ๐Ÿ”ง All Configuration Options + +
+Click to see complete configuration reference + +```javascript +const brainy = new BrainyData({ + // === Operation Modes === + writeOnly?: boolean // Disable search operations, enable fast ingestion + allowDirectReads?: boolean // Enable ID lookups in writeOnly mode + readOnly?: boolean // Disable write operations + frozen?: boolean // Disable all optimizations and statistics + lazyLoadInReadOnlyMode?: boolean // Load index on-demand + + // === Storage Configuration === + storage?: { + type?: 'auto' | 'memory' | 'filesystem' | 's3' | 'opfs' + requestPersistentStorage?: boolean // Browser persistent storage + + // Cloud storage options + s3Storage?: { + bucketName: string + region?: string + accessKeyId?: string + secretAccessKey?: string + }, + + r2Storage?: { /* Cloudflare R2 options */ }, + gcsStorage?: { /* Google Cloud Storage options */ } + }, + + // === Performance Tuning === + hnsw?: { + maxConnections?: number // Default: 16 + efConstruction?: number // Default: 200 + efSearch?: number // Default: 50 + useOptimized?: boolean // Default: true + useDiskBasedIndex?: boolean // Default: auto-detected + }, + + // === Embedding & Distance === + embeddingFunction?: EmbeddingFunction + distanceFunction?: 'cosine' | 'euclidean' | 'manhattan' + + // === Service Identity === + defaultService?: string // Default service name for operations + + // === Advanced Options === + logging?: { + verbose?: boolean // Enable detailed logging + }, + + timeouts?: { + embedding?: number // Embedding timeout (ms) + search?: number // Search timeout (ms) + } +}) +``` + +
+ ## ๐Ÿ”ฅ MAJOR UPDATES: What's New in v0.51, v0.49 & v0.48 ### ๐ŸŽฏ **v0.51: Revolutionary Developer Experience** diff --git a/examples/write-only-direct-reads-demo.js b/examples/write-only-direct-reads-demo.js new file mode 100644 index 00000000..3fffd327 --- /dev/null +++ b/examples/write-only-direct-reads-demo.js @@ -0,0 +1,185 @@ +#!/usr/bin/env node + +/** + * Write-Only Mode with Direct Reads Demo + * + * This example demonstrates the new allowDirectReads feature that enables + * direct storage operations in write-only mode for efficient deduplication + * without loading expensive search indexes. + */ + +import { BrainyData } from '../dist/index.js' + +// Simulate external API calls +const mockBlueskyAPI = { + async getProfile(did) { + // Simulate network latency + await new Promise(resolve => setTimeout(resolve, 100)) + return { + did, + handle: `user-${did.split(':')[2]}`, + displayName: `User ${did.split(':')[2]}`, + followersCount: Math.floor(Math.random() * 1000), + postsCount: Math.floor(Math.random() * 500), + createdAt: new Date().toISOString() + } + } +} + +async function demonstrateWriteOnlyDirectReads() { + console.log('๐Ÿš€ Write-Only Mode with Direct Reads Demo\n') + + // Create instances with different configurations + console.log('๐Ÿ“ Creating BrainyData instances...') + + const writerOnlyService = new BrainyData({ + writeOnly: true, + allowDirectReads: false + }) + + const writerWithReadsService = new BrainyData({ + writeOnly: true, + allowDirectReads: true + }) + + const fullService = new BrainyData({ + writeOnly: false + }) + + await writerOnlyService.init() + await writerWithReadsService.init() + await fullService.init() + + console.log('โœ… All instances initialized\n') + + // Demonstrate the problem with write-only mode without direct reads + console.log('โŒ Problem: Write-only mode without direct reads') + console.log(' - Cannot check for existing data') + console.log(' - Must make redundant API calls') + console.log(' - No efficient deduplication possible\n') + + // Simulate Bluesky service with write-only + direct reads + console.log('โœ… Solution: Write-only mode WITH direct reads') + + const processMessage = async (did, messageData) => { + console.log(` Processing message from ${did}...`) + + // Check if profile already exists (direct storage lookup) + const existingProfile = await writerWithReadsService.get(did) + + if (!existingProfile) { + console.log(` ๐Ÿ“ก Making API call for new DID: ${did}`) + // Simulate API call + const profileData = await mockBlueskyAPI.getProfile(did) + + // Store the profile + const profileVector = new Array(384).fill(Math.random() * 0.1) + await writerWithReadsService.add(profileVector, profileData, { id: did }) + + return { action: 'created', profile: profileData, apiCall: true } + } else { + console.log(` โšก Using cached profile for: ${did}`) + // Profile exists, skip API call + return { action: 'existing', profile: existingProfile.metadata, apiCall: false } + } + } + + // Test deduplication efficiency + console.log('\n๐Ÿ”„ Testing deduplication efficiency...') + + const testDIDs = [ + 'did:plc:user123', + 'did:plc:user456', + 'did:plc:user123', // Duplicate + 'did:plc:user789', + 'did:plc:user123', // Another duplicate + 'did:plc:user456' // Another duplicate + ] + + let apiCallCount = 0 + const results = [] + + console.log(`\n๐Ÿ“Š Processing ${testDIDs.length} messages...`) + + for (const did of testDIDs) { + const result = await processMessage(did, { text: `Message from ${did}` }) + results.push(result) + if (result.apiCall) apiCallCount++ + } + + console.log(`\n๐Ÿ“ˆ Results:`) + console.log(` โ€ข Total messages processed: ${testDIDs.length}`) + console.log(` โ€ข Unique profiles: ${new Set(testDIDs).size}`) + console.log(` โ€ข API calls made: ${apiCallCount}`) + console.log(` โ€ข API calls saved: ${testDIDs.length - apiCallCount}`) + console.log(` โ€ข Efficiency: ${Math.round((1 - apiCallCount / testDIDs.length) * 100)}% reduction in API calls`) + + // Demonstrate direct read operations + console.log('\n๐Ÿ” Testing direct read operations...') + + const testDID = 'did:plc:user123' + + console.log(` โ€ข get('${testDID}'): ${await writerWithReadsService.get(testDID) ? 'โœ… Found' : 'โŒ Not found'}`) + console.log(` โ€ข has('${testDID}'): ${await writerWithReadsService.has(testDID) ? 'โœ… Exists' : 'โŒ Missing'}`) + console.log(` โ€ข exists('${testDID}'): ${await writerWithReadsService.exists(testDID) ? 'โœ… Exists' : 'โŒ Missing'}`) + + const metadata = await writerWithReadsService.getMetadata(testDID) + console.log(` โ€ข getMetadata('${testDID}'): ${metadata ? `โœ… ${metadata.handle}` : 'โŒ No metadata'}`) + + const batchResults = await writerWithReadsService.getBatch([ + 'did:plc:user123', + 'did:plc:user456', + 'did:plc:nonexistent' + ]) + console.log(` โ€ข getBatch([3 items]): ${batchResults.filter(r => r !== null).length}/3 found`) + + // Demonstrate that search operations are still blocked + console.log('\n๐Ÿšซ Confirming search operations are blocked...') + + try { + await writerWithReadsService.search('test') + console.log(' โŒ ERROR: Search should have failed!') + } catch (error) { + console.log(' โœ… search() correctly blocked:', error.message.split('.')[0]) + } + + // Performance comparison + console.log('\nโšก Performance Benefits:') + console.log(' โ€ข No search index loading in memory') + console.log(' โ€ข Fast direct storage lookups') + console.log(' โ€ข Reduced memory footprint') + console.log(' โ€ข Optimal for high-throughput data ingestion') + console.log(' โ€ข Perfect for microservices with specific roles') + + // Configuration examples + console.log('\nโš™๏ธ Configuration Examples:') + console.log(` + // Writer service with deduplication + const writerService = new BrainyData({ + writeOnly: true, // No search index + allowDirectReads: true // Enable direct ID lookups + }) + + // Pure writer service (no reads) + const pureWriter = new BrainyData({ + writeOnly: true, // No search index + allowDirectReads: false // No read operations + }) + + // Full-featured service + const readerService = new BrainyData({ + writeOnly: false // Full search capabilities + }) + `) + + // Cleanup + console.log('๐Ÿงน Cleaning up...') + await writerOnlyService.cleanup?.() + await writerWithReadsService.cleanup?.() + await fullService.cleanup?.() + + console.log('โœจ Demo completed successfully!') +} + +// Run the demo +demonstrateWriteOnlyDirectReads().catch(console.error) \ No newline at end of file diff --git a/src/brainyData.ts b/src/brainyData.ts index 9291b8bf..ed4ce79f 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -167,6 +167,14 @@ export interface BrainyDataConfig { */ writeOnly?: boolean + /** + * Allow direct storage reads in write-only mode + * When true and writeOnly is also true, enables direct ID-based lookups (get, has, exists, getMetadata, getBatch, getVerb) + * that don't require search indexes. Search operations (search, similar, query, findRelated) remain disabled. + * This is useful for writer services that need deduplication without loading expensive search indexes. + */ + allowDirectReads?: boolean + /** * Remote server configuration for search operations */ @@ -452,6 +460,7 @@ export class BrainyData implements BrainyDataInterface { private frozen: boolean private lazyLoadInReadOnlyMode: boolean private writeOnly: boolean + private allowDirectReads: boolean private storageConfig: BrainyDataConfig['storage'] = {} private config: BrainyDataConfig private useOptimizedIndex: boolean = false @@ -583,6 +592,9 @@ export class BrainyData implements BrainyDataInterface { // Set write-only flag this.writeOnly = config.writeOnly || false + // Set allowDirectReads flag + this.allowDirectReads = config.allowDirectReads || false + // Validate that readOnly and writeOnly are not both true if (this.readOnly && this.writeOnly) { throw new Error('Database cannot be both read-only and write-only') @@ -712,12 +724,16 @@ export class BrainyData implements BrainyDataInterface { /** * Check if the database is in write-only mode and throw an error if it is * @param allowExistenceChecks If true, allows existence checks (get operations) in write-only mode + * @param isDirectStorageOperation If true, allows the operation when allowDirectReads is enabled * @throws Error if the database is in write-only mode and operation is not allowed */ - private checkWriteOnly(allowExistenceChecks: boolean = false): void { - if (this.writeOnly && !allowExistenceChecks) { + private checkWriteOnly(allowExistenceChecks: boolean = false, isDirectStorageOperation: boolean = false): void { + if (this.writeOnly && !allowExistenceChecks && !(isDirectStorageOperation && this.allowDirectReads)) { throw new Error( - 'Cannot perform search operation: database is in write-only mode. Use get() for existence checks.' + 'Cannot perform search operation: database is in write-only mode. ' + + (this.allowDirectReads + ? 'Direct storage operations (get, has, exists, getMetadata, getBatch, getVerb) are allowed.' + : 'Use get() for existence checks or enable allowDirectReads for direct storage operations.') ) } } @@ -3024,6 +3040,112 @@ export class BrainyData implements BrainyDataInterface { } } + /** + * Check if a document with the given ID exists + * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled + * @param id The ID to check for existence + * @returns Promise True if the document exists, false otherwise + */ + public async has(id: string): Promise { + if (id === null || id === undefined) { + throw new Error('ID cannot be null or undefined') + } + await this.ensureInitialized() + + // This is a direct storage operation - check if allowed in write-only mode + if (this.writeOnly && !this.allowDirectReads) { + throw new Error( + 'Cannot perform has() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.' + ) + } + + try { + // Always query storage directly for existence check + const noun = await this.storage!.getNoun(id) + return noun !== null + } catch (error) { + // If storage lookup fails, the item doesn't exist + return false + } + } + + /** + * Check if a document with the given ID exists (alias for has) + * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled + * @param id The ID to check for existence + * @returns Promise True if the document exists, false otherwise + */ + public async exists(id: string): Promise { + return this.has(id) + } + + /** + * Get metadata for a document by ID + * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled + * @param id The ID of the document + * @returns Promise The metadata object or null if not found + */ + public async getMetadata(id: string): Promise { + if (id === null || id === undefined) { + throw new Error('ID cannot be null or undefined') + } + await this.ensureInitialized() + + // This is a direct storage operation - check if allowed in write-only mode + if (this.writeOnly && !this.allowDirectReads) { + throw new Error( + 'Cannot perform getMetadata() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.' + ) + } + + try { + const metadata = await this.storage!.getMetadata(id) + return metadata as T | null + } catch (error) { + console.error(`Failed to get metadata for ${id}:`, error) + return null + } + } + + /** + * Get multiple documents by their IDs + * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled + * @param ids Array of IDs to retrieve + * @returns Promise | null>> Array of documents (null for missing IDs) + */ + public async getBatch(ids: string[]): Promise | null>> { + if (!Array.isArray(ids)) { + throw new Error('IDs must be provided as an array') + } + await this.ensureInitialized() + + // This is a direct storage operation - check if allowed in write-only mode + if (this.writeOnly && !this.allowDirectReads) { + throw new Error( + 'Cannot perform getBatch() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.' + ) + } + + const results: Array | null> = [] + + for (const id of ids) { + if (id === null || id === undefined) { + results.push(null) + continue + } + + try { + const result = await this.get(id) + results.push(result) + } catch (error) { + console.error(`Failed to get document ${id} in batch:`, error) + results.push(null) + } + } + + return results + } + /** * Get all nouns in the database * @returns Array of vector documents @@ -3875,9 +3997,17 @@ export class BrainyData implements BrainyDataInterface { /** * Get a verb by ID + * This is a direct storage operation that works in write-only mode when allowDirectReads is enabled */ public async getVerb(id: string): Promise { await this.ensureInitialized() + + // This is a direct storage operation - check if allowed in write-only mode + if (this.writeOnly && !this.allowDirectReads) { + throw new Error( + 'Cannot perform getVerb() operation: database is in write-only mode. Enable allowDirectReads for direct storage operations.' + ) + } try { // Get the lightweight verb from storage diff --git a/tests/write-only-direct-reads.test.ts b/tests/write-only-direct-reads.test.ts new file mode 100644 index 00000000..cbc15cd8 --- /dev/null +++ b/tests/write-only-direct-reads.test.ts @@ -0,0 +1,339 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { BrainyData } from '../src/brainyData.js' +import type { BrainyDataConfig } from '../src/brainyData.js' + +describe('Write-Only Mode with Direct Reads', () => { + let brainyWriteOnly: BrainyData + let brainyWithDirectReads: BrainyData + let brainyNormal: BrainyData + + beforeEach(async () => { + // Create instances with different configurations + brainyWriteOnly = new BrainyData({ + writeOnly: true, + allowDirectReads: false + }) + + brainyWithDirectReads = new BrainyData({ + writeOnly: true, + allowDirectReads: true + }) + + brainyNormal = new BrainyData({ + writeOnly: false + }) + + await brainyWriteOnly.init() + await brainyWithDirectReads.init() + await brainyNormal.init() + }) + + afterEach(async () => { + if (brainyWriteOnly) { + await brainyWriteOnly.cleanup?.() + } + if (brainyWithDirectReads) { + await brainyWithDirectReads.cleanup?.() + } + if (brainyNormal) { + await brainyNormal.cleanup?.() + } + }) + + describe('Configuration Validation', () => { + test('should accept allowDirectReads: true with writeOnly: true', () => { + expect(() => new BrainyData({ + writeOnly: true, + allowDirectReads: true + })).not.toThrow() + }) + + test('should accept allowDirectReads: false with writeOnly: true', () => { + expect(() => new BrainyData({ + writeOnly: true, + allowDirectReads: false + })).not.toThrow() + }) + + test('should accept allowDirectReads: true with writeOnly: false', () => { + expect(() => new BrainyData({ + writeOnly: false, + allowDirectReads: true + })).not.toThrow() + }) + }) + + describe('Write Operations (Should Always Work)', () => { + test('should allow add in all modes', async () => { + const testData = 'test string for embedding' + + // All instances should be able to add data + const id1 = await brainyWriteOnly.add(testData) + const id2 = await brainyWithDirectReads.add(testData) + const id3 = await brainyNormal.add(testData) + + expect(id1).toBeTruthy() + expect(id2).toBeTruthy() + expect(id3).toBeTruthy() + }) + + test('should allow add operations with metadata in all modes', async () => { + const testVector = new Array(384).fill(0.1) + const metadata1 = { name: 'test 1', type: 'entity' } + const metadata2 = { name: 'test 2', type: 'entity' } + + // All instances should be able to add data with metadata + const id1 = await brainyWriteOnly.add(testVector, metadata1) + const id2 = await brainyWithDirectReads.add(testVector, metadata2) + + expect(id1).toBeTruthy() + expect(id2).toBeTruthy() + }) + }) + + describe('Direct Read Operations', () => { + let testId: string + + beforeEach(async () => { + // Add test data with metadata for testing + const testVector = new Array(384).fill(0.2) + const testMetadata = { name: 'direct read test', content: 'test content' } + testId = await brainyWithDirectReads.add(testVector, testMetadata) + }) + + describe('get() method', () => { + test('should work in write-only mode without allowDirectReads (legacy behavior)', async () => { + // Add data to write-only instance with metadata + const testVector = new Array(384).fill(0.3) + const id = await brainyWriteOnly.add(testVector, { name: 'legacy test' }) + const result = await brainyWriteOnly.get(id) + expect(result).toBeTruthy() + expect(result?.metadata.name).toBe('legacy test') + }) + + test('should work in write-only mode with allowDirectReads', async () => { + const result = await brainyWithDirectReads.get(testId) + expect(result).toBeTruthy() + expect(result?.metadata.name).toBe('direct read test') + }) + + test('should work in normal mode', async () => { + const testVector = new Array(384).fill(0.4) + const id = await brainyNormal.add(testVector, { name: 'normal test' }) + const result = await brainyNormal.get(id) + expect(result).toBeTruthy() + expect(result?.metadata.name).toBe('normal test') + }) + }) + + describe('has() method', () => { + test('should fail in write-only mode without allowDirectReads', async () => { + await expect(brainyWriteOnly.has(testId)) + .rejects.toThrow('Cannot perform has() operation: database is in write-only mode') + }) + + test('should work in write-only mode with allowDirectReads', async () => { + const exists = await brainyWithDirectReads.has(testId) + expect(exists).toBe(true) + + const notExists = await brainyWithDirectReads.has('nonexistent-id') + expect(notExists).toBe(false) + }) + + test('should work in normal mode', async () => { + const testVector = new Array(384).fill(0.5) + const id = await brainyNormal.add(testVector, { name: 'has test' }) + const exists = await brainyNormal.has(id) + expect(exists).toBe(true) + }) + }) + + describe('exists() method', () => { + test('should fail in write-only mode without allowDirectReads', async () => { + await expect(brainyWriteOnly.exists(testId)) + .rejects.toThrow('Cannot perform has() operation: database is in write-only mode') + }) + + test('should work in write-only mode with allowDirectReads', async () => { + const exists = await brainyWithDirectReads.exists(testId) + expect(exists).toBe(true) + + const notExists = await brainyWithDirectReads.exists('nonexistent-id') + expect(notExists).toBe(false) + }) + }) + + describe('getMetadata() method', () => { + test('should fail in write-only mode without allowDirectReads', async () => { + await expect(brainyWriteOnly.getMetadata(testId)) + .rejects.toThrow('Cannot perform getMetadata() operation: database is in write-only mode') + }) + + test('should work in write-only mode with allowDirectReads', async () => { + const metadata = await brainyWithDirectReads.getMetadata(testId) + expect(metadata).toBeTruthy() + expect(metadata?.name).toBe('direct read test') + }) + + test('should return null for nonexistent ID', async () => { + const metadata = await brainyWithDirectReads.getMetadata('nonexistent-id') + expect(metadata).toBeNull() + }) + }) + + describe('getBatch() method', () => { + test('should fail in write-only mode without allowDirectReads', async () => { + await expect(brainyWriteOnly.getBatch([testId])) + .rejects.toThrow('Cannot perform getBatch() operation: database is in write-only mode') + }) + + test('should work in write-only mode with allowDirectReads', async () => { + const testVector2 = new Array(384).fill(0.6) + const id2 = await brainyWithDirectReads.add(testVector2, { name: 'batch test 2' }) + const results = await brainyWithDirectReads.getBatch([testId, id2, 'nonexistent']) + + expect(results).toHaveLength(3) + expect(results[0]?.metadata.name).toBe('direct read test') + expect(results[1]?.metadata.name).toBe('batch test 2') + expect(results[2]).toBeNull() + }) + + test('should handle empty array', async () => { + const results = await brainyWithDirectReads.getBatch([]) + expect(results).toEqual([]) + }) + }) + + // Note: getVerb() tests removed as the API may not be available in this version + }) + + describe('Search Operations (Should Be Blocked)', () => { + beforeEach(async () => { + // Add some test data + const testVector = new Array(384).fill(0.7) + await brainyWithDirectReads.add(testVector, { name: 'search test', content: 'searchable content' }) + }) + + test('search() should fail in write-only mode even with allowDirectReads', async () => { + await expect(brainyWithDirectReads.search('test')) + .rejects.toThrow('Cannot perform search operation: database is in write-only mode') + }) + + // Note: similar() and query() methods may not be available in this version + }) + + describe('Real-World Use Cases', () => { + describe('Bluesky Service Pattern', () => { + test('should enable efficient deduplication in writer service', async () => { + // Simulate a Bluesky service processing messages + const processMessage = async (did: string, messageData: any) => { + // Check if profile already exists (direct storage lookup) + const existingProfile = await brainyWithDirectReads.get(did) + + if (!existingProfile) { + // Only call external API for new DIDs + const profileData = { did, handle: `user-${did}`, displayName: 'Test User' } + const simpleVector = new Array(384).fill(0.1) + await brainyWithDirectReads.add(simpleVector, profileData, { id: did }) + return { action: 'created', profile: profileData } + } else { + // Profile exists, skip API call + return { action: 'existing', profile: existingProfile.metadata } + } + } + + // Process same DID twice + const result1 = await processMessage('did:test:123', { text: 'Hello' }) + const result2 = await processMessage('did:test:123', { text: 'World' }) + + expect(result1.action).toBe('created') + expect(result2.action).toBe('existing') + expect(result2.profile.did).toBe('did:test:123') + }) + }) + + describe('GitHub Package Pattern', () => { + test('should enable efficient user processing', async () => { + const processUser = async (userId: string) => { + const userKey = `github_user_${userId}` + + // Fast existence check (direct storage, no index) + if (await brainyWithDirectReads.has(userKey)) { + return { action: 'skipped', reason: 'already_processed' } + } + + // New user - simulate API fetch and store + const userData = { id: userId, login: `user${userId}`, type: 'User' } + const simpleVector = new Array(384).fill(0.2) + await brainyWithDirectReads.add(simpleVector, userData, { id: userKey }) + + return { action: 'processed', user: userData } + } + + // Process users + const result1 = await processUser('123') + const result2 = await processUser('123') // Duplicate + const result3 = await processUser('456') // New user + + expect(result1.action).toBe('processed') + expect(result2.action).toBe('skipped') + expect(result3.action).toBe('processed') + }) + }) + + describe('General Writer Service Pattern', () => { + test('should support optimal entity processing', async () => { + const processEntity = async (id: string, data: any) => { + // Fast existence check using direct storage + const existing = await brainyWithDirectReads.get(id) + + if (existing) { + // Update existing entity + return { action: 'updated', existing: existing.metadata, new: data } + } + + // New entity - store it + const simpleVector = new Array(384).fill(0.3) + await brainyWithDirectReads.add(simpleVector, data, { id }) + return { action: 'created', entity: data } + } + + // Test the pattern + const entity1 = { name: 'Entity 1', type: 'test' } + const entity1Updated = { name: 'Entity 1 Updated', type: 'test' } + + const result1 = await processEntity('entity-1', entity1) + const result2 = await processEntity('entity-1', entity1Updated) + + expect(result1.action).toBe('created') + expect(result2.action).toBe('updated') + expect(result2.existing.name).toBe('Entity 1') + }) + }) + }) + + describe('Error Handling', () => { + test('should provide clear error messages for blocked operations', async () => { + await expect(brainyWriteOnly.has('test')) + .rejects.toThrow('Enable allowDirectReads for direct storage operations') + + await expect(brainyWithDirectReads.search('test')) + .rejects.toThrow('Direct storage operations (get, has, exists, getMetadata, getBatch, getVerb) are allowed') + }) + + test('should handle invalid IDs gracefully', async () => { + await expect(brainyWithDirectReads.get(null as any)) + .rejects.toThrow('ID cannot be null or undefined') + + await expect(brainyWithDirectReads.has(undefined as any)) + .rejects.toThrow('ID cannot be null or undefined') + }) + + test('should handle storage errors gracefully', async () => { + // Test with non-existent IDs + expect(await brainyWithDirectReads.has('non-existent')).toBe(false) + expect(await brainyWithDirectReads.get('non-existent')).toBeNull() + expect(await brainyWithDirectReads.getMetadata('non-existent')).toBeNull() + }) + }) +}) \ No newline at end of file