brainy/tests/opfs-storage.test.ts
David Snelling abc17397b1 feat: Remove dangerous getAllNouns/getAllVerbs methods, add safe pagination
BREAKING CHANGE: Removed getAllNouns() and getAllVerbs() from StorageAdapter interface
These methods could cause expensive full scans on cloud storage (S3/R2) leading to
high costs and performance issues. Replaced with safe paginated methods.

Changes:
- Remove getAllNouns/getAllVerbs from StorageAdapter interface and implementations
- Add internal optimization methods for intelligent preloading when safe
- Fix OPFS storage file naming consistency (.json extension)
- Fix S3 high-volume mode detection thresholds (was too aggressive)
- Fix TypeScript compilation errors with async methods
- Update all tests to use paginated methods

Performance:
- Add smart dataset size detection for automatic optimization
- Maintain all internal performance optimizations through safe preloading
- Only preload data in read-only mode or when dataset is small (<10k entities)

Fixes:
- Fix intelligent verb scoring tests metadata structure
- Fix S3 storage getVerbsBySource/Target/Type methods
- Fix memory usage in search operations using pagination

Docs:
- Add comprehensive storage architecture documentation
- Document known bash redirection issue
- Update README with architecture doc link

All affected tests passing
2025-08-10 16:25:12 -07:00

257 lines
No EOL
8.3 KiB
TypeScript

/**
* OPFS Storage Tests
* Tests for the OPFS storage adapter using a simulated OPFS environment
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { setupOPFSMock, cleanupOPFSMock } from './mocks/opfs-mock'
import { Vector } from '../src/coreTypes'
describe('OPFSStorage', () => {
// Import modules inside tests to avoid issues with dynamic imports
let OPFSStorage: any
let opfsMock: any
beforeEach(async () => {
// Setup OPFS mock environment
opfsMock = setupOPFSMock()
// Import storage factory
const storageFactory = await import('../src/storage/storageFactory.js')
OPFSStorage = storageFactory.OPFSStorage
})
afterEach(() => {
// Clean up OPFS mock environment
cleanupOPFSMock()
// Reset mocks
vi.resetAllMocks()
})
it('should detect OPFS availability correctly', () => {
// Create a new instance with our mocked environment
const opfsStorage = new OPFSStorage()
// With our mocks in place, OPFS should be available
expect(opfsStorage.isOPFSAvailable()).toBe(true)
// Now remove the getDirectory method to simulate OPFS not being available
delete global.navigator.storage.getDirectory
// Create a new instance with the modified environment
const opfsStorage2 = new OPFSStorage()
expect(opfsStorage2.isOPFSAvailable()).toBe(false)
})
it('should initialize and perform basic operations with OPFS storage', async () => {
// Create a new instance with our mocked environment
const opfsStorage = new OPFSStorage()
// Initialize the storage
await opfsStorage.init()
// Test basic metadata operations
const testMetadata = { test: 'data', value: 123 }
await opfsStorage.saveMetadata('test-key', testMetadata)
const retrievedMetadata = await opfsStorage.getMetadata('test-key')
expect(retrievedMetadata).toEqual(testMetadata)
// Clean up
await opfsStorage.clear()
})
it('should handle noun operations correctly', async () => {
// Create a new instance with our mocked environment
const opfsStorage = new OPFSStorage()
// Initialize the storage
await opfsStorage.init()
// Create test noun
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const testNoun = {
id: 'test-noun-1',
vector: testVector,
connections: new Map([
[0, new Set(['test-noun-2', 'test-noun-3'])]
])
}
// Save the noun
await opfsStorage.saveNoun(testNoun)
// Retrieve the noun
const retrievedNoun = await opfsStorage.getNoun('test-noun-1')
// Verify the noun was saved and retrieved correctly
expect(retrievedNoun).toBeDefined()
expect(retrievedNoun?.id).toBe('test-noun-1')
expect(retrievedNoun?.vector).toEqual(testVector)
// Verify connections were saved correctly
// Note: connections are stored as a Map in memory but might be serialized differently
expect(retrievedNoun?.connections).toBeDefined()
expect(retrievedNoun?.connections.get(0)).toBeDefined()
expect(retrievedNoun?.connections.get(0)?.has('test-noun-2')).toBe(true)
expect(retrievedNoun?.connections.get(0)?.has('test-noun-3')).toBe(true)
// Check if the noun is actually stored first
console.log('DEBUG: Checking if noun exists after save')
const storedNoun = await opfsStorage.getNoun('test-noun-1')
console.log('DEBUG: storedNoun:', storedNoun ? 'EXISTS' : 'NOT FOUND')
// Test getNouns with pagination
console.log('DEBUG: About to test getNouns')
const nounsResult = await opfsStorage.getNouns({ pagination: { limit: 10 } })
console.log('DEBUG: getNouns result:', nounsResult.items.length)
expect(nounsResult.items.length).toBe(1)
expect(nounsResult.items[0].id).toBe('test-noun-1')
// Test deleteNoun
await opfsStorage.deleteNoun('test-noun-1')
const deletedNoun = await opfsStorage.getNoun('test-noun-1')
expect(deletedNoun).toBeNull()
// Clean up
await opfsStorage.clear()
})
it('should handle verb operations correctly', async () => {
// Create a new instance with our mocked environment
const opfsStorage = new OPFSStorage()
// Initialize the storage
await opfsStorage.init()
// Create test verb
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const timestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
const testVerb = {
id: 'test-verb-1',
vector: testVector,
connections: new Map(),
source: 'source-noun-1',
target: 'target-noun-1',
verb: 'test-relation',
weight: 0.75,
metadata: { description: 'Test relation' },
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
augmentation: 'test-service',
version: '1.0'
}
}
// Save the verb
await opfsStorage.saveVerb(testVerb)
// Retrieve the verb
const retrievedVerb = await opfsStorage.getVerb('test-verb-1')
// Verify the verb was saved and retrieved correctly
expect(retrievedVerb).toBeDefined()
expect(retrievedVerb?.id).toBe('test-verb-1')
expect(retrievedVerb?.vector).toEqual(testVector)
expect(retrievedVerb?.source).toBe('source-noun-1')
expect(retrievedVerb?.target).toBe('target-noun-1')
expect(retrievedVerb?.verb).toBe('test-relation')
expect(retrievedVerb?.weight).toBe(0.75)
expect(retrievedVerb?.metadata).toEqual({ description: 'Test relation' })
expect(retrievedVerb?.createdAt).toEqual(timestamp)
expect(retrievedVerb?.updatedAt).toEqual(timestamp)
expect(retrievedVerb?.createdBy).toEqual({
augmentation: 'test-service',
version: '1.0'
})
// Test getVerbs with pagination
const verbsResult = await opfsStorage.getVerbs({ pagination: { limit: 10 } })
expect(verbsResult.items.length).toBe(1)
expect(verbsResult.items[0].id).toBe('test-verb-1')
// Test getVerbsBySource
const verbsBySource = await opfsStorage.getVerbsBySource('source-noun-1')
expect(verbsBySource.length).toBe(1)
expect(verbsBySource[0].id).toBe('test-verb-1')
// Test getVerbsByTarget
const verbsByTarget = await opfsStorage.getVerbsByTarget('target-noun-1')
expect(verbsByTarget.length).toBe(1)
expect(verbsByTarget[0].id).toBe('test-verb-1')
// Test getVerbsByType
const verbsByType = await opfsStorage.getVerbsByType('test-relation')
expect(verbsByType.length).toBe(1)
expect(verbsByType[0].id).toBe('test-verb-1')
// Test deleteVerb
await opfsStorage.deleteVerb('test-verb-1')
const deletedVerb = await opfsStorage.getVerb('test-verb-1')
expect(deletedVerb).toBeNull()
// Clean up
await opfsStorage.clear()
})
it('should handle storage status correctly', async () => {
// Create a new instance with our mocked environment
const opfsStorage = new OPFSStorage()
// Initialize the storage
await opfsStorage.init()
// Add some data to the storage
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const testNoun = {
id: 'test-noun-1',
vector: testVector,
connections: new Map([
[0, new Set(['test-noun-2', 'test-noun-3'])]
])
}
await opfsStorage.saveNoun(testNoun)
await opfsStorage.saveMetadata('test-key', { test: 'data', value: 123 })
// Get storage status
const status = await opfsStorage.getStorageStatus()
// Verify status
expect(status.type).toBe('opfs')
expect(status.used).toBeGreaterThan(0)
expect(status.quota).toBeGreaterThan(0)
// Clean up
await opfsStorage.clear()
})
it('should handle persistence correctly', async () => {
// Create a new instance with our mocked environment
const opfsStorage = new OPFSStorage()
// Initialize the storage
await opfsStorage.init()
// Test persistence methods
const isPersisted = await opfsStorage.isPersistent()
expect(isPersisted).toBe(true)
// Get the current persistence state
const initialPersistence = await opfsStorage.isPersistent()
expect(initialPersistence).toBe(true)
// Request persistence (should return true with our mock)
const persistResult = await opfsStorage.requestPersistentStorage()
expect(persistResult).toBe(true)
// Clean up
await opfsStorage.clear()
})
})