/** * Comprehensive tests for BlobStorage (src/storage/blobStorage.ts) * * Tests: * - Content-addressable storage (SHA-256, integrity verification) * - Deduplication via reference counting * - Compression (zstd, MIME-aware auto mode) * - LRU caching (bounded cache still serves correct bytes) * - Reference counting + delete-at-zero * - Error handling * - Performance characteristics * - Wrapped-binary-data regressions (key-based dispatch) * * Cache-bypass pattern: the store has no cache-introspection API (the LRU is * an internal optimization), so tests that must force a storage read create a * FRESH BlobStorage over the same adapter — a cold cache by construction. */ import { describe, it, expect, beforeEach } from 'vitest' import { BlobStorage, BlobStoreAdapter } from '../../../src/storage/blobStorage.js' /** * Simple in-memory blob store adapter for testing */ class InMemoryBlobAdapter implements BlobStoreAdapter { private store = new Map() async get(key: string): Promise { return this.store.get(key) } async put(key: string, data: Buffer): Promise { this.store.set(key, data) } async delete(key: string): Promise { this.store.delete(key) } async list(prefix: string): Promise { const keys: string[] = [] for (const key of this.store.keys()) { if (key.startsWith(prefix)) { keys.push(key) } } return keys.sort() } } describe('BlobStorage', () => { let adapter: BlobStoreAdapter let blobStorage: BlobStorage beforeEach(() => { adapter = new InMemoryBlobAdapter() blobStorage = new BlobStorage(adapter) }) describe('Content-Addressable Storage', () => { it('should compute SHA-256 hash of data', () => { const data = Buffer.from('hello world') const hash = BlobStorage.hash(data) expect(hash).toBe('b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9') expect(hash).toHaveLength(64) }) it('should write blob and return hash', async () => { const data = Buffer.from('test data') const hash = await blobStorage.write(data) expect(hash).toBeTruthy() expect(hash).toHaveLength(64) }) it('should read blob by hash', async () => { const data = Buffer.from('test data') const hash = await blobStorage.write(data) const retrieved = await blobStorage.read(hash) expect(retrieved.toString()).toBe('test data') }) it('should verify data integrity on read', async () => { const data = Buffer.from('test data') const hash = await blobStorage.write(data) // Corrupt the blob data behind the store's back await adapter.put(`blob:${hash}`, Buffer.from('corrupted')) // A fresh instance has a cold cache, so read() must hit storage and // detect the corruption via hash verification. const coldStore = new BlobStorage(adapter) await expect(coldStore.read(hash)).rejects.toThrow('integrity check failed') }) it('should check if blob exists', async () => { const data = Buffer.from('test data') const hash = await blobStorage.write(data) expect(await blobStorage.has(hash)).toBe(true) expect(await blobStorage.has('0'.repeat(64))).toBe(false) }) }) describe('Deduplication', () => { it('should deduplicate identical blobs', async () => { const data = Buffer.from('duplicate data') const hash1 = await blobStorage.write(data) const hash2 = await blobStorage.write(data) expect(hash1).toBe(hash2) // Exactly one stored blob + one metadata record expect(await adapter.list('blob:')).toHaveLength(1) expect(await adapter.list('blob-meta:')).toHaveLength(1) }) it('should increment ref count on duplicate write', async () => { const data = Buffer.from('test data') await blobStorage.write(data) const hash = await blobStorage.write(data) const metadata = await blobStorage.getMetadata(hash) expect(metadata?.refCount).toBe(2) }) it('should track every duplicate write in the reference count', async () => { const data = Buffer.from('x'.repeat(1000)) const hash1 = await blobStorage.write(data) const hash2 = await blobStorage.write(data) const hash3 = await blobStorage.write(data) expect(hash2).toBe(hash1) expect(hash3).toBe(hash1) const metadata = await blobStorage.getMetadata(hash1) expect(metadata?.refCount).toBe(3) }) }) describe('Compression', () => { it('should compress large text data with zstd', async () => { const data = Buffer.from('a'.repeat(10000)) const hash = await blobStorage.write(data, { compression: 'zstd' }) const metadata = await blobStorage.getMetadata(hash) // zstd may not be available in test environment - falls back to 'none' // This is expected behavior (see BlobStorage.ensureCompressionReady fallback) if (metadata?.compression === 'zstd') { expect(metadata.compressedSize).toBeLessThan(metadata.size) } else { // Fallback to 'none' is acceptable when zstd unavailable expect(metadata?.compression).toBe('none') } }) it('should decompress zstd data on read', async () => { const originalData = Buffer.from('test data '.repeat(100)) const hash = await blobStorage.write(originalData, { compression: 'zstd' }) const retrieved = await blobStorage.read(hash) expect(retrieved.toString()).toBe(originalData.toString()) }) it('should not compress small blobs', async () => { const data = Buffer.from('small') const hash = await blobStorage.write(data, { compression: 'auto' }) const metadata = await blobStorage.getMetadata(hash) expect(metadata?.compression).toBe('none') }) it('should skip compression for already-compressed MIME types in auto mode', async () => { const data = Buffer.from('x'.repeat(5000)) const hash = await blobStorage.write(data, { compression: 'auto', mimeType: 'image/jpeg' }) const metadata = await blobStorage.getMetadata(hash) expect(metadata?.compression).toBe('none') }) it('should auto-compress large compressible payloads', async () => { const data = Buffer.from('x'.repeat(5000)) const hash = await blobStorage.write(data, { compression: 'auto' }) const metadata = await blobStorage.getMetadata(hash) // Should compress if zstd is available if (metadata?.compression === 'zstd') { expect(metadata.compressedSize).toBeLessThan(metadata.size) } }) }) describe('LRU Caching', () => { it('should serve identical bytes from cache and from storage', async () => { const data = Buffer.from('cached data') const hash = await blobStorage.write(data) // Warm read (write-through cache) and cold read (fresh instance) const warm = await blobStorage.read(hash) const cold = await new BlobStorage(adapter).read(hash) expect(warm.equals(data)).toBe(true) expect(cold.equals(data)).toBe(true) }) it('should keep serving correct bytes when the cache evicts under pressure', async () => { const smallCache = new BlobStorage(adapter, { cacheMaxSize: 100 }) // Write blobs that exceed cache size (forces LRU eviction) const blobs = [ Buffer.from('x'.repeat(50)), Buffer.from('y'.repeat(50)), Buffer.from('z'.repeat(50)) ] const hashes: string[] = [] for (const blob of blobs) { hashes.push(await smallCache.write(blob)) } // Every blob still reads back correctly, evicted or not for (let i = 0; i < blobs.length; i++) { const retrieved = await smallCache.read(hashes[i]) expect(retrieved.equals(blobs[i])).toBe(true) } }) }) describe('Reference Counting', () => { it('should track reference count', async () => { const data = Buffer.from('test') const hash = await blobStorage.write(data) let metadata = await blobStorage.getMetadata(hash) expect(metadata?.refCount).toBe(1) // Write duplicate (increments ref count) await blobStorage.write(data) metadata = await blobStorage.getMetadata(hash) expect(metadata?.refCount).toBe(2) }) it('should only delete when refCount reaches 0', async () => { const data = Buffer.from('test') // Write twice (refCount = 2) const hash = await blobStorage.write(data) await blobStorage.write(data) // Delete once (refCount = 1, blob still exists) await blobStorage.delete(hash) expect(await blobStorage.has(hash)).toBe(true) // Delete again (refCount = 0, blob deleted) await blobStorage.delete(hash) expect(await blobStorage.has(hash)).toBe(false) }) }) describe('Metadata', () => { it('should store blob metadata', async () => { const data = Buffer.from('test') const hash = await blobStorage.write(data, { compression: 'none' }) const metadata = await blobStorage.getMetadata(hash) expect(metadata?.hash).toBe(hash) expect(metadata?.size).toBe(data.length) expect(metadata?.compression).toBe('none') expect(metadata?.createdAt).toBeGreaterThan(0) expect(metadata?.refCount).toBe(1) }) it('should return undefined metadata for unknown hashes', async () => { expect(await blobStorage.getMetadata('f'.repeat(64))).toBeUndefined() }) }) describe('Error Handling', () => { it('should throw on reading non-existent blob', async () => { await expect( blobStorage.read('f'.repeat(64)) ).rejects.toThrow('Blob metadata not found') }) it('should throw on reading blob with missing metadata', async () => { const data = Buffer.from('test') const hash = await blobStorage.write(data) // Delete metadata but keep blob bytes await adapter.delete(`blob-meta:${hash}`) // A fresh instance has a cold cache, so read() must consult metadata const coldStore = new BlobStorage(adapter) await expect(coldStore.read(hash)).rejects.toThrow('metadata not found') }) }) describe('Performance', () => { it('should write 1000 small blobs quickly', async () => { const start = Date.now() for (let i = 0; i < 1000; i++) { await blobStorage.write(Buffer.from(`blob${i}`)) } const elapsed = Date.now() - start expect(elapsed).toBeLessThan(5000) // Should complete in < 5s }) it('should read 1000 cached blobs very quickly', async () => { // Write and cache blobs const hashes: string[] = [] for (let i = 0; i < 1000; i++) { const hash = await blobStorage.write(Buffer.from(`blob${i}`)) hashes.push(hash) } // First read (populate cache) for (const hash of hashes) { await blobStorage.read(hash) } // Second read (from cache) const start = Date.now() for (const hash of hashes) { await blobStorage.read(hash) } const elapsed = Date.now() - start expect(elapsed).toBeLessThan(1000) // Should be very fast from cache }) }) describe('Wrapped-binary regression (v5.10.0) - hash verification on unwrapped bytes', () => { it('should handle wrapped binary data without hash mismatch', async () => { // Import TestWrappingAdapter that actually wraps data like production const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) const originalData = Buffer.from('test content for v5.10.0 regression test') // Write blob (TestWrappingAdapter wraps as {_binary: true, data: "base64..."}) const hash = await testBlobStorage.write(originalData) // Fresh instance = cold cache: read() must fetch + unwrap from storage. // v5.10.0 bug: read() hashed the wrapped bytes instead of the unwrapped // content; the fix runs unwrapBinaryData() before hash verification. const retrieved = await new BlobStorage(wrappingAdapter).read(hash) expect(retrieved.equals(originalData)).toBe(true) expect(retrieved.toString()).toBe('test content for v5.10.0 regression test') }) it('should handle multiple wrapped blobs without integrity errors', async () => { const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) const testData = [ Buffer.from('first blob'), Buffer.from('second blob'), Buffer.from('third blob with more content'), Buffer.from(JSON.stringify({ test: 'json data' })), ] const hashes: string[] = [] // Write all blobs for (const data of testData) { const hash = await testBlobStorage.write(data) hashes.push(hash) } // Cold-cache instance forces storage reads const coldStore = new BlobStorage(wrappingAdapter) for (let i = 0; i < hashes.length; i++) { const retrieved = await coldStore.read(hashes[i]) expect(retrieved.equals(testData[i])).toBe(true) } }) it('should work even if adapter fails to unwrap (defense-in-depth)', async () => { // Create a buggy adapter that doesn't unwrap properly class BuggyAdapter implements BlobStoreAdapter { private storage = new Map() async get(key: string): Promise { // Simulate a bug where adapter returns wrapped data instead of Buffer const data = this.storage.get(key) if (!data) return undefined // Return wrapped object (bug) instead of unwrapped Buffer return data // {_binary: true, data: "base64..."} } async put(key: string, data: Buffer): Promise { // Store as wrapped object (like real storage) this.storage.set(key, { _binary: true, data: data.toString('base64') }) } async delete(key: string): Promise { this.storage.delete(key) } async list(prefix: string): Promise { return Array.from(this.storage.keys()).filter(k => k.startsWith(prefix)) } } const buggyAdapter = new BuggyAdapter() const testBlobStorage = new BlobStorage(buggyAdapter) const originalData = Buffer.from('test defense-in-depth') const hash = await testBlobStorage.write(originalData) // Even with buggy adapter (and a cold cache), read() should unwrap and // verify correctly const retrieved = await new BlobStorage(buggyAdapter).read(hash) expect(retrieved.equals(originalData)).toBe(true) }) }) describe('Key-based dispatch regression (v6.2.0)', () => { it('should handle JSON-like compressed data without integrity failures', async () => { // THE KILLER TEST CASE: Data that looks like JSON when compressed // This would fail with v5.10.1 wrapBinaryData() guessing approach const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) // Create JSON data that will be compressed const jsonData = { nested: { key: 'value', array: [1, 2, 3] }, repeated: 'x'.repeat(1000) } const originalData = Buffer.from(JSON.stringify(jsonData)) // Write blob (compression happens, might create JSON-parseable bytes) const hash = await testBlobStorage.write(originalData) // v5.10.1 bug: If compressed bytes accidentally parse as JSON, // wrapBinaryData() would store parsed object instead of wrapped binary // On read: JSON.stringify(object) !== original compressed bytes → hash mismatch // // v6.2.0 fix: Key-based dispatch eliminates guessing // 'blob:hash' → Always wrapped as binary, never parsed const retrieved = await new BlobStorage(wrappingAdapter).read(hash) // Hash MUST match expect(BlobStorage.hash(retrieved)).toBe(hash) expect(retrieved.equals(originalData)).toBe(true) }) it('should handle metadata keys correctly', async () => { // Verify that key-based dispatch correctly handles metadata keys // This test verifies the dispatch logic, not the full read/write cycle const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() // Test metadata key dispatch: should parse as JSON const metadataObj = { hash: 'test123', size: 42, compression: 'none' as const } const metadataBuffer = Buffer.from(JSON.stringify(metadataObj)) await wrappingAdapter.put('blob-meta:test123', metadataBuffer) const retrieved = await wrappingAdapter.get('blob-meta:test123') // Should be parsed as JSON object (not wrapped as binary) expect(retrieved).toBeDefined() expect(typeof retrieved).toBe('object') expect(retrieved.hash).toBe('test123') expect(retrieved.size).toBe(42) }) it('should never parse blob bytes as JSON on the write path', async () => { // Verify that the blob bridge uses key-based dispatch, // NOT wrapBinaryData() guessing const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) // Create data that would trigger wrapBinaryData() bug if used const problematicData = Buffer.from('[{"looks":"like","valid":"json"}]') const hash = await testBlobStorage.write(problematicData) const retrieved = await new BlobStorage(wrappingAdapter).read(hash) // Should retrieve exact same bytes (no JSON parsing occurred) expect(retrieved.equals(problematicData)).toBe(true) expect(BlobStorage.hash(retrieved)).toBe(hash) }) }) })