brainy/tests/unit/storage/cow/BlobStorage.test.ts

/**
 * Comprehensive tests for BlobStorage
 *
 * Tests:
 * - Content-addressable storage (SHA-256)
 * - Deduplication
 * - Compression (zstd)
 * - LRU caching
 * - Batch operations
 * - Reference counting
 * - Garbage collection
 * - Error handling
 * - Performance characteristics
 */

import { describe, it, expect, beforeEach } from 'vitest'
import { BlobStorage, COWStorageAdapter } from '../../../../src/storage/cow/BlobStorage.js'

/**
 * Simple in-memory COW storage adapter for testing
 */
class InMemoryCOWAdapter implements COWStorageAdapter {
  private store = new Map<string, Buffer>()

  async get(key: string): Promise<Buffer | undefined> {
    return this.store.get(key)
  }

  async put(key: string, data: Buffer): Promise<void> {
    this.store.set(key, data)
  }

  async delete(key: string): Promise<void> {
    this.store.delete(key)
  }

  async list(prefix: string): Promise<string[]> {
    const keys: string[] = []
    for (const key of this.store.keys()) {
      if (key.startsWith(prefix)) {
        keys.push(key)
      }
    }
    return keys.sort()
  }
}

describe('BlobStorage', () => {
  let adapter: COWStorageAdapter
  let blobStorage: BlobStorage

  beforeEach(() => {
    adapter = new InMemoryCOWAdapter()
    blobStorage = new BlobStorage(adapter, { enableCompression: true })
  })

  describe('Content-Addressable Storage', () => {
    it('should compute SHA-256 hash of data', () => {
      const data = Buffer.from('hello world')
      const hash = BlobStorage.hash(data)

      expect(hash).toBe('b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9')
      expect(hash).toHaveLength(64)
    })

    it('should write blob and return hash', async () => {
      const data = Buffer.from('test data')
      const hash = await blobStorage.write(data)

      expect(hash).toBeTruthy()
      expect(hash).toHaveLength(64)
    })

    it('should read blob by hash', async () => {
      const data = Buffer.from('test data')
      const hash = await blobStorage.write(data)

      const retrieved = await blobStorage.read(hash)

      expect(retrieved.toString()).toBe('test data')
    })

    it('should verify data integrity on read', async () => {
      const data = Buffer.from('test data')
      const hash = await blobStorage.write(data)

      // Clear cache so read() fetches from storage
      blobStorage.clearCache()

      // Corrupt the blob data
      await adapter.put(`blob:${hash}`, Buffer.from('corrupted'))

      // Should detect corruption via hash verification
      await expect(blobStorage.read(hash)).rejects.toThrow('integrity check failed')
    })

    it('should check if blob exists', async () => {
      const data = Buffer.from('test data')
      const hash = await blobStorage.write(data)

      expect(await blobStorage.has(hash)).toBe(true)
      expect(await blobStorage.has('0'.repeat(64))).toBe(false)
    })
  })

  describe('Deduplication', () => {
    it('should deduplicate identical blobs', async () => {
      const data = Buffer.from('duplicate data')

      const hash1 = await blobStorage.write(data)
      const hash2 = await blobStorage.write(data)

      expect(hash1).toBe(hash2)

      const stats = blobStorage.getStats()
      expect(stats.totalBlobs).toBe(1)
      expect(stats.dedupSavings).toBe(data.length)
    })

    it('should increment ref count on duplicate write', async () => {
      const data = Buffer.from('test data')

      await blobStorage.write(data)
      const hash = await blobStorage.write(data)

      const metadata = await blobStorage.getMetadata(hash)
      expect(metadata?.refCount).toBe(2)
    })

    it('should track deduplication savings', async () => {
      const data = Buffer.from('x'.repeat(1000))

      const hash1 = await blobStorage.write(data)
      const hash2 = await blobStorage.write(data)
      const hash3 = await blobStorage.write(data)

      const stats = blobStorage.getStats()
      expect(stats.dedupSavings).toBe(2000)  // 2 duplicates × 1000 bytes
    })
  })

  describe('Compression', () => {
    it('should compress large text data with zstd', async () => {
      const data = Buffer.from('a'.repeat(10000))

      const hash = await blobStorage.write(data, {
        type: 'metadata',
        compression: 'zstd'
      })

      const metadata = await blobStorage.getMetadata(hash)

      // zstd may not be available in test environment - falls back to 'none'
      // This is expected behavior (see BlobStorage initCompression fallback)
      if (metadata?.compression === 'zstd') {
        expect(metadata.compressedSize).toBeLessThan(metadata.size)
      } else {
        // Fallback to 'none' is acceptable when zstd unavailable
        expect(metadata?.compression).toBe('none')
      }
    })

    it('should decompress zstd data on read', async () => {
      const originalData = Buffer.from('test data '.repeat(100))

      const hash = await blobStorage.write(originalData, {
        type: 'metadata',
        compression: 'zstd'
      })

      const retrieved = await blobStorage.read(hash)

      expect(retrieved.toString()).toBe(originalData.toString())
    })

    it('should not compress small blobs', async () => {
      const data = Buffer.from('small')

      const hash = await blobStorage.write(data, {
        compression: 'auto'
      })

      const metadata = await blobStorage.getMetadata(hash)

      expect(metadata?.compression).toBe('none')
    })

    it('should not compress vector data (already dense)', async () => {
      const vectorData = Buffer.from(new Float32Array([1, 2, 3, 4, 5]))

      const hash = await blobStorage.write(vectorData, {
        type: 'vector',
        compression: 'auto'
      })

      const metadata = await blobStorage.getMetadata(hash)

      expect(metadata?.compression).toBe('none')
    })

    it('should auto-compress metadata/tree/commit types', async () => {
      const data = Buffer.from('x'.repeat(5000))

      const hash = await blobStorage.write(data, {
        type: 'metadata',
        compression: 'auto'
      })

      const metadata = await blobStorage.getMetadata(hash)

      // Should compress if zstd is available
      if (metadata?.compression === 'zstd') {
        expect(metadata.compressedSize).toBeLessThan(metadata.size)
      }
    })
  })

  describe('LRU Caching', () => {
    it('should cache blob on read', async () => {
      const data = Buffer.from('cached data')
      const hash = await blobStorage.write(data)

      // First read (cache miss)
      await blobStorage.read(hash)

      // Second read (cache hit)
      await blobStorage.read(hash)

      const stats = blobStorage.getStats()
      expect(stats.cacheHits).toBeGreaterThan(0)
    })

    it('should evict LRU entries when cache is full', async () => {
      const smallCache = new BlobStorage(adapter, {
        cacheMaxSize: 100,  // Very small cache
        enableCompression: false
      })

      // Write blobs that exceed cache size
      const blob1 = Buffer.from('x'.repeat(50))
      const blob2 = Buffer.from('y'.repeat(50))
      const blob3 = Buffer.from('z'.repeat(50))

      const hash1 = await smallCache.write(blob1)
      const hash2 = await smallCache.write(blob2)
      const hash3 = await smallCache.write(blob3)  // Should evict hash1

      // Read all blobs
      await smallCache.read(hash1)
      await smallCache.read(hash2)
      await smallCache.read(hash3)

      // hash1 should have been evicted, causing cache miss
      const stats = smallCache.getStats()
      expect(stats.cacheMisses).toBeGreaterThan(0)
    })

    it('should clear cache on demand', async () => {
      const data = Buffer.from('test')
      const hash = await blobStorage.write(data)

      await blobStorage.read(hash)  // Cache it

      blobStorage.clearCache()

      await blobStorage.read(hash)  // Should be cache miss

      const stats = blobStorage.getStats()
      expect(stats.cacheMisses).toBeGreaterThan(0)
    })
  })

  describe('Batch Operations', () => {
    it('should write multiple blobs in parallel', async () => {
      const blobs: Array<[Buffer, any]> = [
        [Buffer.from('blob1'), undefined],
        [Buffer.from('blob2'), undefined],
        [Buffer.from('blob3'), undefined]
      ]

      const hashes = await blobStorage.writeBatch(blobs)

      expect(hashes).toHaveLength(3)
      expect(hashes[0]).toHaveLength(64)
      expect(hashes[1]).toHaveLength(64)
      expect(hashes[2]).toHaveLength(64)
    })

    it('should read multiple blobs in parallel', async () => {
      const data1 = Buffer.from('blob1')
      const data2 = Buffer.from('blob2')
      const data3 = Buffer.from('blob3')

      const hash1 = await blobStorage.write(data1)
      const hash2 = await blobStorage.write(data2)
      const hash3 = await blobStorage.write(data3)

      const blobs = await blobStorage.readBatch([hash1, hash2, hash3])

      expect(blobs).toHaveLength(3)
      expect(blobs[0].toString()).toBe('blob1')
      expect(blobs[1].toString()).toBe('blob2')
      expect(blobs[2].toString()).toBe('blob3')
    })
  })

  describe('Reference Counting', () => {
    it('should track reference count', async () => {
      const data = Buffer.from('test')

      const hash = await blobStorage.write(data)

      let metadata = await blobStorage.getMetadata(hash)
      expect(metadata?.refCount).toBe(1)

      // Write duplicate (increments ref count)
      await blobStorage.write(data)

      metadata = await blobStorage.getMetadata(hash)
      expect(metadata?.refCount).toBe(2)
    })

    it('should only delete when refCount reaches 0', async () => {
      const data = Buffer.from('test')

      // Write twice (refCount = 2)
      const hash = await blobStorage.write(data)
      await blobStorage.write(data)

      // Delete once (refCount = 1, blob still exists)
      await blobStorage.delete(hash)

      expect(await blobStorage.has(hash)).toBe(true)

      // Delete again (refCount = 0, blob deleted)
      await blobStorage.delete(hash)

      expect(await blobStorage.has(hash)).toBe(false)
    })
  })

  describe('Garbage Collection', () => {
    it('should delete unreferenced blobs', async () => {
      const blob1 = Buffer.from('referenced')
      const blob2 = Buffer.from('unreferenced')

      const hash1 = await blobStorage.write(blob1)
      const hash2 = await blobStorage.write(blob2)

      // Manually set refCount to 0 for unreferenced blob
      // (In real COW usage, refCount tracks actual references from commits/trees)
      // GC only deletes when refCount === 0 AND not in referenced set
      const metadata2 = await blobStorage.getMetadata(hash2)
      if (metadata2) {
        metadata2.refCount = 0
        await adapter.put(`blob-meta:${hash2}`, Buffer.from(JSON.stringify(metadata2)))
      }

      // Mark only hash1 as referenced
      const referenced = new Set([hash1])

      const deleted = await blobStorage.garbageCollect(referenced)

      expect(deleted).toBeGreaterThan(0)
      expect(await blobStorage.has(hash1)).toBe(true)
      expect(await blobStorage.has(hash2)).toBe(false)
    })

    it('should not delete referenced blobs', async () => {
      const blob1 = Buffer.from('ref1')
      const blob2 = Buffer.from('ref2')

      const hash1 = await blobStorage.write(blob1)
      const hash2 = await blobStorage.write(blob2)

      // Both referenced
      const referenced = new Set([hash1, hash2])

      const deleted = await blobStorage.garbageCollect(referenced)

      expect(deleted).toBe(0)
      expect(await blobStorage.has(hash1)).toBe(true)
      expect(await blobStorage.has(hash2)).toBe(true)
    })
  })

  describe('Metadata', () => {
    it('should store blob metadata', async () => {
      const data = Buffer.from('test')

      const hash = await blobStorage.write(data, {
        type: 'metadata',
        compression: 'none'
      })

      const metadata = await blobStorage.getMetadata(hash)

      expect(metadata?.hash).toBe(hash)
      expect(metadata?.size).toBe(data.length)
      expect(metadata?.type).toBe('metadata')
      expect(metadata?.compression).toBe('none')
      expect(metadata?.createdAt).toBeGreaterThan(0)
      expect(metadata?.refCount).toBe(1)
    })
  })

  describe('List Operations', () => {
    it('should list all blobs', async () => {
      const hash1 = await blobStorage.write(Buffer.from('blob1'))
      const hash2 = await blobStorage.write(Buffer.from('blob2'))
      const hash3 = await blobStorage.write(Buffer.from('blob3'))

      const blobs = await blobStorage.listBlobs()

      expect(blobs).toContain(hash1)
      expect(blobs).toContain(hash2)
      expect(blobs).toContain(hash3)
    })
  })

  describe('Statistics', () => {
    it('should track storage statistics', async () => {
      const data1 = Buffer.from('x'.repeat(1000))
      const data2 = Buffer.from('y'.repeat(2000))

      await blobStorage.write(data1)
      await blobStorage.write(data2)

      const stats = blobStorage.getStats()

      expect(stats.totalBlobs).toBe(2)
      expect(stats.totalSize).toBe(3000)
      expect(stats.avgBlobSize).toBe(1500)
    })

    it('should calculate compression ratio', async () => {
      const data = Buffer.from('a'.repeat(10000))

      await blobStorage.write(data, {
        type: 'metadata',
        compression: 'zstd'
      })

      const stats = blobStorage.getStats()

      // Compression ratio should be > 1 if compressed
      if (stats.compressedSize < stats.totalSize) {
        expect(stats.compressionRatio).toBeGreaterThan(1)
      }
    })
  })

  describe('Error Handling', () => {
    it('should throw on reading non-existent blob', async () => {
      // Use 'f' instead of '0' to avoid NULL_HASH sentinel value
      await expect(
        blobStorage.read('f'.repeat(64))
      ).rejects.toThrow('Blob metadata not found')
    })

    it('should throw on reading blob with missing metadata', async () => {
      const data = Buffer.from('test')
      const hash = await blobStorage.write(data)

      // Clear cache so read() actually checks metadata
      blobStorage.clearCache()

      // Delete metadata but keep blob
      await adapter.delete(`blob-meta:${hash}`)

      // Use skipCache to ensure we check metadata
      await expect(blobStorage.read(hash, { skipCache: true })).rejects.toThrow('metadata not found')
    })
  })

  describe('Performance', () => {
    it('should write 1000 small blobs quickly', async () => {
      const start = Date.now()

      for (let i = 0; i < 1000; i++) {
        await blobStorage.write(Buffer.from(`blob${i}`))
      }

      const elapsed = Date.now() - start

      expect(elapsed).toBeLessThan(5000)  // Should complete in < 5s
    })

    it('should read 1000 cached blobs very quickly', async () => {
      // Write and cache blobs
      const hashes: string[] = []
      for (let i = 0; i < 1000; i++) {
        const hash = await blobStorage.write(Buffer.from(`blob${i}`))
        hashes.push(hash)
      }

      // First read (populate cache)
      for (const hash of hashes) {
        await blobStorage.read(hash)
      }

      // Second read (from cache)
      const start = Date.now()
      for (const hash of hashes) {
        await blobStorage.read(hash)
      }
      const elapsed = Date.now() - start

      expect(elapsed).toBeLessThan(1000)  // Should be very fast from cache
    })
  })

  describe('v5.10.0 Regression - Blob Integrity with Wrapped Data', () => {
    it('should handle wrapped binary data without hash mismatch (v5.10.0 fix)', async () => {
      // Import TestWrappingAdapter that actually wraps data like production
      const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
      const wrappingAdapter = new TestWrappingAdapter()
      const testBlobStorage = new BlobStorage(wrappingAdapter)

      const originalData = Buffer.from('test content for v5.10.0 regression test')

      // Write blob (TestWrappingAdapter wraps as {_binary: true, data: "base64..."})
      const hash = await testBlobStorage.write(originalData)

      // Clear cache to force re-read from storage (bypasses in-memory cache)
      testBlobStorage.clearCache()

      // v5.10.0 bug: This would fail with "Blob integrity check failed"
      // because BlobStorage.read() was hashing the wrapped data instead of unwrapped
      // v5.10.1 fix: unwrapBinaryData() is called before hash verification
      const retrieved = await testBlobStorage.read(hash)

      expect(retrieved.equals(originalData)).toBe(true)
      expect(retrieved.toString()).toBe('test content for v5.10.0 regression test')
    })

    it('should handle multiple wrapped blobs without integrity errors', async () => {
      const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
      const wrappingAdapter = new TestWrappingAdapter()
      const testBlobStorage = new BlobStorage(wrappingAdapter)

      const testData = [
        Buffer.from('first blob'),
        Buffer.from('second blob'),
        Buffer.from('third blob with more content'),
        Buffer.from(JSON.stringify({ test: 'json data' })),
      ]

      const hashes: string[] = []

      // Write all blobs
      for (const data of testData) {
        const hash = await testBlobStorage.write(data)
        hashes.push(hash)
      }

      // Clear cache
      testBlobStorage.clearCache()

      // Read all blobs (would fail in v5.10.0)
      for (let i = 0; i < hashes.length; i++) {
        const retrieved = await testBlobStorage.read(hashes[i])
        expect(retrieved.equals(testData[i])).toBe(true)
      }
    })

    it('should work even if adapter fails to unwrap (defense-in-depth)', async () => {
      // Create a buggy adapter that doesn't unwrap properly
      class BuggyAdapter implements COWStorageAdapter {
        private storage = new Map<string, any>()

        async get(key: string): Promise<any | undefined> {
          // Simulate a bug where adapter returns wrapped data instead of Buffer
          const data = this.storage.get(key)
          if (!data) return undefined

          // Return wrapped object (bug) instead of unwrapped Buffer
          return data // {_binary: true, data: "base64..."}
        }

        async put(key: string, data: Buffer): Promise<void> {
          // Store as wrapped object (like real storage)
          this.storage.set(key, {
            _binary: true,
            data: data.toString('base64')
          })
        }

        async delete(key: string): Promise<void> {
          this.storage.delete(key)
        }

        async list(prefix: string): Promise<string[]> {
          return Array.from(this.storage.keys()).filter(k => k.startsWith(prefix))
        }
      }

      const buggyAdapter = new BuggyAdapter()
      const testBlobStorage = new BlobStorage(buggyAdapter)

      const originalData = Buffer.from('test defense-in-depth')
      const hash = await testBlobStorage.write(originalData)

      testBlobStorage.clearCache()

      // Even with buggy adapter, BlobStorage.read() should unwrap and verify correctly
      const retrieved = await testBlobStorage.read(hash)
      expect(retrieved.equals(originalData)).toBe(true)
    })
  })

  describe('v6.2.0 Regression - Key-Based Dispatch (Permanent Fix)', () => {
    it('should handle JSON-like compressed data without integrity failures', async () => {
      // THE KILLER TEST CASE: Data that looks like JSON when compressed
      // This would fail with v5.10.1 wrapBinaryData() guessing approach
      const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
      const wrappingAdapter = new TestWrappingAdapter()
      const testBlobStorage = new BlobStorage(wrappingAdapter, { enableCompression: true })

      // Create JSON data that will be compressed
      const jsonData = { nested: { key: 'value', array: [1, 2, 3] }, repeated: 'x'.repeat(1000) }
      const originalData = Buffer.from(JSON.stringify(jsonData))

      // Write blob (compression happens, might create JSON-parseable bytes)
      const hash = await testBlobStorage.write(originalData)

      // Clear cache to force re-read from storage
      testBlobStorage.clearCache()

      // v5.10.1 bug: If compressed bytes accidentally parse as JSON,
      // wrapBinaryData() would store parsed object instead of wrapped binary
      // On read: JSON.stringify(object) !== original compressed bytes → hash mismatch
      //
      // v6.2.0 fix: Key-based dispatch eliminates guessing
      // 'blob:hash' → Always wrapped as binary, never parsed
      const retrieved = await testBlobStorage.read(hash)

      // Hash MUST match
      expect(BlobStorage.hash(retrieved)).toBe(hash)
      expect(retrieved.equals(originalData)).toBe(true)
    })

    it('should correctly dispatch all key types', async () => {
      // Verify key-based dispatch works for all COW key patterns
      const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
      const wrappingAdapter = new TestWrappingAdapter()
      const testBlobStorage = new BlobStorage(wrappingAdapter)

      // Test binary blob keys
      const blobData = Buffer.from('binary blob content')
      const blobHash = await testBlobStorage.write(blobData, { type: 'blob' })
      testBlobStorage.clearCache()
      const retrievedBlob = await testBlobStorage.read(blobHash)
      expect(retrievedBlob.equals(blobData)).toBe(true)

      // Test commit keys
      const commitData = Buffer.from('commit content')
      const commitHash = await testBlobStorage.write(commitData, { type: 'commit' })
      testBlobStorage.clearCache()
      const retrievedCommit = await testBlobStorage.read(commitHash)
      expect(retrievedCommit.equals(commitData)).toBe(true)

      // Test tree keys
      const treeData = Buffer.from('tree content')
      const treeHash = await testBlobStorage.write(treeData, { type: 'tree' })
      testBlobStorage.clearCache()
      const retrievedTree = await testBlobStorage.read(treeHash)
      expect(retrievedTree.equals(treeData)).toBe(true)
    })

    it('should handle metadata keys correctly', async () => {
      // Verify that key-based dispatch correctly handles metadata keys
      // This test verifies the dispatch logic, not the full read/write cycle
      const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
      const wrappingAdapter = new TestWrappingAdapter()

      // Test metadata key dispatch: should parse as JSON
      const metadataObj = { hash: 'test123', size: 42, compression: 'none' as const }
      const metadataBuffer = Buffer.from(JSON.stringify(metadataObj))

      await wrappingAdapter.put('blob-meta:test123', metadataBuffer)
      const retrieved = await wrappingAdapter.get('blob-meta:test123')

      // Should be parsed as JSON object (not wrapped as binary)
      expect(retrieved).toBeDefined()
      expect(typeof retrieved).toBe('object')
      expect(retrieved.hash).toBe('test123')
      expect(retrieved.size).toBe(42)
    })

    it('should never call wrapBinaryData on write path', async () => {
      // Verify that baseStorage COW adapter uses key-based dispatch,
      // NOT wrapBinaryData() guessing
      const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
      const wrappingAdapter = new TestWrappingAdapter()
      const testBlobStorage = new BlobStorage(wrappingAdapter)

      // Create data that would trigger wrapBinaryData() bug if used
      const problematicData = Buffer.from('[{"looks":"like","valid":"json"}]')
      const hash = await testBlobStorage.write(problematicData)

      testBlobStorage.clearCache()
      const retrieved = await testBlobStorage.read(hash)

      // Should retrieve exact same bytes (no JSON parsing occurred)
      expect(retrieved.equals(problematicData)).toBe(true)
      expect(BlobStorage.hash(retrieved)).toBe(hash)
    })
  })
})