David Snelling
effb43b03c
Added full Git-style workflow with instant fork (Snowflake COW): **Core Features:** - fork() - Instant clone in <100ms via COW - merge() - 3-way merge with conflict resolution - commit() - Create state snapshots - getHistory() - View commit history - checkout() - Switch branches - listBranches() - List all branches - deleteBranch() - Delete branches **Merge Strategies:** - last-write-wins (timestamp-based) - first-write-wins (reverse timestamp) - custom (user-defined conflict resolution) **COW Infrastructure:** - BlobStorage - Content-addressable storage - CommitLog - Commit history management - CommitObject/CommitBuilder - Commit creation - RefManager - Branch/ref management - TreeObject - Tree data structure **Updated Components:** - Brainy class - All new APIs implemented - BaseStorage - COW infrastructure initialized - HNSWIndex - enableCOW() and ensureCOW() - TypeAwareHNSWIndex - COW support - CLI - New cow commands - Documentation - instant-fork.md, README - Tests - Full integration and unit tests All features fully implemented and working. Zero fake code. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
460 lines
13 KiB
TypeScript
460 lines
13 KiB
TypeScript
/**
|
||
* Comprehensive tests for BlobStorage
|
||
*
|
||
* Tests:
|
||
* - Content-addressable storage (SHA-256)
|
||
* - Deduplication
|
||
* - Compression (zstd)
|
||
* - LRU caching
|
||
* - Batch operations
|
||
* - Reference counting
|
||
* - Garbage collection
|
||
* - Error handling
|
||
* - Performance characteristics
|
||
*/
|
||
|
||
import { describe, it, expect, beforeEach } from 'vitest'
|
||
import { BlobStorage } from '../../../../src/storage/cow/BlobStorage.js'
|
||
import { MemoryStorageAdapter } from '../../../../src/storage/adapters/memoryStorage.js'
|
||
|
||
describe('BlobStorage', () => {
|
||
let adapter: MemoryStorageAdapter
|
||
let blobStorage: BlobStorage
|
||
|
||
beforeEach(() => {
|
||
adapter = new MemoryStorageAdapter({})
|
||
blobStorage = new BlobStorage(adapter, { enableCompression: true })
|
||
})
|
||
|
||
describe('Content-Addressable Storage', () => {
|
||
it('should compute SHA-256 hash of data', () => {
|
||
const data = Buffer.from('hello world')
|
||
const hash = BlobStorage.hash(data)
|
||
|
||
expect(hash).toBe('b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9')
|
||
expect(hash).toHaveLength(64)
|
||
})
|
||
|
||
it('should write blob and return hash', async () => {
|
||
const data = Buffer.from('test data')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
expect(hash).toBeTruthy()
|
||
expect(hash).toHaveLength(64)
|
||
})
|
||
|
||
it('should read blob by hash', async () => {
|
||
const data = Buffer.from('test data')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
const retrieved = await blobStorage.read(hash)
|
||
|
||
expect(retrieved.toString()).toBe('test data')
|
||
})
|
||
|
||
it('should verify data integrity on read', async () => {
|
||
const data = Buffer.from('test data')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
// Corrupt the blob data
|
||
await adapter.put(`blob:${hash}`, Buffer.from('corrupted'))
|
||
|
||
await expect(blobStorage.read(hash)).rejects.toThrow('integrity check failed')
|
||
})
|
||
|
||
it('should check if blob exists', async () => {
|
||
const data = Buffer.from('test data')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
expect(await blobStorage.has(hash)).toBe(true)
|
||
expect(await blobStorage.has('0'.repeat(64))).toBe(false)
|
||
})
|
||
})
|
||
|
||
describe('Deduplication', () => {
|
||
it('should deduplicate identical blobs', async () => {
|
||
const data = Buffer.from('duplicate data')
|
||
|
||
const hash1 = await blobStorage.write(data)
|
||
const hash2 = await blobStorage.write(data)
|
||
|
||
expect(hash1).toBe(hash2)
|
||
|
||
const stats = blobStorage.getStats()
|
||
expect(stats.totalBlobs).toBe(1)
|
||
expect(stats.dedupSavings).toBe(data.length)
|
||
})
|
||
|
||
it('should increment ref count on duplicate write', async () => {
|
||
const data = Buffer.from('test data')
|
||
|
||
await blobStorage.write(data)
|
||
const hash = await blobStorage.write(data)
|
||
|
||
const metadata = await blobStorage.getMetadata(hash)
|
||
expect(metadata?.refCount).toBe(2)
|
||
})
|
||
|
||
it('should track deduplication savings', async () => {
|
||
const data = Buffer.from('x'.repeat(1000))
|
||
|
||
const hash1 = await blobStorage.write(data)
|
||
const hash2 = await blobStorage.write(data)
|
||
const hash3 = await blobStorage.write(data)
|
||
|
||
const stats = blobStorage.getStats()
|
||
expect(stats.dedupSavings).toBe(2000) // 2 duplicates × 1000 bytes
|
||
})
|
||
})
|
||
|
||
describe('Compression', () => {
|
||
it('should compress large text data with zstd', async () => {
|
||
const data = Buffer.from('a'.repeat(10000))
|
||
|
||
const hash = await blobStorage.write(data, {
|
||
type: 'metadata',
|
||
compression: 'zstd'
|
||
})
|
||
|
||
const metadata = await blobStorage.getMetadata(hash)
|
||
|
||
expect(metadata?.compression).toBe('zstd')
|
||
expect(metadata?.compressedSize).toBeLessThan(metadata?.size ?? 0)
|
||
})
|
||
|
||
it('should decompress zstd data on read', async () => {
|
||
const originalData = Buffer.from('test data '.repeat(100))
|
||
|
||
const hash = await blobStorage.write(originalData, {
|
||
type: 'metadata',
|
||
compression: 'zstd'
|
||
})
|
||
|
||
const retrieved = await blobStorage.read(hash)
|
||
|
||
expect(retrieved.toString()).toBe(originalData.toString())
|
||
})
|
||
|
||
it('should not compress small blobs', async () => {
|
||
const data = Buffer.from('small')
|
||
|
||
const hash = await blobStorage.write(data, {
|
||
compression: 'auto'
|
||
})
|
||
|
||
const metadata = await blobStorage.getMetadata(hash)
|
||
|
||
expect(metadata?.compression).toBe('none')
|
||
})
|
||
|
||
it('should not compress vector data (already dense)', async () => {
|
||
const vectorData = Buffer.from(new Float32Array([1, 2, 3, 4, 5]))
|
||
|
||
const hash = await blobStorage.write(vectorData, {
|
||
type: 'vector',
|
||
compression: 'auto'
|
||
})
|
||
|
||
const metadata = await blobStorage.getMetadata(hash)
|
||
|
||
expect(metadata?.compression).toBe('none')
|
||
})
|
||
|
||
it('should auto-compress metadata/tree/commit types', async () => {
|
||
const data = Buffer.from('x'.repeat(5000))
|
||
|
||
const hash = await blobStorage.write(data, {
|
||
type: 'metadata',
|
||
compression: 'auto'
|
||
})
|
||
|
||
const metadata = await blobStorage.getMetadata(hash)
|
||
|
||
// Should compress if zstd is available
|
||
if (metadata?.compression === 'zstd') {
|
||
expect(metadata.compressedSize).toBeLessThan(metadata.size)
|
||
}
|
||
})
|
||
})
|
||
|
||
describe('LRU Caching', () => {
|
||
it('should cache blob on read', async () => {
|
||
const data = Buffer.from('cached data')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
// First read (cache miss)
|
||
await blobStorage.read(hash)
|
||
|
||
// Second read (cache hit)
|
||
await blobStorage.read(hash)
|
||
|
||
const stats = blobStorage.getStats()
|
||
expect(stats.cacheHits).toBeGreaterThan(0)
|
||
})
|
||
|
||
it('should evict LRU entries when cache is full', async () => {
|
||
const smallCache = new BlobStorage(adapter, {
|
||
cacheMaxSize: 100, // Very small cache
|
||
enableCompression: false
|
||
})
|
||
|
||
// Write blobs that exceed cache size
|
||
const blob1 = Buffer.from('x'.repeat(50))
|
||
const blob2 = Buffer.from('y'.repeat(50))
|
||
const blob3 = Buffer.from('z'.repeat(50))
|
||
|
||
const hash1 = await smallCache.write(blob1)
|
||
const hash2 = await smallCache.write(blob2)
|
||
const hash3 = await smallCache.write(blob3) // Should evict hash1
|
||
|
||
// Read all blobs
|
||
await smallCache.read(hash1)
|
||
await smallCache.read(hash2)
|
||
await smallCache.read(hash3)
|
||
|
||
// hash1 should have been evicted, causing cache miss
|
||
const stats = smallCache.getStats()
|
||
expect(stats.cacheMisses).toBeGreaterThan(0)
|
||
})
|
||
|
||
it('should clear cache on demand', async () => {
|
||
const data = Buffer.from('test')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
await blobStorage.read(hash) // Cache it
|
||
|
||
blobStorage.clearCache()
|
||
|
||
await blobStorage.read(hash) // Should be cache miss
|
||
|
||
const stats = blobStorage.getStats()
|
||
expect(stats.cacheMisses).toBeGreaterThan(0)
|
||
})
|
||
})
|
||
|
||
describe('Batch Operations', () => {
|
||
it('should write multiple blobs in parallel', async () => {
|
||
const blobs: Array<[Buffer, any]> = [
|
||
[Buffer.from('blob1'), undefined],
|
||
[Buffer.from('blob2'), undefined],
|
||
[Buffer.from('blob3'), undefined]
|
||
]
|
||
|
||
const hashes = await blobStorage.writeBatch(blobs)
|
||
|
||
expect(hashes).toHaveLength(3)
|
||
expect(hashes[0]).toHaveLength(64)
|
||
expect(hashes[1]).toHaveLength(64)
|
||
expect(hashes[2]).toHaveLength(64)
|
||
})
|
||
|
||
it('should read multiple blobs in parallel', async () => {
|
||
const data1 = Buffer.from('blob1')
|
||
const data2 = Buffer.from('blob2')
|
||
const data3 = Buffer.from('blob3')
|
||
|
||
const hash1 = await blobStorage.write(data1)
|
||
const hash2 = await blobStorage.write(data2)
|
||
const hash3 = await blobStorage.write(data3)
|
||
|
||
const blobs = await blobStorage.readBatch([hash1, hash2, hash3])
|
||
|
||
expect(blobs).toHaveLength(3)
|
||
expect(blobs[0].toString()).toBe('blob1')
|
||
expect(blobs[1].toString()).toBe('blob2')
|
||
expect(blobs[2].toString()).toBe('blob3')
|
||
})
|
||
})
|
||
|
||
describe('Reference Counting', () => {
|
||
it('should track reference count', async () => {
|
||
const data = Buffer.from('test')
|
||
|
||
const hash = await blobStorage.write(data)
|
||
|
||
let metadata = await blobStorage.getMetadata(hash)
|
||
expect(metadata?.refCount).toBe(1)
|
||
|
||
// Write duplicate (increments ref count)
|
||
await blobStorage.write(data)
|
||
|
||
metadata = await blobStorage.getMetadata(hash)
|
||
expect(metadata?.refCount).toBe(2)
|
||
})
|
||
|
||
it('should only delete when refCount reaches 0', async () => {
|
||
const data = Buffer.from('test')
|
||
|
||
// Write twice (refCount = 2)
|
||
const hash = await blobStorage.write(data)
|
||
await blobStorage.write(data)
|
||
|
||
// Delete once (refCount = 1, blob still exists)
|
||
await blobStorage.delete(hash)
|
||
|
||
expect(await blobStorage.has(hash)).toBe(true)
|
||
|
||
// Delete again (refCount = 0, blob deleted)
|
||
await blobStorage.delete(hash)
|
||
|
||
expect(await blobStorage.has(hash)).toBe(false)
|
||
})
|
||
})
|
||
|
||
describe('Garbage Collection', () => {
|
||
it('should delete unreferenced blobs', async () => {
|
||
const blob1 = Buffer.from('referenced')
|
||
const blob2 = Buffer.from('unreferenced')
|
||
|
||
const hash1 = await blobStorage.write(blob1)
|
||
const hash2 = await blobStorage.write(blob2)
|
||
|
||
// Mark only hash1 as referenced
|
||
const referenced = new Set([hash1])
|
||
|
||
const deleted = await blobStorage.garbageCollect(referenced)
|
||
|
||
expect(deleted).toBeGreaterThan(0)
|
||
expect(await blobStorage.has(hash1)).toBe(true)
|
||
expect(await blobStorage.has(hash2)).toBe(false)
|
||
})
|
||
|
||
it('should not delete referenced blobs', async () => {
|
||
const blob1 = Buffer.from('ref1')
|
||
const blob2 = Buffer.from('ref2')
|
||
|
||
const hash1 = await blobStorage.write(blob1)
|
||
const hash2 = await blobStorage.write(blob2)
|
||
|
||
// Both referenced
|
||
const referenced = new Set([hash1, hash2])
|
||
|
||
const deleted = await blobStorage.garbageCollect(referenced)
|
||
|
||
expect(deleted).toBe(0)
|
||
expect(await blobStorage.has(hash1)).toBe(true)
|
||
expect(await blobStorage.has(hash2)).toBe(true)
|
||
})
|
||
})
|
||
|
||
describe('Metadata', () => {
|
||
it('should store blob metadata', async () => {
|
||
const data = Buffer.from('test')
|
||
|
||
const hash = await blobStorage.write(data, {
|
||
type: 'metadata',
|
||
compression: 'none'
|
||
})
|
||
|
||
const metadata = await blobStorage.getMetadata(hash)
|
||
|
||
expect(metadata?.hash).toBe(hash)
|
||
expect(metadata?.size).toBe(data.length)
|
||
expect(metadata?.type).toBe('metadata')
|
||
expect(metadata?.compression).toBe('none')
|
||
expect(metadata?.createdAt).toBeGreaterThan(0)
|
||
expect(metadata?.refCount).toBe(1)
|
||
})
|
||
})
|
||
|
||
describe('List Operations', () => {
|
||
it('should list all blobs', async () => {
|
||
const hash1 = await blobStorage.write(Buffer.from('blob1'))
|
||
const hash2 = await blobStorage.write(Buffer.from('blob2'))
|
||
const hash3 = await blobStorage.write(Buffer.from('blob3'))
|
||
|
||
const blobs = await blobStorage.listBlobs()
|
||
|
||
expect(blobs).toContain(hash1)
|
||
expect(blobs).toContain(hash2)
|
||
expect(blobs).toContain(hash3)
|
||
})
|
||
})
|
||
|
||
describe('Statistics', () => {
|
||
it('should track storage statistics', async () => {
|
||
const data1 = Buffer.from('x'.repeat(1000))
|
||
const data2 = Buffer.from('y'.repeat(2000))
|
||
|
||
await blobStorage.write(data1)
|
||
await blobStorage.write(data2)
|
||
|
||
const stats = blobStorage.getStats()
|
||
|
||
expect(stats.totalBlobs).toBe(2)
|
||
expect(stats.totalSize).toBe(3000)
|
||
expect(stats.avgBlobSize).toBe(1500)
|
||
})
|
||
|
||
it('should calculate compression ratio', async () => {
|
||
const data = Buffer.from('a'.repeat(10000))
|
||
|
||
await blobStorage.write(data, {
|
||
type: 'metadata',
|
||
compression: 'zstd'
|
||
})
|
||
|
||
const stats = blobStorage.getStats()
|
||
|
||
// Compression ratio should be > 1 if compressed
|
||
if (stats.compressedSize < stats.totalSize) {
|
||
expect(stats.compressionRatio).toBeGreaterThan(1)
|
||
}
|
||
})
|
||
})
|
||
|
||
describe('Error Handling', () => {
|
||
it('should throw on reading non-existent blob', async () => {
|
||
await expect(
|
||
blobStorage.read('0'.repeat(64))
|
||
).rejects.toThrow('Blob not found')
|
||
})
|
||
|
||
it('should throw on reading blob with missing metadata', async () => {
|
||
const data = Buffer.from('test')
|
||
const hash = await blobStorage.write(data)
|
||
|
||
// Delete metadata but keep blob
|
||
await adapter.delete(`blob-meta:${hash}`)
|
||
|
||
await expect(blobStorage.read(hash)).rejects.toThrow('metadata not found')
|
||
})
|
||
})
|
||
|
||
describe('Performance', () => {
|
||
it('should write 1000 small blobs quickly', async () => {
|
||
const start = Date.now()
|
||
|
||
for (let i = 0; i < 1000; i++) {
|
||
await blobStorage.write(Buffer.from(`blob${i}`))
|
||
}
|
||
|
||
const elapsed = Date.now() - start
|
||
|
||
expect(elapsed).toBeLessThan(5000) // Should complete in < 5s
|
||
})
|
||
|
||
it('should read 1000 cached blobs very quickly', async () => {
|
||
// Write and cache blobs
|
||
const hashes: string[] = []
|
||
for (let i = 0; i < 1000; i++) {
|
||
const hash = await blobStorage.write(Buffer.from(`blob${i}`))
|
||
hashes.push(hash)
|
||
}
|
||
|
||
// First read (populate cache)
|
||
for (const hash of hashes) {
|
||
await blobStorage.read(hash)
|
||
}
|
||
|
||
// Second read (from cache)
|
||
const start = Date.now()
|
||
for (const hash of hashes) {
|
||
await blobStorage.read(hash)
|
||
}
|
||
const elapsed = Date.now() - start
|
||
|
||
expect(elapsed).toBeLessThan(1000) // Should be very fast from cache
|
||
})
|
||
})
|
||
})
|