feat(8.0)!: delete fork/branch/commit/history/versions — superseded by the Db API
The COW version-control surface (fork, branches, checkout, commit,
getHistory/streamHistory, asOfCommit, brain.versions) is gone, along with
its subsystems: src/versioning/, the COW object store (CommitLog,
CommitObject, RefManager, TreeObject), HistoricalStorageAdapter, and the
TypeAwareHNSWIndex path it kept alive. The Db API (now/transact/asOf/
with/persist/restore) is the one versioning model in 8.0.
Survivors and replacements:
- BlobStorage survives (the VFS stores file content through it), relocated
to src/storage/blobStorage.ts with binaryDataCodec.ts; its adapter
interface is now BlobStoreAdapter, slimmed to the consumed surface
(write/read/has/delete/getMetadata + MIME-aware compression policy).
- brain.migrate() backup branches are replaced by persist-before-migrate:
MigrateOptions.backupTo persists a hard-link snapshot of the current
generation before any transform runs; MigrationResult.backupPath reports
it, and brain.restore(path) brings it back wholesale.
- CLI: cow.ts (fork/branch/checkout/history/migrate) is replaced by
snapshot.ts — snapshot <path>, restore <path>, history (tx-log),
generation.
- New public read API: brain.transactionLog({limit}) exposes the reified
tx-log (generation/timestamp/meta, newest first) that backs the CLI
history command; TxLogEntry is exported.
Tests: superseded suites deleted; fork/commit blocks excised from shared
suites; BlobStorage tests relocated + reworked against the slimmed store;
migration tests now prove the backupTo snapshot/restore round trip; new
transactionLog coverage in db-mvcc.
This commit is contained in:
parent
431cd64406
commit
8f93add705
64 changed files with 1444 additions and 16313 deletions
|
|
@ -2,7 +2,7 @@
|
|||
* @module BlobStorage.compression-policy.test
|
||||
* @description 2.5.0 #32 — content-type-aware compression policy.
|
||||
*
|
||||
* COW's BlobStorage now consults `BlobWriteOptions.mimeType` in `auto` mode
|
||||
* BlobStorage consults `BlobWriteOptions.mimeType` in `auto` mode
|
||||
* and skips zstd for MIME types known to be already heavily compressed (JPEG,
|
||||
* PNG, MP4, MP3, ZIP, PDF, etc.). zstd over those formats is reliably a CPU
|
||||
* loss for no measurable byte savings.
|
||||
|
|
@ -24,11 +24,11 @@
|
|||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import {
|
||||
BlobStorage,
|
||||
COWStorageAdapter,
|
||||
BlobStoreAdapter,
|
||||
isAlreadyCompressedMimeType
|
||||
} from '../../../../src/storage/cow/BlobStorage.js'
|
||||
} from '../../../src/storage/blobStorage.js'
|
||||
|
||||
class InMemoryCOWAdapter implements COWStorageAdapter {
|
||||
class InMemoryBlobAdapter implements BlobStoreAdapter {
|
||||
private store = new Map<string, Buffer>()
|
||||
async get(key: string): Promise<Buffer | undefined> { return this.store.get(key) }
|
||||
async put(key: string, data: Buffer): Promise<void> { this.store.set(key, data) }
|
||||
|
|
@ -45,12 +45,12 @@ class InMemoryCOWAdapter implements COWStorageAdapter {
|
|||
}
|
||||
|
||||
describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
||||
let adapter: InMemoryCOWAdapter
|
||||
let adapter: InMemoryBlobAdapter
|
||||
let blob: BlobStorage
|
||||
|
||||
beforeEach(() => {
|
||||
adapter = new InMemoryCOWAdapter()
|
||||
blob = new BlobStorage(adapter, { enableCompression: true })
|
||||
adapter = new InMemoryBlobAdapter()
|
||||
blob = new BlobStorage(adapter)
|
||||
})
|
||||
|
||||
describe('isAlreadyCompressedMimeType helper', () => {
|
||||
|
|
@ -90,7 +90,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
|
||||
it('skips compression for image/jpeg in auto mode (metadata records "none")', async () => {
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'image/jpeg'
|
||||
compression: 'auto', mimeType: 'image/jpeg'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
|
|
@ -101,7 +101,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
|
||||
it('skips compression for video/mp4 in auto mode', async () => {
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'video/mp4'
|
||||
compression: 'auto', mimeType: 'video/mp4'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
|
|
@ -109,7 +109,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
|
||||
it('skips compression for application/zip in auto mode', async () => {
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'application/zip'
|
||||
compression: 'auto', mimeType: 'application/zip'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
|
|
@ -122,7 +122,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
// falls back to 'none' — same outcome as a no-zstd install would see
|
||||
// in production. Either way, the policy didn't block on text/plain.
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'text/plain'
|
||||
compression: 'auto', mimeType: 'text/plain'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression === 'zstd' || meta?.compression === 'none').toBe(true)
|
||||
|
|
@ -130,10 +130,10 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
|
||||
it('decompresses transparently on read regardless of compression decision', async () => {
|
||||
const jpegHash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'image/jpeg'
|
||||
compression: 'auto', mimeType: 'image/jpeg'
|
||||
})
|
||||
const textHash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'text/plain'
|
||||
compression: 'auto', mimeType: 'text/plain'
|
||||
})
|
||||
// Different mime-type policies, same bytes back on read.
|
||||
expect((await blob.read(jpegHash)).equals(HIGHLY_COMPRESSIBLE)).toBe(true)
|
||||
|
|
@ -145,7 +145,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
it('compression: "none" is honoured for text/plain (policy can\'t force compression on)', async () => {
|
||||
const data = Buffer.alloc(4096, 0x43)
|
||||
const hash = await blob.write(data, {
|
||||
compression: 'none', type: 'blob', mimeType: 'text/plain'
|
||||
compression: 'none', mimeType: 'text/plain'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
|
|
@ -156,7 +156,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
|||
it('compression: "zstd" is recorded as the intent for image/jpeg (policy applies only to auto)', async () => {
|
||||
const data = Buffer.alloc(4096, 0x42)
|
||||
const hash = await blob.write(data, {
|
||||
compression: 'zstd', type: 'blob', mimeType: 'image/jpeg'
|
||||
compression: 'zstd', mimeType: 'image/jpeg'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
// Explicit zstd → the policy DOESN'T short-circuit; the metadata
|
||||
|
|
@ -1,25 +1,28 @@
|
|||
/**
|
||||
* Comprehensive tests for BlobStorage
|
||||
* Comprehensive tests for BlobStorage (src/storage/blobStorage.ts)
|
||||
*
|
||||
* Tests:
|
||||
* - Content-addressable storage (SHA-256)
|
||||
* - Deduplication
|
||||
* - Compression (zstd)
|
||||
* - LRU caching
|
||||
* - Batch operations
|
||||
* - Reference counting
|
||||
* - Garbage collection
|
||||
* - Content-addressable storage (SHA-256, integrity verification)
|
||||
* - Deduplication via reference counting
|
||||
* - Compression (zstd, MIME-aware auto mode)
|
||||
* - LRU caching (bounded cache still serves correct bytes)
|
||||
* - Reference counting + delete-at-zero
|
||||
* - Error handling
|
||||
* - Performance characteristics
|
||||
* - Wrapped-binary-data regressions (key-based dispatch)
|
||||
*
|
||||
* Cache-bypass pattern: the store has no cache-introspection API (the LRU is
|
||||
* an internal optimization), so tests that must force a storage read create a
|
||||
* FRESH BlobStorage over the same adapter — a cold cache by construction.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { BlobStorage, COWStorageAdapter } from '../../../../src/storage/cow/BlobStorage.js'
|
||||
import { BlobStorage, BlobStoreAdapter } from '../../../src/storage/blobStorage.js'
|
||||
|
||||
/**
|
||||
* Simple in-memory COW storage adapter for testing
|
||||
* Simple in-memory blob store adapter for testing
|
||||
*/
|
||||
class InMemoryCOWAdapter implements COWStorageAdapter {
|
||||
class InMemoryBlobAdapter implements BlobStoreAdapter {
|
||||
private store = new Map<string, Buffer>()
|
||||
|
||||
async get(key: string): Promise<Buffer | undefined> {
|
||||
|
|
@ -46,12 +49,12 @@ class InMemoryCOWAdapter implements COWStorageAdapter {
|
|||
}
|
||||
|
||||
describe('BlobStorage', () => {
|
||||
let adapter: COWStorageAdapter
|
||||
let adapter: BlobStoreAdapter
|
||||
let blobStorage: BlobStorage
|
||||
|
||||
beforeEach(() => {
|
||||
adapter = new InMemoryCOWAdapter()
|
||||
blobStorage = new BlobStorage(adapter, { enableCompression: true })
|
||||
adapter = new InMemoryBlobAdapter()
|
||||
blobStorage = new BlobStorage(adapter)
|
||||
})
|
||||
|
||||
describe('Content-Addressable Storage', () => {
|
||||
|
|
@ -84,14 +87,13 @@ describe('BlobStorage', () => {
|
|||
const data = Buffer.from('test data')
|
||||
const hash = await blobStorage.write(data)
|
||||
|
||||
// Clear cache so read() fetches from storage
|
||||
blobStorage.clearCache()
|
||||
|
||||
// Corrupt the blob data
|
||||
// Corrupt the blob data behind the store's back
|
||||
await adapter.put(`blob:${hash}`, Buffer.from('corrupted'))
|
||||
|
||||
// Should detect corruption via hash verification
|
||||
await expect(blobStorage.read(hash)).rejects.toThrow('integrity check failed')
|
||||
// A fresh instance has a cold cache, so read() must hit storage and
|
||||
// detect the corruption via hash verification.
|
||||
const coldStore = new BlobStorage(adapter)
|
||||
await expect(coldStore.read(hash)).rejects.toThrow('integrity check failed')
|
||||
})
|
||||
|
||||
it('should check if blob exists', async () => {
|
||||
|
|
@ -112,9 +114,9 @@ describe('BlobStorage', () => {
|
|||
|
||||
expect(hash1).toBe(hash2)
|
||||
|
||||
const stats = blobStorage.getStats()
|
||||
expect(stats.totalBlobs).toBe(1)
|
||||
expect(stats.dedupSavings).toBe(data.length)
|
||||
// Exactly one stored blob + one metadata record
|
||||
expect(await adapter.list('blob:')).toHaveLength(1)
|
||||
expect(await adapter.list('blob-meta:')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('should increment ref count on duplicate write', async () => {
|
||||
|
|
@ -127,15 +129,17 @@ describe('BlobStorage', () => {
|
|||
expect(metadata?.refCount).toBe(2)
|
||||
})
|
||||
|
||||
it('should track deduplication savings', async () => {
|
||||
it('should track every duplicate write in the reference count', async () => {
|
||||
const data = Buffer.from('x'.repeat(1000))
|
||||
|
||||
const hash1 = await blobStorage.write(data)
|
||||
const hash2 = await blobStorage.write(data)
|
||||
const hash3 = await blobStorage.write(data)
|
||||
|
||||
const stats = blobStorage.getStats()
|
||||
expect(stats.dedupSavings).toBe(2000) // 2 duplicates × 1000 bytes
|
||||
expect(hash2).toBe(hash1)
|
||||
expect(hash3).toBe(hash1)
|
||||
const metadata = await blobStorage.getMetadata(hash1)
|
||||
expect(metadata?.refCount).toBe(3)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -143,15 +147,12 @@ describe('BlobStorage', () => {
|
|||
it('should compress large text data with zstd', async () => {
|
||||
const data = Buffer.from('a'.repeat(10000))
|
||||
|
||||
const hash = await blobStorage.write(data, {
|
||||
type: 'metadata',
|
||||
compression: 'zstd'
|
||||
})
|
||||
const hash = await blobStorage.write(data, { compression: 'zstd' })
|
||||
|
||||
const metadata = await blobStorage.getMetadata(hash)
|
||||
|
||||
// zstd may not be available in test environment - falls back to 'none'
|
||||
// This is expected behavior (see BlobStorage initCompression fallback)
|
||||
// This is expected behavior (see BlobStorage.ensureCompressionReady fallback)
|
||||
if (metadata?.compression === 'zstd') {
|
||||
expect(metadata.compressedSize).toBeLessThan(metadata.size)
|
||||
} else {
|
||||
|
|
@ -163,10 +164,7 @@ describe('BlobStorage', () => {
|
|||
it('should decompress zstd data on read', async () => {
|
||||
const originalData = Buffer.from('test data '.repeat(100))
|
||||
|
||||
const hash = await blobStorage.write(originalData, {
|
||||
type: 'metadata',
|
||||
compression: 'zstd'
|
||||
})
|
||||
const hash = await blobStorage.write(originalData, { compression: 'zstd' })
|
||||
|
||||
const retrieved = await blobStorage.read(hash)
|
||||
|
||||
|
|
@ -176,36 +174,30 @@ describe('BlobStorage', () => {
|
|||
it('should not compress small blobs', async () => {
|
||||
const data = Buffer.from('small')
|
||||
|
||||
const hash = await blobStorage.write(data, {
|
||||
compression: 'auto'
|
||||
})
|
||||
const hash = await blobStorage.write(data, { compression: 'auto' })
|
||||
|
||||
const metadata = await blobStorage.getMetadata(hash)
|
||||
|
||||
expect(metadata?.compression).toBe('none')
|
||||
})
|
||||
|
||||
it('should not compress vector data (already dense)', async () => {
|
||||
const vectorData = Buffer.from(new Float32Array([1, 2, 3, 4, 5]))
|
||||
|
||||
const hash = await blobStorage.write(vectorData, {
|
||||
type: 'vector',
|
||||
compression: 'auto'
|
||||
})
|
||||
|
||||
const metadata = await blobStorage.getMetadata(hash)
|
||||
|
||||
expect(metadata?.compression).toBe('none')
|
||||
})
|
||||
|
||||
it('should auto-compress metadata/tree/commit types', async () => {
|
||||
it('should skip compression for already-compressed MIME types in auto mode', async () => {
|
||||
const data = Buffer.from('x'.repeat(5000))
|
||||
|
||||
const hash = await blobStorage.write(data, {
|
||||
type: 'metadata',
|
||||
compression: 'auto'
|
||||
compression: 'auto',
|
||||
mimeType: 'image/jpeg'
|
||||
})
|
||||
|
||||
const metadata = await blobStorage.getMetadata(hash)
|
||||
expect(metadata?.compression).toBe('none')
|
||||
})
|
||||
|
||||
it('should auto-compress large compressible payloads', async () => {
|
||||
const data = Buffer.from('x'.repeat(5000))
|
||||
|
||||
const hash = await blobStorage.write(data, { compression: 'auto' })
|
||||
|
||||
const metadata = await blobStorage.getMetadata(hash)
|
||||
|
||||
// Should compress if zstd is available
|
||||
|
|
@ -216,91 +208,37 @@ describe('BlobStorage', () => {
|
|||
})
|
||||
|
||||
describe('LRU Caching', () => {
|
||||
it('should cache blob on read', async () => {
|
||||
it('should serve identical bytes from cache and from storage', async () => {
|
||||
const data = Buffer.from('cached data')
|
||||
const hash = await blobStorage.write(data)
|
||||
|
||||
// First read (cache miss)
|
||||
await blobStorage.read(hash)
|
||||
// Warm read (write-through cache) and cold read (fresh instance)
|
||||
const warm = await blobStorage.read(hash)
|
||||
const cold = await new BlobStorage(adapter).read(hash)
|
||||
|
||||
// Second read (cache hit)
|
||||
await blobStorage.read(hash)
|
||||
|
||||
const stats = blobStorage.getStats()
|
||||
expect(stats.cacheHits).toBeGreaterThan(0)
|
||||
expect(warm.equals(data)).toBe(true)
|
||||
expect(cold.equals(data)).toBe(true)
|
||||
})
|
||||
|
||||
it('should evict LRU entries when cache is full', async () => {
|
||||
const smallCache = new BlobStorage(adapter, {
|
||||
cacheMaxSize: 100, // Very small cache
|
||||
enableCompression: false
|
||||
})
|
||||
it('should keep serving correct bytes when the cache evicts under pressure', async () => {
|
||||
const smallCache = new BlobStorage(adapter, { cacheMaxSize: 100 })
|
||||
|
||||
// Write blobs that exceed cache size
|
||||
const blob1 = Buffer.from('x'.repeat(50))
|
||||
const blob2 = Buffer.from('y'.repeat(50))
|
||||
const blob3 = Buffer.from('z'.repeat(50))
|
||||
|
||||
const hash1 = await smallCache.write(blob1)
|
||||
const hash2 = await smallCache.write(blob2)
|
||||
const hash3 = await smallCache.write(blob3) // Should evict hash1
|
||||
|
||||
// Read all blobs
|
||||
await smallCache.read(hash1)
|
||||
await smallCache.read(hash2)
|
||||
await smallCache.read(hash3)
|
||||
|
||||
// hash1 should have been evicted, causing cache miss
|
||||
const stats = smallCache.getStats()
|
||||
expect(stats.cacheMisses).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should clear cache on demand', async () => {
|
||||
const data = Buffer.from('test')
|
||||
const hash = await blobStorage.write(data)
|
||||
|
||||
await blobStorage.read(hash) // Cache it
|
||||
|
||||
blobStorage.clearCache()
|
||||
|
||||
await blobStorage.read(hash) // Should be cache miss
|
||||
|
||||
const stats = blobStorage.getStats()
|
||||
expect(stats.cacheMisses).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Batch Operations', () => {
|
||||
it('should write multiple blobs in parallel', async () => {
|
||||
const blobs: Array<[Buffer, any]> = [
|
||||
[Buffer.from('blob1'), undefined],
|
||||
[Buffer.from('blob2'), undefined],
|
||||
[Buffer.from('blob3'), undefined]
|
||||
// Write blobs that exceed cache size (forces LRU eviction)
|
||||
const blobs = [
|
||||
Buffer.from('x'.repeat(50)),
|
||||
Buffer.from('y'.repeat(50)),
|
||||
Buffer.from('z'.repeat(50))
|
||||
]
|
||||
const hashes: string[] = []
|
||||
for (const blob of blobs) {
|
||||
hashes.push(await smallCache.write(blob))
|
||||
}
|
||||
|
||||
const hashes = await blobStorage.writeBatch(blobs)
|
||||
|
||||
expect(hashes).toHaveLength(3)
|
||||
expect(hashes[0]).toHaveLength(64)
|
||||
expect(hashes[1]).toHaveLength(64)
|
||||
expect(hashes[2]).toHaveLength(64)
|
||||
})
|
||||
|
||||
it('should read multiple blobs in parallel', async () => {
|
||||
const data1 = Buffer.from('blob1')
|
||||
const data2 = Buffer.from('blob2')
|
||||
const data3 = Buffer.from('blob3')
|
||||
|
||||
const hash1 = await blobStorage.write(data1)
|
||||
const hash2 = await blobStorage.write(data2)
|
||||
const hash3 = await blobStorage.write(data3)
|
||||
|
||||
const blobs = await blobStorage.readBatch([hash1, hash2, hash3])
|
||||
|
||||
expect(blobs).toHaveLength(3)
|
||||
expect(blobs[0].toString()).toBe('blob1')
|
||||
expect(blobs[1].toString()).toBe('blob2')
|
||||
expect(blobs[2].toString()).toBe('blob3')
|
||||
// Every blob still reads back correctly, evicted or not
|
||||
for (let i = 0; i < blobs.length; i++) {
|
||||
const retrieved = await smallCache.read(hashes[i])
|
||||
expect(retrieved.equals(blobs[i])).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -339,120 +277,28 @@ describe('BlobStorage', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('Garbage Collection', () => {
|
||||
it('should delete unreferenced blobs', async () => {
|
||||
const blob1 = Buffer.from('referenced')
|
||||
const blob2 = Buffer.from('unreferenced')
|
||||
|
||||
const hash1 = await blobStorage.write(blob1)
|
||||
const hash2 = await blobStorage.write(blob2)
|
||||
|
||||
// Manually set refCount to 0 for unreferenced blob
|
||||
// (In real COW usage, refCount tracks actual references from commits/trees)
|
||||
// GC only deletes when refCount === 0 AND not in referenced set
|
||||
const metadata2 = await blobStorage.getMetadata(hash2)
|
||||
if (metadata2) {
|
||||
metadata2.refCount = 0
|
||||
await adapter.put(`blob-meta:${hash2}`, Buffer.from(JSON.stringify(metadata2)))
|
||||
}
|
||||
|
||||
// Mark only hash1 as referenced
|
||||
const referenced = new Set([hash1])
|
||||
|
||||
const deleted = await blobStorage.garbageCollect(referenced)
|
||||
|
||||
expect(deleted).toBeGreaterThan(0)
|
||||
expect(await blobStorage.has(hash1)).toBe(true)
|
||||
expect(await blobStorage.has(hash2)).toBe(false)
|
||||
})
|
||||
|
||||
it('should not delete referenced blobs', async () => {
|
||||
const blob1 = Buffer.from('ref1')
|
||||
const blob2 = Buffer.from('ref2')
|
||||
|
||||
const hash1 = await blobStorage.write(blob1)
|
||||
const hash2 = await blobStorage.write(blob2)
|
||||
|
||||
// Both referenced
|
||||
const referenced = new Set([hash1, hash2])
|
||||
|
||||
const deleted = await blobStorage.garbageCollect(referenced)
|
||||
|
||||
expect(deleted).toBe(0)
|
||||
expect(await blobStorage.has(hash1)).toBe(true)
|
||||
expect(await blobStorage.has(hash2)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Metadata', () => {
|
||||
it('should store blob metadata', async () => {
|
||||
const data = Buffer.from('test')
|
||||
|
||||
const hash = await blobStorage.write(data, {
|
||||
type: 'metadata',
|
||||
compression: 'none'
|
||||
})
|
||||
const hash = await blobStorage.write(data, { compression: 'none' })
|
||||
|
||||
const metadata = await blobStorage.getMetadata(hash)
|
||||
|
||||
expect(metadata?.hash).toBe(hash)
|
||||
expect(metadata?.size).toBe(data.length)
|
||||
expect(metadata?.type).toBe('metadata')
|
||||
expect(metadata?.compression).toBe('none')
|
||||
expect(metadata?.createdAt).toBeGreaterThan(0)
|
||||
expect(metadata?.refCount).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('List Operations', () => {
|
||||
it('should list all blobs', async () => {
|
||||
const hash1 = await blobStorage.write(Buffer.from('blob1'))
|
||||
const hash2 = await blobStorage.write(Buffer.from('blob2'))
|
||||
const hash3 = await blobStorage.write(Buffer.from('blob3'))
|
||||
|
||||
const blobs = await blobStorage.listBlobs()
|
||||
|
||||
expect(blobs).toContain(hash1)
|
||||
expect(blobs).toContain(hash2)
|
||||
expect(blobs).toContain(hash3)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Statistics', () => {
|
||||
it('should track storage statistics', async () => {
|
||||
const data1 = Buffer.from('x'.repeat(1000))
|
||||
const data2 = Buffer.from('y'.repeat(2000))
|
||||
|
||||
await blobStorage.write(data1)
|
||||
await blobStorage.write(data2)
|
||||
|
||||
const stats = blobStorage.getStats()
|
||||
|
||||
expect(stats.totalBlobs).toBe(2)
|
||||
expect(stats.totalSize).toBe(3000)
|
||||
expect(stats.avgBlobSize).toBe(1500)
|
||||
})
|
||||
|
||||
it('should calculate compression ratio', async () => {
|
||||
const data = Buffer.from('a'.repeat(10000))
|
||||
|
||||
await blobStorage.write(data, {
|
||||
type: 'metadata',
|
||||
compression: 'zstd'
|
||||
})
|
||||
|
||||
const stats = blobStorage.getStats()
|
||||
|
||||
// Compression ratio should be > 1 if compressed
|
||||
if (stats.compressedSize < stats.totalSize) {
|
||||
expect(stats.compressionRatio).toBeGreaterThan(1)
|
||||
}
|
||||
it('should return undefined metadata for unknown hashes', async () => {
|
||||
expect(await blobStorage.getMetadata('f'.repeat(64))).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('Error Handling', () => {
|
||||
it('should throw on reading non-existent blob', async () => {
|
||||
// Use 'f' instead of '0' to avoid NULL_HASH sentinel value
|
||||
await expect(
|
||||
blobStorage.read('f'.repeat(64))
|
||||
).rejects.toThrow('Blob metadata not found')
|
||||
|
|
@ -462,14 +308,12 @@ describe('BlobStorage', () => {
|
|||
const data = Buffer.from('test')
|
||||
const hash = await blobStorage.write(data)
|
||||
|
||||
// Clear cache so read() actually checks metadata
|
||||
blobStorage.clearCache()
|
||||
|
||||
// Delete metadata but keep blob
|
||||
// Delete metadata but keep blob bytes
|
||||
await adapter.delete(`blob-meta:${hash}`)
|
||||
|
||||
// Use skipCache to ensure we check metadata
|
||||
await expect(blobStorage.read(hash, { skipCache: true })).rejects.toThrow('metadata not found')
|
||||
// A fresh instance has a cold cache, so read() must consult metadata
|
||||
const coldStore = new BlobStorage(adapter)
|
||||
await expect(coldStore.read(hash)).rejects.toThrow('metadata not found')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -510,10 +354,10 @@ describe('BlobStorage', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('v5.10.0 Regression - Blob Integrity with Wrapped Data', () => {
|
||||
it('should handle wrapped binary data without hash mismatch (v5.10.0 fix)', async () => {
|
||||
describe('Wrapped-binary regression (v5.10.0) - hash verification on unwrapped bytes', () => {
|
||||
it('should handle wrapped binary data without hash mismatch', async () => {
|
||||
// Import TestWrappingAdapter that actually wraps data like production
|
||||
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
|
||||
const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
|
||||
const wrappingAdapter = new TestWrappingAdapter()
|
||||
const testBlobStorage = new BlobStorage(wrappingAdapter)
|
||||
|
||||
|
|
@ -522,20 +366,17 @@ describe('BlobStorage', () => {
|
|||
// Write blob (TestWrappingAdapter wraps as {_binary: true, data: "base64..."})
|
||||
const hash = await testBlobStorage.write(originalData)
|
||||
|
||||
// Clear cache to force re-read from storage (bypasses in-memory cache)
|
||||
testBlobStorage.clearCache()
|
||||
|
||||
// v5.10.0 bug: This would fail with "Blob integrity check failed"
|
||||
// because BlobStorage.read() was hashing the wrapped data instead of unwrapped
|
||||
// v5.10.1 fix: unwrapBinaryData() is called before hash verification
|
||||
const retrieved = await testBlobStorage.read(hash)
|
||||
// Fresh instance = cold cache: read() must fetch + unwrap from storage.
|
||||
// v5.10.0 bug: read() hashed the wrapped bytes instead of the unwrapped
|
||||
// content; the fix runs unwrapBinaryData() before hash verification.
|
||||
const retrieved = await new BlobStorage(wrappingAdapter).read(hash)
|
||||
|
||||
expect(retrieved.equals(originalData)).toBe(true)
|
||||
expect(retrieved.toString()).toBe('test content for v5.10.0 regression test')
|
||||
})
|
||||
|
||||
it('should handle multiple wrapped blobs without integrity errors', async () => {
|
||||
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
|
||||
const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
|
||||
const wrappingAdapter = new TestWrappingAdapter()
|
||||
const testBlobStorage = new BlobStorage(wrappingAdapter)
|
||||
|
||||
|
|
@ -554,19 +395,17 @@ describe('BlobStorage', () => {
|
|||
hashes.push(hash)
|
||||
}
|
||||
|
||||
// Clear cache
|
||||
testBlobStorage.clearCache()
|
||||
|
||||
// Read all blobs (would fail in v5.10.0)
|
||||
// Cold-cache instance forces storage reads
|
||||
const coldStore = new BlobStorage(wrappingAdapter)
|
||||
for (let i = 0; i < hashes.length; i++) {
|
||||
const retrieved = await testBlobStorage.read(hashes[i])
|
||||
const retrieved = await coldStore.read(hashes[i])
|
||||
expect(retrieved.equals(testData[i])).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('should work even if adapter fails to unwrap (defense-in-depth)', async () => {
|
||||
// Create a buggy adapter that doesn't unwrap properly
|
||||
class BuggyAdapter implements COWStorageAdapter {
|
||||
class BuggyAdapter implements BlobStoreAdapter {
|
||||
private storage = new Map<string, any>()
|
||||
|
||||
async get(key: string): Promise<any | undefined> {
|
||||
|
|
@ -601,21 +440,20 @@ describe('BlobStorage', () => {
|
|||
const originalData = Buffer.from('test defense-in-depth')
|
||||
const hash = await testBlobStorage.write(originalData)
|
||||
|
||||
testBlobStorage.clearCache()
|
||||
|
||||
// Even with buggy adapter, BlobStorage.read() should unwrap and verify correctly
|
||||
const retrieved = await testBlobStorage.read(hash)
|
||||
// Even with buggy adapter (and a cold cache), read() should unwrap and
|
||||
// verify correctly
|
||||
const retrieved = await new BlobStorage(buggyAdapter).read(hash)
|
||||
expect(retrieved.equals(originalData)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('v6.2.0 Regression - Key-Based Dispatch (Permanent Fix)', () => {
|
||||
describe('Key-based dispatch regression (v6.2.0)', () => {
|
||||
it('should handle JSON-like compressed data without integrity failures', async () => {
|
||||
// THE KILLER TEST CASE: Data that looks like JSON when compressed
|
||||
// This would fail with v5.10.1 wrapBinaryData() guessing approach
|
||||
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
|
||||
const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
|
||||
const wrappingAdapter = new TestWrappingAdapter()
|
||||
const testBlobStorage = new BlobStorage(wrappingAdapter, { enableCompression: true })
|
||||
const testBlobStorage = new BlobStorage(wrappingAdapter)
|
||||
|
||||
// Create JSON data that will be compressed
|
||||
const jsonData = { nested: { key: 'value', array: [1, 2, 3] }, repeated: 'x'.repeat(1000) }
|
||||
|
|
@ -624,54 +462,23 @@ describe('BlobStorage', () => {
|
|||
// Write blob (compression happens, might create JSON-parseable bytes)
|
||||
const hash = await testBlobStorage.write(originalData)
|
||||
|
||||
// Clear cache to force re-read from storage
|
||||
testBlobStorage.clearCache()
|
||||
|
||||
// v5.10.1 bug: If compressed bytes accidentally parse as JSON,
|
||||
// wrapBinaryData() would store parsed object instead of wrapped binary
|
||||
// On read: JSON.stringify(object) !== original compressed bytes → hash mismatch
|
||||
//
|
||||
// v6.2.0 fix: Key-based dispatch eliminates guessing
|
||||
// 'blob:hash' → Always wrapped as binary, never parsed
|
||||
const retrieved = await testBlobStorage.read(hash)
|
||||
const retrieved = await new BlobStorage(wrappingAdapter).read(hash)
|
||||
|
||||
// Hash MUST match
|
||||
expect(BlobStorage.hash(retrieved)).toBe(hash)
|
||||
expect(retrieved.equals(originalData)).toBe(true)
|
||||
})
|
||||
|
||||
it('should correctly dispatch all key types', async () => {
|
||||
// Verify key-based dispatch works for all COW key patterns
|
||||
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
|
||||
const wrappingAdapter = new TestWrappingAdapter()
|
||||
const testBlobStorage = new BlobStorage(wrappingAdapter)
|
||||
|
||||
// Test binary blob keys
|
||||
const blobData = Buffer.from('binary blob content')
|
||||
const blobHash = await testBlobStorage.write(blobData, { type: 'blob' })
|
||||
testBlobStorage.clearCache()
|
||||
const retrievedBlob = await testBlobStorage.read(blobHash)
|
||||
expect(retrievedBlob.equals(blobData)).toBe(true)
|
||||
|
||||
// Test commit keys
|
||||
const commitData = Buffer.from('commit content')
|
||||
const commitHash = await testBlobStorage.write(commitData, { type: 'commit' })
|
||||
testBlobStorage.clearCache()
|
||||
const retrievedCommit = await testBlobStorage.read(commitHash)
|
||||
expect(retrievedCommit.equals(commitData)).toBe(true)
|
||||
|
||||
// Test tree keys
|
||||
const treeData = Buffer.from('tree content')
|
||||
const treeHash = await testBlobStorage.write(treeData, { type: 'tree' })
|
||||
testBlobStorage.clearCache()
|
||||
const retrievedTree = await testBlobStorage.read(treeHash)
|
||||
expect(retrievedTree.equals(treeData)).toBe(true)
|
||||
})
|
||||
|
||||
it('should handle metadata keys correctly', async () => {
|
||||
// Verify that key-based dispatch correctly handles metadata keys
|
||||
// This test verifies the dispatch logic, not the full read/write cycle
|
||||
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
|
||||
const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
|
||||
const wrappingAdapter = new TestWrappingAdapter()
|
||||
|
||||
// Test metadata key dispatch: should parse as JSON
|
||||
|
|
@ -688,10 +495,10 @@ describe('BlobStorage', () => {
|
|||
expect(retrieved.size).toBe(42)
|
||||
})
|
||||
|
||||
it('should never call wrapBinaryData on write path', async () => {
|
||||
// Verify that baseStorage COW adapter uses key-based dispatch,
|
||||
it('should never parse blob bytes as JSON on the write path', async () => {
|
||||
// Verify that the blob bridge uses key-based dispatch,
|
||||
// NOT wrapBinaryData() guessing
|
||||
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
|
||||
const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
|
||||
const wrappingAdapter = new TestWrappingAdapter()
|
||||
const testBlobStorage = new BlobStorage(wrappingAdapter)
|
||||
|
||||
|
|
@ -699,8 +506,7 @@ describe('BlobStorage', () => {
|
|||
const problematicData = Buffer.from('[{"looks":"like","valid":"json"}]')
|
||||
const hash = await testBlobStorage.write(problematicData)
|
||||
|
||||
testBlobStorage.clearCache()
|
||||
const retrieved = await testBlobStorage.read(hash)
|
||||
const retrieved = await new BlobStorage(wrappingAdapter).read(hash)
|
||||
|
||||
// Should retrieve exact same bytes (no JSON parsing occurred)
|
||||
expect(retrieved.equals(problematicData)).toBe(true)
|
||||
Loading…
Add table
Add a link
Reference in a new issue