feat: content-type-aware compression policy in COW BlobStorage (2.5.0 #32)
BlobStorage.write() in `auto` compression mode now consults the new `BlobWriteOptions.mimeType` and skips zstd for MIME types known to be already heavily compressed — JPEG, PNG, WebP, MP4, WebM, MP3, ZIP, PDF, Office formats, etc. Gzip/zstd over these formats wastes CPU for no measurable byte savings; the payload entropy is already near maximal, so the output is the same size or slightly larger plus the cost of running the compressor. The denylist `ALREADY_COMPRESSED_MIME_TYPES` is a conservative set of well-known formats. False negatives (compressing something we should have skipped) waste CPU; false positives (skipping something we could have compressed) waste a few percent of bytes. The denylist favours CPU-cycle safety because the formats listed here are the ones where gzip/zstd is reliably a net loss. The policy applies only to `auto` mode. Explicit `'zstd'` and `'none'` are honoured because the caller is asserting the choice. The new `isAlreadyCompressedMimeType()` is exported for consumers that want to make the same decision before calling `write()`. VFS / consumer wiring (pass mimeType from VFS through to BlobStorage) is part of the 2.5.0 #27 storage unification work — when that lands, every media upload through VFS will engage this policy automatically. For now consumers opt-in by passing `mimeType` in BlobWriteOptions. Tests (1447 total, +10): - isAlreadyCompressedMimeType helper: canonical types, case-insensitive + parameter stripping, false-on-missing. - write() auto-mode: image/jpeg + video/mp4 + application/zip all skip compression (metadata.compression === 'none'); text/plain is allowed through to zstd (either 'zstd' or 'none' depending on optional dep). - Read decompresses transparently regardless of write-side decision. - Explicit compression options bypass the policy.
This commit is contained in:
parent
d30eb6f39f
commit
178ff02045
2 changed files with 242 additions and 0 deletions
169
tests/unit/storage/cow/BlobStorage.compression-policy.test.ts
Normal file
169
tests/unit/storage/cow/BlobStorage.compression-policy.test.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
/**
|
||||
* @module BlobStorage.compression-policy.test
|
||||
* @description 2.5.0 #32 — content-type-aware compression policy.
|
||||
*
|
||||
* COW's BlobStorage now consults `BlobWriteOptions.mimeType` in `auto` mode
|
||||
* and skips zstd for MIME types known to be already heavily compressed (JPEG,
|
||||
* PNG, MP4, MP3, ZIP, PDF, etc.). zstd over those formats is reliably a CPU
|
||||
* loss for no measurable byte savings.
|
||||
*
|
||||
* Coverage:
|
||||
* 1. The MIME-type denylist helper recognises common already-compressed
|
||||
* formats (case-insensitive, parameters stripped).
|
||||
* 2. Auto-mode write with `mimeType: 'image/jpeg'` does NOT compress, even
|
||||
* when the payload is highly compressible (a buffer of repeating bytes).
|
||||
* 3. Auto-mode write with `mimeType: 'text/plain'` DOES compress (default
|
||||
* behaviour is preserved for non-listed types).
|
||||
* 4. Auto-mode write with no `mimeType` falls back to the existing per-type
|
||||
* heuristic (default: compress) so the change is non-breaking for
|
||||
* callers that don't pass the new option.
|
||||
* 5. Explicit `compression: 'zstd'` is honoured even for JPEG — the caller
|
||||
* asserts the choice; the policy only applies to `auto`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import {
|
||||
BlobStorage,
|
||||
COWStorageAdapter,
|
||||
isAlreadyCompressedMimeType
|
||||
} from '../../../../src/storage/cow/BlobStorage.js'
|
||||
|
||||
class InMemoryCOWAdapter implements COWStorageAdapter {
|
||||
private store = new Map<string, Buffer>()
|
||||
async get(key: string): Promise<Buffer | undefined> { return this.store.get(key) }
|
||||
async put(key: string, data: Buffer): Promise<void> { this.store.set(key, data) }
|
||||
async delete(key: string): Promise<void> { this.store.delete(key) }
|
||||
async list(prefix: string): Promise<string[]> {
|
||||
const keys: string[] = []
|
||||
for (const key of this.store.keys()) {
|
||||
if (key.startsWith(prefix)) keys.push(key)
|
||||
}
|
||||
return keys.sort()
|
||||
}
|
||||
/** Test helper: peek at the stored bytes for a key. */
|
||||
rawGet(key: string): Buffer | undefined { return this.store.get(key) }
|
||||
}
|
||||
|
||||
describe('BlobStorage compression policy (2.5.0 #32)', () => {
|
||||
let adapter: InMemoryCOWAdapter
|
||||
let blob: BlobStorage
|
||||
|
||||
beforeEach(() => {
|
||||
adapter = new InMemoryCOWAdapter()
|
||||
blob = new BlobStorage(adapter, { enableCompression: true })
|
||||
})
|
||||
|
||||
describe('isAlreadyCompressedMimeType helper', () => {
|
||||
it('recognises the canonical already-compressed types', () => {
|
||||
for (const t of [
|
||||
'image/jpeg', 'image/png', 'image/webp', 'image/gif',
|
||||
'video/mp4', 'video/webm',
|
||||
'audio/mpeg', 'audio/aac',
|
||||
'application/zip', 'application/gzip', 'application/pdf'
|
||||
]) {
|
||||
expect(isAlreadyCompressedMimeType(t)).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('strips parameters and is case-insensitive', () => {
|
||||
expect(isAlreadyCompressedMimeType('IMAGE/JPEG')).toBe(true)
|
||||
expect(isAlreadyCompressedMimeType('image/jpeg; charset=binary')).toBe(true)
|
||||
expect(isAlreadyCompressedMimeType(' Image/PNG ;param=x')).toBe(true)
|
||||
})
|
||||
|
||||
it('returns false for compressible types and missing input', () => {
|
||||
expect(isAlreadyCompressedMimeType('text/plain')).toBe(false)
|
||||
expect(isAlreadyCompressedMimeType('application/json')).toBe(false)
|
||||
expect(isAlreadyCompressedMimeType('text/html')).toBe(false)
|
||||
expect(isAlreadyCompressedMimeType(undefined)).toBe(false)
|
||||
expect(isAlreadyCompressedMimeType('')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('write() auto-mode honours the MIME-type policy', () => {
|
||||
// Highly compressible payload (4KB of one byte) so that *if* zstd is
|
||||
// available the policy decision shows up in the persisted size. The
|
||||
// policy-decision assertions below check `metadata.compression` directly
|
||||
// — that is the source of truth and works regardless of whether the
|
||||
// optional `@mongodb-js/zstd` module is installed in the test env.
|
||||
const HIGHLY_COMPRESSIBLE = Buffer.alloc(4096, 0x41) // 4 KB of 'A'
|
||||
|
||||
it('skips compression for image/jpeg in auto mode (metadata records "none")', async () => {
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'image/jpeg'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
// And the persisted bytes equal the input exactly (no codec wrap).
|
||||
const stored = adapter.rawGet(`blob:${hash}`)
|
||||
expect(stored!.length).toBe(HIGHLY_COMPRESSIBLE.length)
|
||||
})
|
||||
|
||||
it('skips compression for video/mp4 in auto mode', async () => {
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'video/mp4'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
})
|
||||
|
||||
it('skips compression for application/zip in auto mode', async () => {
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'application/zip'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
})
|
||||
|
||||
it('does NOT proactively skip text/plain in auto mode (policy lets it through to zstd)', async () => {
|
||||
// When zstd is installed, metadata.compression === 'zstd' (proof the
|
||||
// policy is gated by mimeType, not blocking everything). When zstd is
|
||||
// NOT installed (the optional `@mongodb-js/zstd` dep), it gracefully
|
||||
// falls back to 'none' — same outcome as a no-zstd install would see
|
||||
// in production. Either way, the policy didn't block on text/plain.
|
||||
const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'text/plain'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression === 'zstd' || meta?.compression === 'none').toBe(true)
|
||||
})
|
||||
|
||||
it('decompresses transparently on read regardless of compression decision', async () => {
|
||||
const jpegHash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'image/jpeg'
|
||||
})
|
||||
const textHash = await blob.write(HIGHLY_COMPRESSIBLE, {
|
||||
compression: 'auto', type: 'blob', mimeType: 'text/plain'
|
||||
})
|
||||
// Different mime-type policies, same bytes back on read.
|
||||
expect((await blob.read(jpegHash)).equals(HIGHLY_COMPRESSIBLE)).toBe(true)
|
||||
expect((await blob.read(textHash)).equals(HIGHLY_COMPRESSIBLE)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('explicit compression options bypass the policy', () => {
|
||||
it('compression: "none" is honoured for text/plain (policy can\'t force compression on)', async () => {
|
||||
const data = Buffer.alloc(4096, 0x43)
|
||||
const hash = await blob.write(data, {
|
||||
compression: 'none', type: 'blob', mimeType: 'text/plain'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
expect(meta?.compression).toBe('none')
|
||||
const stored = adapter.rawGet(`blob:${hash}`)
|
||||
expect(stored!.length).toBe(data.length)
|
||||
})
|
||||
|
||||
it('compression: "zstd" is recorded as the intent for image/jpeg (policy applies only to auto)', async () => {
|
||||
const data = Buffer.alloc(4096, 0x42)
|
||||
const hash = await blob.write(data, {
|
||||
compression: 'zstd', type: 'blob', mimeType: 'image/jpeg'
|
||||
})
|
||||
const meta = await blob.getMetadata(hash)
|
||||
// Explicit zstd → the policy DOESN'T short-circuit; the metadata
|
||||
// records 'zstd' if it actually ran, else 'none' if the optional dep
|
||||
// wasn't installed (matches BlobStorage's "store actual, not intent"
|
||||
// contract). Either way, no auto-mode skip happened.
|
||||
expect(meta?.compression === 'zstd' || meta?.compression === 'none').toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue