feat(storage): add raw binary-blob primitive to every storage adapter
Introduce a first-class binary-blob storage primitive on the StorageAdapter contract and implement it across all storage backends. This stores opaque byte payloads verbatim instead of base64-in-JSON, eliminating the ~33% inflation and full-materialization cost of the JSON envelope. It unblocks zero-copy, mmap-able column-store segments and batch vector I/O at billion scale. New methods (declared abstract on BaseStorageAdapter, the class that implements StorageAdapter, and added to the StorageAdapter interface): saveBinaryBlob(key, data) raw write, atomic on real filesystems loadBinaryBlob(key) exact bytes, or null if absent deleteBinaryBlob(key) idempotent (missing is ignored) getBinaryBlobPath(key) real local fs path where one exists, else null Shared key -> location convention across every adapter: the key's "/"-separated segments nest under a `_blobs/` prefix and are suffixed with `.bin`, e.g. "graph-lsm/source/sstable-123" -> "<root>/_blobs/graph-lsm/source/sstable-123.bin". Blobs are not branch-scoped (COW): they are immutable producer-managed segments. Per-adapter behavior: - FileSystemStorage: writes under <rootDir>/_blobs via tmp+rename; returns the real on-disk path so native code can mmap it directly. Path convention matches the existing MmapFileSystemStorage subclass byte-for-byte. - S3CompatibleStorage / R2Storage / GcsStorage / AzureBlobStorage: put/get/delete raw octet-stream objects; getBinaryBlobPath returns null (remote stores have no local path). - MemoryStorage: defensive-copied Map<string, Buffer>; null path; cleared on clear(). - OPFSStorage: stores raw bytes in the OPFS tree; null path. - HistoricalStorageAdapter: read-only — save/delete throw; load resolves the blob from the historical commit tree; null path. Tests: tests/unit/storage/binaryBlob.test.ts exercises save/load round-trip (byte-identical, incl. non-UTF8 bytes), overwrite, delete-then-load, load-missing, and getBinaryBlobPath behavior for all eight adapters. Cloud adapters run against in-memory client fakes that drive the real adapter code; OPFS runs against an in-memory FileSystem Access API mock; the historical adapter commits a blob into a real COW tree. 59 new tests; full unit suite (1398 tests) green.
This commit is contained in:
parent
547721ae14
commit
298b572671
12 changed files with 1552 additions and 0 deletions
|
|
@ -402,6 +402,122 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Raw binary-blob primitive
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Resolve a blob key to its OPFS path under the shared `_blobs/` prefix, e.g.
|
||||
* `"graph-lsm/source/sstable-123"` →
|
||||
* `["_blobs", "graph-lsm", "source", "sstable-123.bin"]` (returned as segments
|
||||
* to navigate the OPFS directory tree). Mirrors the on-disk convention used by
|
||||
* filesystem storage.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
* @returns Path segments for the blob, including the trailing `.bin` filename.
|
||||
* @private
|
||||
*/
|
||||
private blobPathSegments(key: string): string[] {
|
||||
const parts = key.split('/')
|
||||
const filename = parts.pop()! + '.bin'
|
||||
return ['_blobs', ...parts, filename]
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a raw binary blob in OPFS under `key`, writing the bytes verbatim (no
|
||||
* JSON envelope). Parent directories are created on demand. Overwrites any
|
||||
* existing blob at the same key.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
* @param data - The exact bytes to store.
|
||||
*/
|
||||
public async saveBinaryBlob(key: string, data: Buffer): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const segments = this.blobPathSegments(key)
|
||||
const filename = segments.pop()!
|
||||
|
||||
let currentDir = this.rootDir!
|
||||
for (const dirName of segments) {
|
||||
currentDir = await currentDir.getDirectoryHandle(dirName, { create: true })
|
||||
}
|
||||
|
||||
const fileHandle = await currentDir.getFileHandle(filename, { create: true })
|
||||
const writable = await fileHandle.createWritable()
|
||||
// Write a copy of the exact bytes; Uint8Array view keeps it binary-clean.
|
||||
await writable.write(new Uint8Array(data))
|
||||
await writable.close()
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the raw bytes of the OPFS blob for `key`, or `null` if it does not
|
||||
* exist.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
* @returns The blob bytes, or `null` if absent.
|
||||
*/
|
||||
public async loadBinaryBlob(key: string): Promise<Buffer | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const segments = this.blobPathSegments(key)
|
||||
const filename = segments.pop()!
|
||||
|
||||
let currentDir = this.rootDir!
|
||||
for (const dirName of segments) {
|
||||
currentDir = await currentDir.getDirectoryHandle(dirName)
|
||||
}
|
||||
|
||||
const fileHandle = await currentDir.getFileHandle(filename)
|
||||
const file = await fileHandle.getFile()
|
||||
const arrayBuffer = await file.arrayBuffer()
|
||||
return Buffer.from(arrayBuffer)
|
||||
} catch (error: any) {
|
||||
if (error.name === 'NotFoundError') {
|
||||
return null
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the OPFS blob for `key`. Missing blobs are ignored.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
*/
|
||||
public async deleteBinaryBlob(key: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
try {
|
||||
const segments = this.blobPathSegments(key)
|
||||
const filename = segments.pop()!
|
||||
|
||||
let currentDir = this.rootDir!
|
||||
for (const dirName of segments) {
|
||||
currentDir = await currentDir.getDirectoryHandle(dirName)
|
||||
}
|
||||
|
||||
await currentDir.removeEntry(filename)
|
||||
} catch (error: any) {
|
||||
if (error.name === 'NotFoundError') {
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* OPFS is browser-sandboxed storage with no real local filesystem path to
|
||||
* mmap, so this always returns `null`. Callers must use {@link loadBinaryBlob}
|
||||
* instead.
|
||||
*
|
||||
* @param _key - The blob key (unused).
|
||||
* @returns Always `null`.
|
||||
*/
|
||||
public getBinaryBlobPath(_key: string): string | null {
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* OPFS implementation uses controlled concurrency for file operations
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue