feat(storage): add raw binary-blob primitive to every storage adapter
Introduce a first-class binary-blob storage primitive on the StorageAdapter contract and implement it across all storage backends. This stores opaque byte payloads verbatim instead of base64-in-JSON, eliminating the ~33% inflation and full-materialization cost of the JSON envelope. It unblocks zero-copy, mmap-able column-store segments and batch vector I/O at billion scale. New methods (declared abstract on BaseStorageAdapter, the class that implements StorageAdapter, and added to the StorageAdapter interface): saveBinaryBlob(key, data) raw write, atomic on real filesystems loadBinaryBlob(key) exact bytes, or null if absent deleteBinaryBlob(key) idempotent (missing is ignored) getBinaryBlobPath(key) real local fs path where one exists, else null Shared key -> location convention across every adapter: the key's "/"-separated segments nest under a `_blobs/` prefix and are suffixed with `.bin`, e.g. "graph-lsm/source/sstable-123" -> "<root>/_blobs/graph-lsm/source/sstable-123.bin". Blobs are not branch-scoped (COW): they are immutable producer-managed segments. Per-adapter behavior: - FileSystemStorage: writes under <rootDir>/_blobs via tmp+rename; returns the real on-disk path so native code can mmap it directly. Path convention matches the existing MmapFileSystemStorage subclass byte-for-byte. - S3CompatibleStorage / R2Storage / GcsStorage / AzureBlobStorage: put/get/delete raw octet-stream objects; getBinaryBlobPath returns null (remote stores have no local path). - MemoryStorage: defensive-copied Map<string, Buffer>; null path; cleared on clear(). - OPFSStorage: stores raw bytes in the OPFS tree; null path. - HistoricalStorageAdapter: read-only — save/delete throw; load resolves the blob from the historical commit tree; null path. Tests: tests/unit/storage/binaryBlob.test.ts exercises save/load round-trip (byte-identical, incl. non-UTF8 bytes), overwrite, delete-then-load, load-missing, and getBinaryBlobPath behavior for all eight adapters. Cloud adapters run against in-memory client fakes that drive the real adapter code; OPFS runs against an in-memory FileSystem Access API mock; the historical adapter commits a blob into a real COW tree. 59 new tests; full unit suite (1398 tests) green.
This commit is contained in:
parent
547721ae14
commit
298b572671
12 changed files with 1552 additions and 0 deletions
|
|
@ -89,6 +89,10 @@ export class FileSystemStorage extends BaseStorage {
|
|||
private indexDir!: string // Legacy - for backward compatibility
|
||||
private systemDir!: string
|
||||
private lockDir!: string
|
||||
// Root for raw binary blobs (`<rootDir>/_blobs`). Blobs are stored verbatim
|
||||
// (no JSON envelope, no compression) so native code can mmap them directly via
|
||||
// getBinaryBlobPath(). Set in init() once the path module is loaded.
|
||||
private blobsDir!: string
|
||||
private activeLocks: Set<string> = new Set()
|
||||
private lockTimers: Map<string, NodeJS.Timeout> = new Map() // Track timers for cleanup
|
||||
private allTimers: Set<NodeJS.Timeout> = new Set() // Track all timers for cleanup
|
||||
|
|
@ -213,6 +217,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.indexDir = path.join(this.rootDir, 'indexes')
|
||||
this.systemDir = path.join(this.rootDir, SYSTEM_DIR)
|
||||
this.lockDir = path.join(this.rootDir, 'locks')
|
||||
this.blobsDir = path.join(this.rootDir, '_blobs')
|
||||
|
||||
// Create the root directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
|
|
@ -242,6 +247,9 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// Create the locks directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.lockDir)
|
||||
|
||||
// Create the binary blobs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.blobsDir)
|
||||
|
||||
// Initialize count management
|
||||
this.countsFilePath = path.join(this.systemDir, 'counts.json')
|
||||
await this.initializeCounts()
|
||||
|
|
@ -946,6 +954,91 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Raw binary-blob primitive (mmap-friendly)
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Resolve a blob key to its on-disk path under `<rootDir>/_blobs`.
|
||||
*
|
||||
* The key's "/"-separated segments become nested directories and the file is
|
||||
* suffixed with `.bin`, e.g. `"graph-lsm/source/sstable-123"` →
|
||||
* `<rootDir>/_blobs/graph-lsm/source/sstable-123.bin`. This convention is
|
||||
* shared with cortex's `MmapFileSystemStorage` so native code and the TS layer
|
||||
* agree on exactly where each blob lives.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
* @returns The absolute on-disk path for the blob.
|
||||
* @private
|
||||
*/
|
||||
private blobPath(key: string): string {
|
||||
// `path` and `blobsDir` are populated in init(). If a caller resolves a blob
|
||||
// path before init() (e.g. native code probing for a mmap target), fall back
|
||||
// to a POSIX-style join off rootDir so this synchronous method never throws.
|
||||
if (path && this.blobsDir) {
|
||||
return path.join(this.blobsDir, ...key.split('/')) + '.bin'
|
||||
}
|
||||
return `${this.rootDir}/_blobs/${key}.bin`
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a raw binary blob verbatim under `key`, using an atomic
|
||||
* temp-file + rename so concurrent readers never observe a torn write.
|
||||
* Parent directories are created on demand.
|
||||
*
|
||||
* @param key - The blob key (see {@link getBinaryBlobPath} for the convention).
|
||||
* @param data - The exact bytes to store.
|
||||
*/
|
||||
public async saveBinaryBlob(key: string, data: Buffer): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
const filePath = this.blobPath(key)
|
||||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||||
const tmpPath = filePath + '.tmp'
|
||||
await fs.promises.writeFile(tmpPath, data)
|
||||
await fs.promises.rename(tmpPath, filePath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the raw bytes stored under `key`, or `null` if the blob does not exist.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
* @returns The blob bytes, or `null` if absent.
|
||||
*/
|
||||
public async loadBinaryBlob(key: string): Promise<Buffer | null> {
|
||||
await this.ensureInitialized()
|
||||
try {
|
||||
return await fs.promises.readFile(this.blobPath(key))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the blob stored under `key`. Missing blobs are ignored.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
*/
|
||||
public async deleteBinaryBlob(key: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
try {
|
||||
await fs.promises.unlink(this.blobPath(key))
|
||||
} catch {
|
||||
/* ignore missing files */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the real on-disk path for `key` so native code can mmap the file
|
||||
* directly. The path is returned whether or not the file currently exists —
|
||||
* callers are expected to write before mapping.
|
||||
*
|
||||
* @param key - The blob key.
|
||||
* @returns The absolute on-disk path (never `null` for filesystem storage).
|
||||
*/
|
||||
public getBinaryBlobPath(key: string): string | null {
|
||||
return this.blobPath(key)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* FileSystem implementation uses controlled concurrency to prevent too many file reads
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue