feat(8.0)!: delete fork/branch/commit/history/versions — superseded by the Db API

The COW version-control surface (fork, branches, checkout, commit,
getHistory/streamHistory, asOfCommit, brain.versions) is gone, along with
its subsystems: src/versioning/, the COW object store (CommitLog,
CommitObject, RefManager, TreeObject), HistoricalStorageAdapter, and the
TypeAwareHNSWIndex path it kept alive. The Db API (now/transact/asOf/
with/persist/restore) is the one versioning model in 8.0.

Survivors and replacements:
- BlobStorage survives (the VFS stores file content through it), relocated
  to src/storage/blobStorage.ts with binaryDataCodec.ts; its adapter
  interface is now BlobStoreAdapter, slimmed to the consumed surface
  (write/read/has/delete/getMetadata + MIME-aware compression policy).
- brain.migrate() backup branches are replaced by persist-before-migrate:
  MigrateOptions.backupTo persists a hard-link snapshot of the current
  generation before any transform runs; MigrationResult.backupPath reports
  it, and brain.restore(path) brings it back wholesale.
- CLI: cow.ts (fork/branch/checkout/history/migrate) is replaced by
  snapshot.ts — snapshot <path>, restore <path>, history (tx-log),
  generation.
- New public read API: brain.transactionLog({limit}) exposes the reified
  tx-log (generation/timestamp/meta, newest first) that backs the CLI
  history command; TxLogEntry is exported.

Tests: superseded suites deleted; fork/commit blocks excised from shared
suites; BlobStorage tests relocated + reworked against the slimmed store;
migration tests now prove the backupTo snapshot/restore round trip; new
transactionLog coverage in db-mvcc.
This commit is contained in:
David Snelling 2026-06-10 15:22:47 -07:00
parent 431cd64406
commit 8f93add705
64 changed files with 1444 additions and 16313 deletions

View file

@ -152,9 +152,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
// "graph-lsm/source/sstable-123" "<root>/_blobs/graph-lsm/source/sstable-123.bin"
//
// i.e. the key's "/"-separated segments are nested under a `_blobs/` prefix and
// suffixed with `.bin`. Blobs are deliberately NOT branch-scoped (COW): they
// are immutable, content-addressed segments managed by their producer,
// mirroring how COW metadata under `_cow/` is also kept global.
// suffixed with `.bin`. Blobs are immutable, content-addressed segments
// managed by their producer.
/**
* Persist a raw binary blob under `key`, writing the bytes verbatim (no JSON

View file

@ -912,7 +912,7 @@ export class FileSystemStorage extends BaseStorage {
if (entry.isFile()) {
// Handle multiple compression formats for broad compatibility
// - .json.gz: Standard entity/metadata files (JSON compressed)
// - .gz: COW files (refs, blobs, commits - raw compressed)
// - .gz: Raw compressed payloads (e.g. blob-store binary values)
// - .json: Uncompressed JSON files
if (entry.name.endsWith('.json.gz')) {
// Strip .gz extension and add the .json path
@ -923,7 +923,7 @@ export class FileSystemStorage extends BaseStorage {
seen.add(normalizedPath)
}
} else if (entry.name.endsWith('.gz')) {
// COW files stored as .gz (not .json.gz)
// Raw payloads stored as .gz (not .json.gz)
// Strip .gz extension and return path
const normalizedName = entry.name.slice(0, -3) // Remove .gz
const normalizedPath = path.join(prefix, normalizedName)
@ -1387,12 +1387,10 @@ export class FileSystemStorage extends BaseStorage {
}
}
// Clear the entire branches/ directory (branch-based storage)
// Bug fix: Data is stored in branches/main/entities/, not just entities/
// The branch-based structure was introduced for COW support
const branchesDir = path.join(this.rootDir, 'branches')
if (await this.directoryExists(branchesDir)) {
await removeDirectoryContents(branchesDir)
// Clear the canonical entity/verb data area
const entitiesDir = path.join(this.rootDir, 'entities')
if (await this.directoryExists(entitiesDir)) {
await removeDirectoryContents(entitiesDir)
}
// Remove all files in both system directories
@ -1401,19 +1399,16 @@ export class FileSystemStorage extends BaseStorage {
await removeDirectoryContents(this.indexDir)
}
// Remove COW (copy-on-write) version control data
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
// Must be deleted to fully clear all data including version history
const cowDir = path.join(this.rootDir, '_cow')
if (await this.directoryExists(cowDir)) {
// Delete the entire _cow/ directory (not just contents)
await fs.promises.rm(cowDir, { recursive: true, force: true })
// Remove the content-addressed blob store (VFS file content)
const casDir = path.join(this.rootDir, '_cas')
if (await this.directoryExists(casDir)) {
// Delete the entire _cas/ directory (not just contents)
await fs.promises.rm(casDir, { recursive: true, force: true })
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
// Drop the BlobStorage instance — its LRU cache holds deleted blobs.
// initializeBlobStorage() re-creates it (brain.clear() does this before
// the VFS is rebuilt).
this.blobStorage = undefined
this.commitLog = undefined
}
// Clear the statistics cache
@ -1426,7 +1421,7 @@ export class FileSystemStorage extends BaseStorage {
;(this as any).totalVerbCount = 0
// Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data
// Without this, readCanonicalObject() would return stale cached data
// after clear(), causing "ghost" entities to appear
this.clearWriteCache()
}
@ -1457,17 +1452,6 @@ export class FileSystemStorage extends BaseStorage {
return result
}
/**
* Check if COW has been explicitly disabled via clear()
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker file exists, false otherwise
* @protected
*/
/**
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
/**
* Get information about storage usage and capacity
*/

View file

@ -1,571 +0,0 @@
/**
* Historical Storage Adapter
*
* Provides lazy-loading read-only access to a historical commit state.
* Uses LRU cache to bound memory usage and prevent eager-loading of entire history.
*
* Architecture:
* - Extends BaseStorage to inherit all storage infrastructure
* - Wraps an underlying storage adapter to access commit state
* - Implements lazy-loading with LRU cache (bounded memory)
* - All writes throw read-only errors
* - All reads load from historical commit state on-demand
*
* Usage:
* const historical = new HistoricalStorageAdapter({
* underlyingStorage: brain.storage as BaseStorage,
* commitId: 'abc123...',
* cacheSize: 10000 // LRU cache size
* })
* await historical.init()
*
* Performance:
* - O(1) cache lookups for frequently accessed entities
* - Bounded memory: max cacheSize entities in memory
* - Lazy loading: only loads entities when accessed
* - No eager-loading of entire commit state
*
* Production-ready, billion-scale historical queries
*/
import { BaseStorage } from '../baseStorage.js'
import { CommitLog } from '../cow/CommitLog.js'
import { BlobStorage } from '../cow/BlobStorage.js'
import {
HNSWNoun,
HNSWVerb,
NounMetadata,
VerbMetadata,
StatisticsData
} from '../../coreTypes.js'
/**
* Simple LRU Cache implementation
* Bounds memory usage by evicting least-recently-used items
*/
class LRUCache<T> {
private cache = new Map<string, T>()
private accessOrder: string[] = []
private maxSize: number
constructor(maxSize: number = 10000) {
this.maxSize = maxSize
}
get(key: string): T | undefined {
const value = this.cache.get(key)
if (value !== undefined) {
// Move to end (most recently used)
this.accessOrder = this.accessOrder.filter(k => k !== key)
this.accessOrder.push(key)
}
return value
}
set(key: string, value: T): void {
// Remove if already exists
if (this.cache.has(key)) {
this.accessOrder = this.accessOrder.filter(k => k !== key)
}
// Add to cache
this.cache.set(key, value)
this.accessOrder.push(key)
// Evict oldest if over capacity
if (this.cache.size > this.maxSize) {
const oldest = this.accessOrder.shift()
if (oldest) {
this.cache.delete(oldest)
}
}
}
has(key: string): boolean {
return this.cache.has(key)
}
clear(): void {
this.cache.clear()
this.accessOrder = []
}
get size(): number {
return this.cache.size
}
}
export interface HistoricalStorageAdapterOptions {
/** Underlying storage to access commit state from */
underlyingStorage: BaseStorage
/** Commit ID to load historical state from */
commitId: string
/** Max number of entities to cache (default: 10000) */
cacheSize?: number
/** Branch containing the commit (default: 'main') */
branch?: string
}
/**
* Historical Storage Adapter
*
* Lazy-loading, read-only storage adapter for historical commit state.
* Implements billion-scale time-travel queries with bounded memory.
*/
export class HistoricalStorageAdapter extends BaseStorage {
private underlyingStorage: BaseStorage
private commitId: string
private branch: string
private cacheSize: number
// LRU caches for lazy-loaded entities
private cache: LRUCache<any>
// Historical commit state (loaded lazily) - must match BaseStorage visibility
public commitLog?: CommitLog
public blobStorage?: BlobStorage
constructor(options: HistoricalStorageAdapterOptions) {
super()
this.underlyingStorage = options.underlyingStorage
this.commitId = options.commitId
this.branch = options.branch || 'main'
this.cacheSize = options.cacheSize || 10000
this.cache = new LRUCache(this.cacheSize)
}
/**
* Initialize historical storage adapter
* Loads commit metadata but NOT entity data (lazy loading)
*/
public async init(): Promise<void> {
// Get COW components from underlying storage
// Fixed property names - use public properties without underscore prefix
this.commitLog = this.underlyingStorage.commitLog
this.blobStorage = this.underlyingStorage.blobStorage
if (!this.commitLog || !this.blobStorage) {
throw new Error(
'Historical storage requires underlying storage to have COW enabled. ' +
'Call brain.init() first to initialize COW.'
)
}
// Verify commit exists
const commit = await this.commitLog.getCommit(this.commitId)
if (!commit) {
throw new Error(`Commit not found: ${this.commitId}`)
}
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
}
// ============= Abstract Method Implementations =============
/**
* Read object from historical commit state
* Uses LRU cache to avoid repeated blob reads
*/
protected async readObjectFromPath(path: string): Promise<any | null> {
// Check cache first
if (this.cache.has(path)) {
return this.cache.get(path) || null
}
try {
// Import COW classes
const { CommitObject } = await import('../cow/CommitObject.js')
const { TreeObject } = await import('../cow/TreeObject.js')
const { isNullHash } = await import('../cow/constants.js')
// Read commit
const commit = await CommitObject.read(this.blobStorage!, this.commitId)
if (isNullHash(commit.tree)) {
return null
}
// Read tree
const tree = await TreeObject.read(this.blobStorage!, commit.tree)
// Walk tree to find matching path
for await (const entry of TreeObject.walk(this.blobStorage!, tree)) {
if (entry.type === 'blob' && entry.name === path) {
// Read blob data
const blobData = await this.blobStorage!.read(entry.hash)
const data = JSON.parse(blobData.toString())
// Cache the result
this.cache.set(path, data)
return data
}
}
return null
} catch (error) {
// Path doesn't exist in historical state
return null
}
}
/**
* List objects under path in historical commit state
*/
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
try {
// Import COW classes
const { CommitObject } = await import('../cow/CommitObject.js')
const { TreeObject } = await import('../cow/TreeObject.js')
const { isNullHash } = await import('../cow/constants.js')
// Read commit
const commit = await CommitObject.read(this.blobStorage!, this.commitId)
if (isNullHash(commit.tree)) {
return []
}
// Read tree
const tree = await TreeObject.read(this.blobStorage!, commit.tree)
// Walk tree to find all paths matching prefix
const paths: string[] = []
for await (const entry of TreeObject.walk(this.blobStorage!, tree)) {
if (entry.name.startsWith(prefix)) {
paths.push(entry.name)
}
}
return paths
} catch (error) {
return []
}
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot write to path: ${path}`
)
}
/**
* DELETE BLOCKED: Historical storage is read-only
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot delete path: ${path}`
)
}
// ===========================================================================
// Raw binary-blob primitive (read-only)
// ===========================================================================
/**
* WRITE BLOCKED: Historical storage is read-only.
*
* @param key - The blob key (unused; included for the error message).
* @throws Always historical state is immutable.
*/
public async saveBinaryBlob(key: string, _data: Buffer): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot save binary blob: ${key}`
)
}
/**
* Load the raw bytes of a binary blob from the historical commit state by
* walking the commit tree for the blob entry at `_blobs/<key>.bin` and reading
* its content-addressed bytes verbatim (no JSON decode). Returns `null` if the
* blob was not present in this commit.
*
* @param key - The blob key (same convention as the live adapters).
* @returns The blob bytes as stored at this commit, or `null` if absent.
*/
public async loadBinaryBlob(key: string): Promise<Buffer | null> {
try {
const { CommitObject } = await import('../cow/CommitObject.js')
const { TreeObject } = await import('../cow/TreeObject.js')
const { isNullHash } = await import('../cow/constants.js')
const commit = await CommitObject.read(this.blobStorage!, this.commitId)
if (isNullHash(commit.tree)) {
return null
}
const tree = await TreeObject.read(this.blobStorage!, commit.tree)
const blobName = `_blobs/${key}.bin`
for await (const entry of TreeObject.walk(this.blobStorage!, tree)) {
if (entry.type === 'blob' && entry.name === blobName) {
return await this.blobStorage!.read(entry.hash)
}
}
return null
} catch (error) {
// Blob not present in historical state
return null
}
}
/**
* DELETE BLOCKED: Historical storage is read-only.
*
* @param key - The blob key (unused; included for the error message).
* @throws Always historical state is immutable.
*/
public async deleteBinaryBlob(key: string): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot delete binary blob: ${key}`
)
}
/**
* Historical state lives in the content-addressed commit store, not on the
* local filesystem, so there is no mmap-able path. Always returns `null`.
*
* @param _key - The blob key (unused).
* @returns Always `null`.
*/
public getBinaryBlobPath(_key: string): string | null {
return null
}
/**
* Get storage statistics from historical commit
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
return await this.readObjectFromPath('_system/statistics.json')
}
/**
* WRITE BLOCKED: Cannot save statistics to historical storage
*/
protected async saveStatisticsData(data: StatisticsData): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save statistics.')
}
/**
* Clear cache (does not affect historical data)
*/
public async clear(): Promise<void> {
this.cache.clear()
}
/**
* Get storage status
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
return {
type: 'historical',
used: this.cache.size,
quota: this.cacheSize,
details: {
commitId: this.commitId,
branch: this.branch,
cached: this.cache.size,
maxCache: this.cacheSize,
readOnly: true
}
}
}
/**
* Check if COW has been explicitly disabled via clear()
* No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
* @returns Always false (read-only adapter doesn't manage COW state)
* @protected
*/
/**
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
// ============= Override Write Methods (Read-Only) =============
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveNoun(noun: HNSWNoun): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save noun.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveNounMetadata(id: string, metadata: NounMetadata): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save noun metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteNoun(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete noun.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteNounMetadata(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete noun metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveVerb(verb: HNSWVerb): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save verb.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save verb metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteVerb(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete verb.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteVerbMetadata(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete verb metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveVectorIndexData(nounId: string, hnswData: {
level: number
connections: Record<string, string[]>
}): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save HNSW data.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
maxLevel: number
}): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save HNSW system data.')
}
// ============= Additional Abstract Methods =============
/**
* Get noun vector from historical state
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
return noun?.vector || null
}
/**
* Get HNSW data from historical state
*/
public async getVectorIndexData(nounId: string): Promise<{
level: number
connections: Record<string, string[]>
} | null> {
const path = `_system/hnsw/nodes/${nounId}.json`
return await this.readObjectFromPath(path)
}
/**
* Get HNSW system data from historical state
*/
public async getHNSWSystem(): Promise<{
entryPointId: string | null
maxLevel: number
} | null> {
return await this.readObjectFromPath('_system/hnsw/system.json')
}
/**
* Initialize counts (no-op for historical storage)
* Counts are loaded from historical state metadata
*/
protected async initializeCounts(): Promise<void> {
// No-op: Historical storage doesn't need to initialize counts
// They're read from commit state metadata
}
/**
* WRITE BLOCKED: Cannot persist counts to historical storage
*/
protected async persistCounts(): Promise<void> {
// No-op: Historical storage is read-only
}
// ===========================================================================
// Generational record layer (8.0 MVCC) — not applicable to this adapter
//
// HistoricalStorageAdapter is the pre-8.0 commit-based time-travel view and
// is read-only by construction. The generation store never registers its
// write hooks on reader-mode instances, so these methods are unreachable in
// normal operation; they throw explicitly rather than fake behavior.
// ===========================================================================
/**
* WRITE BLOCKED: the tx-log belongs to the live store, not a historical view.
*/
public async appendTxLogLine(_line: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot append to the transaction log.')
}
/**
* Not supported: historical commit views predate the generational tx-log.
*/
public async readTxLogLines(): Promise<string[]> {
throw new Error(
'Transaction-log reads are not supported on a historical commit view. ' +
'Use the live store (or a Db from brain.now()/brain.asOf()).'
)
}
/**
* Not supported: snapshot a live store via db.persist(path) instead.
*/
public async snapshotToDirectory(_targetPath: string): Promise<void> {
throw new Error(
'snapshotToDirectory is not supported on a historical commit view. ' +
'Call db.persist(path) on the live store instead.'
)
}
/**
* WRITE BLOCKED: cannot restore into a read-only historical view.
*/
public async restoreFromDirectory(_sourcePath: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot restore from a snapshot.')
}
}

View file

@ -362,8 +362,13 @@ export class MemoryStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// Drop the BlobStorage instance — its LRU cache holds deleted blobs.
// initializeBlobStorage() re-creates it (brain.clear() does this before
// the VFS is rebuilt).
this.blobStorage = undefined
// Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data
// Without this, readCanonicalObject() would return stale cached data
// after clear(), causing "ghost" entities to appear
this.clearWriteCache()
}
@ -390,17 +395,6 @@ export class MemoryStorage extends BaseStorage {
}
}
/**
* Check if COW has been explicitly disabled via clear()
* No-op for MemoryStorage (doesn't persist)
* @returns Always false (marker doesn't persist in memory)
* @protected
*/
/**
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
/**
* Save statistics data to storage
* @param statistics The statistics data to save