soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0)!: delete fork/branch/commit/history/versions — superseded by the Db API

Browse source 
The COW version-control surface (fork, branches, checkout, commit,
getHistory/streamHistory, asOfCommit, brain.versions) is gone, along with
its subsystems: src/versioning/, the COW object store (CommitLog,
CommitObject, RefManager, TreeObject), HistoricalStorageAdapter, and the
TypeAwareHNSWIndex path it kept alive. The Db API (now/transact/asOf/
with/persist/restore) is the one versioning model in 8.0.

Survivors and replacements:
- BlobStorage survives (the VFS stores file content through it), relocated
  to src/storage/blobStorage.ts with binaryDataCodec.ts; its adapter
  interface is now BlobStoreAdapter, slimmed to the consumed surface
  (write/read/has/delete/getMetadata + MIME-aware compression policy).
- brain.migrate() backup branches are replaced by persist-before-migrate:
  MigrateOptions.backupTo persists a hard-link snapshot of the current
  generation before any transform runs; MigrationResult.backupPath reports
  it, and brain.restore(path) brings it back wholesale.
- CLI: cow.ts (fork/branch/checkout/history/migrate) is replaced by
  snapshot.ts — snapshot <path>, restore <path>, history (tx-log),
  generation.
- New public read API: brain.transactionLog({limit}) exposes the reified
  tx-log (generation/timestamp/meta, newest first) that backs the CLI
  history command; TxLogEntry is exported.

Tests: superseded suites deleted; fork/commit blocks excised from shared
suites; BlobStorage tests relocated + reworked against the slimmed store;
migration tests now prove the backupTo snapshot/restore round trip; new
transactionLog coverage in db-mvcc.
This commit is contained in:
David Snelling 2026-06-10 15:22:47 -07:00
parent 431cd64406
commit 8f93add705
64 changed files with 1444 additions and 16313 deletions
Download patch file Download diff file Expand all files Collapse all files

1087
src/brainy.ts
View file

File diff suppressed because it is too large Load diff

465
src/cli/commands/cow.ts
View file

@ -1,465 +0,0 @@
/**
* COW CLI Commands - Copy-on-Write Operations
*
* Fork, branch, merge, and migration operations for instant cloning
*/
import chalk from 'chalk'
import ora from 'ora'
import inquirer from 'inquirer'
import { Brainy } from '../../brainy.js'
import { readFileSync, writeFileSync, existsSync } from 'node:fs'
import { resolve } from 'node:path'
interface CoreOptions {
verbose?: boolean
json?: boolean
pretty?: boolean
}
interface ForkOptions extends CoreOptions {
name?: string
message?: string
author?: string
}
interface MergeOptions extends CoreOptions {
force?: boolean
strategy?: 'last-write-wins' | 'custom'
}
interface MigrateOptions extends CoreOptions {
from?: string
to?: string
backup?: boolean
dryRun?: boolean
}
let brainyInstance: Brainy | null = null
const getBrainy = (): Brainy => {
if (!brainyInstance) {
brainyInstance = new Brainy()
}
return brainyInstance
}
const formatOutput = (data: any, options: CoreOptions): void => {
if (options.json) {
console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data))
}
}
export const cowCommands = {
/**
* Fork the current brain (instant clone)
*/
async fork(name: string | undefined, options: ForkOptions) {
let spinner: any = null
try {
const brain = getBrainy()
await brain.init()
// Interactive mode if no name provided
if (!name) {
const answers = await inquirer.prompt([
{
type: 'input',
name: 'branchName',
message: 'Enter fork/branch name:',
default: `fork-${Date.now()}`,
validate: (input: string) =>
input.trim().length > 0 || 'Branch name cannot be empty'
},
{
type: 'input',
name: 'message',
message: 'Commit message (optional):',
default: 'Fork from main'
}
])
name = answers.branchName
options.message = answers.message
}
spinner = ora(`Forking brain to ${chalk.cyan(name)}...`).start()
const startTime = Date.now()
const fork = await brain.fork(name)
const elapsed = Date.now() - startTime
spinner.succeed(
`Fork created: ${chalk.green(name)} ${chalk.dim(`(${elapsed}ms)`)}`
)
// Show stats
const stats = await fork.getStats()
console.log(`
${chalk.cyan('Fork Statistics:')}
${chalk.dim('Entities:')} ${stats.entities.total || 0}
${chalk.dim('Relationships:')} ${stats.relationships.totalRelationships || 0}
${chalk.dim('Time:')} ${elapsed}ms
${chalk.dim('Storage overhead:')} ~10-20%
`.trim())
if (options.json) {
formatOutput({
branch: name,
time: elapsed,
stats
}, options)
}
} catch (error: any) {
if (spinner) spinner.fail('Fork failed')
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* List all branches/forks
*/
async branchList(options: CoreOptions) {
try {
const brain = getBrainy()
await brain.init()
const branches = await brain.listBranches()
const currentBranch = await brain.getCurrentBranch()
console.log(chalk.cyan('\nBranches:'))
for (const branch of branches) {
const isCurrent = branch === currentBranch
const marker = isCurrent ? chalk.green('*') : ' '
const name = isCurrent ? chalk.green(branch) : branch
// Branch-info enrichment (last-commit timestamp, etc.) requires a
// refManager accessor that brainy hasn't surfaced on the public
// storage adapter yet. CLI shows the bare branch name for now.
console.log(` ${marker} ${name}`)
}
console.log()
if (options.json) {
formatOutput({
branches,
currentBranch
}, options)
}
} catch (error: any) {
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* Switch to a different branch
*/
async checkout(branch: string | undefined, options: CoreOptions) {
let spinner: any = null
try {
const brain = getBrainy()
await brain.init()
// Interactive mode if no branch provided
if (!branch) {
const branches = await brain.listBranches()
const currentBranch = await brain.getCurrentBranch()
const { selected } = await inquirer.prompt([
{
type: 'list',
name: 'selected',
message: 'Select branch:',
choices: branches.map(b => ({
name: b === currentBranch ? `${b} (current)` : b,
value: b
}))
}
])
branch = selected
}
const currentBranch = await brain.getCurrentBranch()
if (branch === currentBranch) {
console.log(chalk.yellow(`Already on branch '${branch}'`))
return
}
spinner = ora(`Switching to ${chalk.cyan(branch)}...`).start()
await brain.checkout(branch!)
spinner.succeed(`Switched to branch ${chalk.green(branch)}`)
if (options.json) {
formatOutput({ branch }, options)
}
} catch (error: any) {
if (spinner) spinner.fail('Checkout failed')
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* Delete a branch/fork
*/
async branchDelete(branch: string | undefined, options: CoreOptions & { force?: boolean }) {
try {
const brain = getBrainy()
await brain.init()
// Interactive mode if no branch provided
if (!branch) {
const branches = await brain.listBranches()
const currentBranch = await brain.getCurrentBranch()
const { selected } = await inquirer.prompt([
{
type: 'list',
name: 'selected',
message: 'Select branch to delete:',
choices: branches
.filter(b => b !== currentBranch) // Can't delete current
.map(b => ({ name: b, value: b }))
}
])
branch = selected
}
// Confirm deletion
if (!options.force) {
const { confirm } = await inquirer.prompt([
{
type: 'confirm',
name: 'confirm',
message: `Delete branch '${branch}'? This cannot be undone.`,
default: false
}
])
if (!confirm) {
console.log(chalk.yellow('Deletion cancelled'))
return
}
}
const spinner = ora(`Deleting branch ${chalk.red(branch)}...`).start()
await brain.deleteBranch(branch!)
spinner.succeed(`Deleted branch ${chalk.red(branch)}`)
if (options.json) {
formatOutput({ deleted: branch }, options)
}
} catch (error: any) {
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* View commit history
*/
async history(options: CoreOptions & { limit?: string }) {
try {
const brain = getBrainy()
await brain.init()
const limit = options.limit ? parseInt(options.limit) : 10
const history = await brain.getHistory({ limit })
console.log(chalk.cyan(`\nCommit History (last ${limit}):\n`))
for (const commit of history) {
const date = new Date(commit.timestamp)
const age = formatAge(Date.now() - commit.timestamp)
console.log(
`${chalk.yellow(commit.hash.substring(0, 8))} ` +
`${chalk.dim(commit.message)} ` +
`${chalk.dim(`by ${commit.author} (${age} ago)`)}`
)
}
console.log()
if (options.json) {
formatOutput(history, options)
}
} catch (error: any) {
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* Migrate storage format (one-time)
*/
async migrate(options: MigrateOptions) {
let spinner: any = null
try {
// Interactive mode if paths not provided
let fromPath = options.from
let toPath = options.to
if (!fromPath || !toPath) {
const answers = await inquirer.prompt([
{
type: 'input',
name: 'from',
message: 'Old Brainy data path (v4.x):',
default: './brainy-data',
when: !fromPath
},
{
type: 'input',
name: 'to',
message: 'New Brainy data path (v5.0.0):',
default: './brainy-data-v5',
when: !toPath
},
{
type: 'confirm',
name: 'backup',
message: 'Create backup before migration?',
default: true,
when: options.backup === undefined
}
])
fromPath = fromPath || answers.from
toPath = toPath || answers.to
options.backup = options.backup ?? answers.backup
}
// Verify old data exists
if (!existsSync(resolve(fromPath!))) {
throw new Error(`Old data path not found: ${fromPath}`)
}
// Create backup if requested
if (options.backup) {
const backupPath = `${fromPath}.backup-${Date.now()}`
spinner = ora(`Creating backup: ${backupPath}...`).start()
const fs = await import('node:fs/promises')
await fs.cp(fromPath, backupPath, { recursive: true, force: false })
spinner.succeed(`Backup created: ${chalk.green(backupPath)}`)
}
if (options.dryRun) {
console.log(chalk.yellow('\n[DRY RUN] Migration plan:'))
console.log(` From: ${fromPath}`)
console.log(` To: ${toPath}`)
console.log(` Backup: ${options.backup ? 'Yes' : 'No'}`)
console.log()
return
}
spinner = ora('Migrating to v5.0.0 COW format...').start()
// Load old brain (v4.x)
const oldBrain = new Brainy({
storage: {
type: 'filesystem',
options: { path: fromPath }
}
})
await oldBrain.init()
// Create new brain
const newBrain = new Brainy({
storage: {
type: 'filesystem',
options: { path: toPath }
}
})
await newBrain.init()
// Migrate all entities
const entities = await oldBrain.find({})
let migrated = 0
spinner.text = `Migrating entities (0/${entities.length})...`
for (const result of entities) {
// Add entity with proper params
await newBrain.add({
type: result.entity.type as any,
data: result.entity.data
})
migrated++
if (migrated % 100 === 0) {
spinner.text = `Migrating entities (${migrated}/${entities.length})...`
}
}
// Create initial commit (will be available after COW integration)
// await newBrain.commit({
// message: `Migrated from v4.x (${entities.length} entities)`,
// author: 'migration-tool'
// })
spinner.succeed(`Migration complete: ${chalk.green(migrated)} entities`)
console.log(`
${chalk.cyan('Migration Summary:')}
${chalk.dim('Old path:')} ${fromPath}
${chalk.dim('New path:')} ${toPath}
${chalk.dim('Entities:')} ${migrated}
${chalk.dim('Format:')} v5.0.0 COW
`.trim())
if (options.json) {
formatOutput({
from: fromPath,
to: toPath,
migrated
}, options)
}
await oldBrain.close()
await newBrain.close()
} catch (error: any) {
if (spinner) spinner.fail('Migration failed')
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
}
}
/**
* Format timestamp age
*/
function formatAge(ms: number): string {
const seconds = Math.floor(ms / 1000)
const minutes = Math.floor(seconds / 60)
const hours = Math.floor(minutes / 60)
const days = Math.floor(hours / 24)
if (days > 0) return `${days}d`
if (hours > 0) return `${hours}h`
if (minutes > 0) return `${minutes}m`
return `${seconds}s`
}

227
src/cli/commands/snapshot.ts Normal file
View file

@ -0,0 +1,227 @@
/**
* @module cli/commands/snapshot
* @description Snapshot & time-travel CLI commands over the 8.0 Db API.
*
* - `snapshot <path>` — persist the current generation as a self-contained,
* hard-link snapshot directory (`brain.now().persist(path)`).
* - `restore <path>` — replace the store's ENTIRE state from a snapshot
* (`brain.restore(path, { confirm: true })`, after an interactive confirm).
* - `history` — the reified transaction log: one line per committed
* `transact()` batch (generation, commit time, metadata).
* - `generation` — the store's current generation watermark.
*/
import chalk from 'chalk'
import ora from 'ora'
import inquirer from 'inquirer'
import { existsSync } from 'node:fs'
import { resolve } from 'node:path'
import { Brainy } from '../../brainy.js'
/**
* @description Shared CLI flags every snapshot command accepts (`--verbose`,
* `--json`, `--pretty`), mirroring the other command modules.
*/
interface SnapshotCliOptions {
verbose?: boolean
json?: boolean
pretty?: boolean
}
let brainyInstance: Brainy | null = null
const getBrainy = (): Brainy => {
if (!brainyInstance) {
brainyInstance = new Brainy()
}
return brainyInstance
}
const formatOutput = (data: unknown, options: SnapshotCliOptions): void => {
if (options.json) {
console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data))
}
}
/**
* @description The `snapshot`/`restore`/`history`/`generation` command
* handlers registered in `src/cli/index.ts`.
*/
export const snapshotCommands = {
/**
* @description Persist the current generation as a self-contained snapshot
* directory. Instant on same-device targets (hard links); the result opens
* with `Brainy.load(path)` or restores wholesale via `brainy restore`.
* @param path - Target directory (created; must be empty or absent).
* @param options - Shared CLI flags.
*/
async snapshot(path: string, options: SnapshotCliOptions) {
let spinner: ReturnType<typeof ora> | null = null
try {
const brain = getBrainy()
await brain.init()
const target = resolve(path)
spinner = ora(`Persisting snapshot to ${chalk.cyan(target)}...`).start()
const startTime = Date.now()
const db = brain.now()
try {
await db.persist(target)
} finally {
await db.release()
}
const elapsed = Date.now() - startTime
spinner.succeed(
`Snapshot persisted: ${chalk.green(target)} ${chalk.dim(`(generation ${db.generation}, ${elapsed}ms)`)}`
)
console.log(`
${chalk.cyan('Snapshot:')}
${chalk.dim('Path:')} ${target}
${chalk.dim('Generation:')} ${db.generation}
${chalk.dim('Open with:')} Brainy.load('${target}')
${chalk.dim('Restore with:')} brainy restore ${target}
`.trim())
formatOutput({ path: target, generation: db.generation, time: elapsed }, options)
} catch (error: any) {
if (spinner) spinner.fail('Snapshot failed')
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* @description Replace the store's ENTIRE current state from a snapshot
* directory. Destructive — asks for interactive confirmation first
* (skipped with `--force`).
* @param path - Snapshot directory produced by `brainy snapshot <path>`.
* @param options - Shared CLI flags plus `--force`.
*/
async restore(path: string, options: SnapshotCliOptions & { force?: boolean }) {
let spinner: ReturnType<typeof ora> | null = null
try {
const target = resolve(path)
if (!existsSync(target)) {
throw new Error(`Snapshot directory not found: ${target}`)
}
if (!options.force) {
const { confirm } = await inquirer.prompt([
{
type: 'confirm',
name: 'confirm',
message: `Replace the store's ENTIRE current state with the snapshot at '${target}'? This cannot be undone.`,
default: false
}
])
if (!confirm) {
console.log(chalk.yellow('Restore cancelled'))
return
}
}
const brain = getBrainy()
await brain.init()
spinner = ora(`Restoring from ${chalk.cyan(target)}...`).start()
const startTime = Date.now()
await brain.restore(target, { confirm: true })
const elapsed = Date.now() - startTime
spinner.succeed(
`Restored from ${chalk.green(target)} ${chalk.dim(`(now at generation ${brain.generation()}, ${elapsed}ms)`)}`
)
formatOutput({ path: target, generation: brain.generation(), time: elapsed }, options)
} catch (error: any) {
if (spinner) spinner.fail('Restore failed')
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* @description Show the reified transaction log, newest first: one entry
* per committed `transact()` batch with its generation, commit time, and
* the `meta` the transaction was submitted with.
* @param options - Shared CLI flags plus `--limit`.
*/
async history(options: SnapshotCliOptions & { limit?: string }) {
try {
const brain = getBrainy()
await brain.init()
const limit = options.limit ? parseInt(options.limit, 10) : 10
const entries = await brain.transactionLog({ limit })
if (entries.length === 0) {
console.log(chalk.yellow('\nNo committed transactions yet (the log records transact() batches).\n'))
formatOutput([], options)
return
}
console.log(chalk.cyan(`\nTransaction Log (last ${entries.length}):\n`))
for (const entry of entries) {
const age = formatAge(Date.now() - entry.timestamp)
const meta = entry.meta ? ` ${chalk.dim(JSON.stringify(entry.meta))}` : ''
console.log(
`${chalk.yellow(`g${entry.generation}`)} ` +
`${chalk.dim(new Date(entry.timestamp).toISOString())} ` +
`${chalk.dim(`(${age} ago)`)}` +
meta
)
}
console.log()
formatOutput(entries, options)
} catch (error: any) {
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
},
/**
* @description Print the store's current generation watermark (bumped once
* per committed `transact()` batch and once per single-operation write
* batch).
* @param options - Shared CLI flags.
*/
async generation(options: SnapshotCliOptions) {
try {
const brain = getBrainy()
await brain.init()
const generation = brain.generation()
console.log(`${chalk.cyan('Generation:')} ${chalk.green(String(generation))}`)
formatOutput({ generation }, options)
} catch (error: any) {
console.error(chalk.red('Error:'), error.message)
if (options.verbose) console.error(error)
process.exit(1)
}
}
}
/**
* @description Format a millisecond duration as a compact age string
* (`3d` / `5h` / `12m` / `42s`).
* @param ms - Elapsed milliseconds.
* @returns The compact age string.
*/
function formatAge(ms: number): string {
const seconds = Math.floor(ms / 1000)
const minutes = Math.floor(seconds / 60)
const hours = Math.floor(minutes / 60)
const days = Math.floor(hours / 24)
if (days > 0) return `${days}d`
if (hours > 0) return `${hours}h`
if (minutes > 0) return `${minutes}m`
return `${seconds}s`
}

56
src/cli/index.ts
View file

@ -17,7 +17,7 @@ import { storageCommands } from './commands/storage.js'
import { nlpCommands } from './commands/nlp.js' import { nlpCommands } from './commands/nlp.js'
import { insightsCommands } from './commands/insights.js' import { insightsCommands } from './commands/insights.js'
import { importCommands } from './commands/import.js' import { importCommands } from './commands/import.js'
import { cowCommands } from './commands/cow.js' import { snapshotCommands } from './commands/snapshot.js'
import { inspectCommands } from './commands/inspect.js' import { inspectCommands } from './commands/inspect.js'
import { readFileSync } from 'fs' import { readFileSync } from 'fs'
import { fileURLToPath } from 'url' import { fileURLToPath } from 'url'
@ -760,58 +760,30 @@ program
.option('--iterations <n>', 'Number of iterations', '100') .option('--iterations <n>', 'Number of iterations', '100')
.action(utilityCommands.benchmark) .action(utilityCommands.benchmark)
// ===== COW Commands - Instant Fork & Branching ===== // ===== Snapshot & Time-Travel Commands - Db API =====
program program
.command('fork [name]') .command('snapshot <path>')
.description('🚀 Fork the brain (instant clone in 1-2 seconds)') .description('📸 Persist the current generation as a self-contained snapshot (instant hard links)')
.option('--message <msg>', 'Commit message') .action(snapshotCommands.snapshot)
.option('--author <name>', 'Author name')
.action(cowCommands.fork)
program program
.command('branch') .command('restore <path>')
.description('🌿 Branch management') .description('Replace the ENTIRE store state from a snapshot (asks for confirmation)')
.addCommand(
new Command('list')
.alias('ls')
.description('List all branches/forks')
.action((options) => {
cowCommands.branchList(options)
})
)
.addCommand(
new Command('delete')
.alias('rm')
.argument('[name]', 'Branch name to delete')
.description('Delete a branch/fork')
.option('-f, --force', 'Skip confirmation') .option('-f, --force', 'Skip confirmation')
.action((name, options) => { .action(snapshotCommands.restore)
cowCommands.branchDelete(name, options)
})
)
program
.command('checkout [branch]')
.alias('co')
.description('Switch to a different branch')
.action(cowCommands.checkout)
program program
.command('history') .command('history')
.alias('log') .alias('log')
.description('Show commit history') .description('Show the transaction log (one entry per committed transact() batch)')
.option('-l, --limit <number>', 'Number of commits to show', '10') .option('-l, --limit <number>', 'Number of entries to show', '10')
.action(cowCommands.history) .action(snapshotCommands.history)
program program
.command('migrate') .command('generation')
.description('🔄 Migrate from v4.x to v5.0.0 (one-time)') .description('Show the store\'s current generation watermark')
.option('--from <path>', 'Old Brainy data path (v4.x)') .action(snapshotCommands.generation)
.option('--to <path>', 'New Brainy data path (v5.0.0)')
.option('--backup', 'Create backup before migration')
.option('--dry-run', 'Show migration plan without executing')
.action(cowCommands.migrate)
// ===== Interactive Mode ===== // ===== Interactive Mode =====

27
src/db/generationStore.ts
View file

@ -621,8 +621,27 @@ export class GenerationStore {
async resolveTimestamp( async resolveTimestamp(
timestampMs: number timestampMs: number
): Promise<{ generation: number; entry: TxLogEntry | null }> { ): Promise<{ generation: number; entry: TxLogEntry | null }> {
const lines = await this.storage.readTxLogLines()
let best: TxLogEntry | null = null let best: TxLogEntry | null = null
for (const entry of await this.txLog()) {
if (entry.timestamp <= timestampMs) {
if (best === null || entry.generation > best.generation) best = entry
}
}
return { generation: best?.generation ?? 0, entry: best }
}
/**
* @description Read all committed tx-log entries, oldest first. Torn
* trailing lines from a crashed append are tolerated and skipped, and
* entries beyond the committed watermark are excluded — the tx-log is
* advisory metadata; the manifest rename is the source of commit truth.
* Compaction never rewrites the log, so entries may reference generations
* whose record-sets were already reclaimed.
* @returns Every committed {@link TxLogEntry}, in commit order.
*/
async txLog(): Promise<TxLogEntry[]> {
const lines = await this.storage.readTxLogLines()
const entries: TxLogEntry[] = []
for (const line of lines) { for (const line of lines) {
let entry: TxLogEntry let entry: TxLogEntry
try { try {
@ -630,11 +649,9 @@ export class GenerationStore {
} catch { } catch {
continue // tolerate a torn trailing line from a crashed append continue // tolerate a torn trailing line from a crashed append
} }
if (entry.timestamp <= timestampMs && entry.generation <= this.committed) { if (entry.generation <= this.committed) entries.push(entry)
if (best === null || entry.generation > best.generation) best = entry
} }
} return entries
return { generation: best?.generation ?? 0, entry: best }
} }
/** /**

107
src/hnsw/hnswIndex.ts
View file

@ -59,10 +59,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
// save, so the migration converges under live traffic with no big-bang. // save, so the migration converges under live traffic with no big-bang.
private connectionsCodec: ConnectionsCodec | null = null private connectionsCodec: ConnectionsCodec | null = null
// COW (Copy-on-Write) support
private cowEnabled: boolean = false
private cowModifiedNodes: Set<string> = new Set()
private cowParent: JsHnswVectorIndex | null = null
// Deferred HNSW persistence for cloud storage performance // Deferred HNSW persistence for cloud storage performance
// In deferred mode, HNSW connections are only persisted on flush/close // In deferred mode, HNSW connections are only persisted on flush/close
@ -301,99 +297,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
return this.persistMode return this.persistMode
} }
/**
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
*
* Snowflake-style instant fork: O(1) shallow copy of Maps, lazy deep copy on write.
*
* @param parent - Parent HNSW index to copy from
*
* Performance:
* - Fork time: <10ms for 1M+ nodes (just copies Map references)
* - Memory: Shared reads, only modified nodes duplicated (~10-20% overhead)
* - Reads: Same speed as parent (shared data structures)
*
* @example
* ```typescript
* const parent = new JsHnswVectorIndex(config)
* // ... parent has 1M nodes ...
*
* const fork = new JsHnswVectorIndex(config)
* fork.enableCOW(parent) // <10ms - instant!
*
* // Reads share data
* await fork.search(query) // Fast, uses parent's data
*
* // Writes trigger COW
* await fork.addItem(newItem) // Deep copies only modified nodes
* ```
*/
public enableCOW(parent: JsHnswVectorIndex): void {
this.cowEnabled = true
this.cowParent = parent
// Shallow copy Maps - O(1) per Map, just copies references
// All nodes/connections are shared until first write
this.nouns = new Map(parent.nouns)
this.highLevelNodes = new Map()
for (const [level, nodeSet] of parent.highLevelNodes.entries()) {
this.highLevelNodes.set(level, new Set(nodeSet))
}
// Copy scalar values
this.entryPointId = parent.entryPointId
this.maxLevel = parent.maxLevel
this.dimension = parent.dimension
// Share cache (COW at cache level)
this.unifiedCache = parent.unifiedCache
// Share config and distance function
this.config = parent.config
this.distanceFunction = parent.distanceFunction
this.useParallelization = parent.useParallelization
prodLog.info(`HNSW COW enabled: ${parent.nouns.size} nodes shallow copied`)
}
/**
* Ensure node is copied before modification (lazy COW)
*
* Deep copies a node only when first modified. Subsequent modifications
* use the already-copied node.
*
* @param nodeId - Node ID to ensure is copied
* @private
*/
private ensureCOW(nodeId: string): void {
if (!this.cowEnabled) return
if (this.cowModifiedNodes.has(nodeId)) return // Already copied
const original = this.nouns.get(nodeId)
if (!original) return
// Deep copy connections Map (separate Map + Sets for each level)
const connectionsCopy = new Map<number, Set<string>>()
for (const [level, ids] of original.connections.entries()) {
connectionsCopy.set(level, new Set(ids))
}
// Deep copy node
const nodeCopy: HNSWNoun = {
id: original.id,
vector: [...original.vector], // Deep copy vector array
connections: connectionsCopy,
level: original.level,
// Copy SQ8 quantized data if present
quantizedVector: original.quantizedVector ? new Uint8Array(original.quantizedVector) : undefined,
codebookMin: original.codebookMin,
codebookMax: original.codebookMax
}
this.nouns.set(nodeId, nodeCopy)
this.cowModifiedNodes.add(nodeId)
}
/** /**
* Calculate distances between a query vector and multiple vectors in parallel * Calculate distances between a query vector and multiple vectors in parallel
* This is used to optimize performance for search operations * This is used to optimize performance for search operations
@ -602,8 +505,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
continue continue
} }
// COW: Ensure neighbor is copied before modification
this.ensureCOW(neighborId)
noun.connections.get(level)!.add(neighborId) noun.connections.get(level)!.add(neighborId)
@ -965,16 +866,12 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
return false return false
} }
// COW: Ensure node is copied before modification
this.ensureCOW(id)
const noun = this.nouns.get(id)! const noun = this.nouns.get(id)!
// Remove connections to this noun from all neighbors // Remove connections to this noun from all neighbors
for (const [level, connections] of noun.connections.entries()) { for (const [level, connections] of noun.connections.entries()) {
for (const neighborId of connections) { for (const neighborId of connections) {
// COW: Ensure neighbor is copied before modification
this.ensureCOW(neighborId)
const neighbor = this.nouns.get(neighborId) const neighbor = this.nouns.get(neighborId)
if (!neighbor) { if (!neighbor) {
@ -994,8 +891,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
for (const [nounId, otherNoun] of this.nouns.entries()) { for (const [nounId, otherNoun] of this.nouns.entries()) {
if (nounId === id) continue // Skip the noun being removed if (nounId === id) continue // Skip the noun being removed
// COW: Ensure noun is copied before modification
this.ensureCOW(nounId)
for (const [level, connections] of otherNoun.connections.entries()) { for (const [level, connections] of otherNoun.connections.entries()) {
if (connections.has(id)) { if (connections.has(id)) {
@ -1881,8 +1776,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
* Ensure a noun doesn't have too many connections at a given level * Ensure a noun doesn't have too many connections at a given level
*/ */
private async pruneConnections(noun: HNSWNoun, level: number): Promise<void> { private async pruneConnections(noun: HNSWNoun, level: number): Promise<void> {
// COW: Ensure noun is copied before modification
this.ensureCOW(noun.id)
const connections = noun.connections.get(level)! const connections = noun.connections.get(level)!
if (connections.size <= this.config.M) { if (connections.size <= this.config.M) {

763
src/hnsw/typeAwareHNSWIndex.ts
View file

@ -1,763 +0,0 @@
/**
* Type-Aware HNSW Index - Phase 2 Billion-Scale Optimization
*
* Maintains separate HNSW graphs per entity type for massive memory savings:
* - Memory @ 1B scale: PROJECTED 384GB → 50GB (-87% from architectural analysis, not yet benchmarked)
* - Query speed: PROJECTED 10x faster for single-type queries (not yet benchmarked)
* - Storage: Already type-first from Phase 1a
*
* Architecture:
* - One JsHnswVectorIndex per NounType (42 total)
* - Lazy initialization (indexes created on first use)
* - Type routing for optimal performance
* - Falls back to multi-type search when type unknown
*/
import { JsHnswVectorIndex } from './hnswIndex.js'
import {
DistanceFunction,
HNSWConfig,
Vector,
VectorDocument
} from '../coreTypes.js'
import { NounType, NOUN_TYPE_COUNT, TypeUtils } from '../types/graphTypes.js'
import { euclideanDistance } from '../utils/index.js'
import type { BaseStorage } from '../storage/baseStorage.js'
import { prodLog } from '../utils/logger.js'
// Default HNSW parameters (same as JsHnswVectorIndex)
const DEFAULT_CONFIG: HNSWConfig = {
M: 16,
efConstruction: 200,
efSearch: 50,
ml: 16
}
/**
* Type-aware HNSW statistics
*/
export interface TypeAwareHNSWStats {
totalNodes: number
totalMemoryMB: number
typeCount: number
typeStats: Map<NounType, {
nodeCount: number
memoryMB: number
maxLevel: number
entryPointId: string | null
}>
memoryReductionPercent: number
estimatedMonolithicMemoryMB: number
}
/**
* TypeAwareHNSWIndex - Separate HNSW graphs per entity type
*
* Phase 2 of billion-scale optimization roadmap.
* PROJECTED: Reduces HNSW memory by 87% @ billion scale (calculated from architecture, not yet benchmarked).
*/
export class TypeAwareHNSWIndex {
// One HNSW index per noun type (lazy initialization)
private indexes: Map<NounType, JsHnswVectorIndex> = new Map()
// Configuration
private config: HNSWConfig
private distanceFunction: DistanceFunction
private storage: BaseStorage | null
private useParallelization: boolean
private persistMode: 'immediate' | 'deferred'
/**
* Create a new TypeAwareHNSWIndex
*
* @param config HNSW configuration (M, efConstruction, efSearch, ml)
* @param distanceFunction Distance function (default: euclidean)
* @param options Additional options (storage, parallelization, persistMode)
*/
constructor(
config: Partial<HNSWConfig> = {},
distanceFunction: DistanceFunction = euclideanDistance,
options: { useParallelization?: boolean; storage?: BaseStorage; persistMode?: 'immediate' | 'deferred' } = {}
) {
this.config = { ...DEFAULT_CONFIG, ...config }
this.distanceFunction = distanceFunction
this.storage = options.storage || null
this.useParallelization =
options.useParallelization !== undefined
? options.useParallelization
: true
this.persistMode = options.persistMode || 'immediate'
prodLog.info('TypeAwareHNSWIndex initialized (Phase 2: Type-Aware HNSW)')
}
/**
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
*
* Propagates enableCOW() to all underlying type-specific HNSW indexes.
* Each index performs O(1) shallow copy of its own data structures.
*
* @param parent - Parent TypeAwareHNSWIndex to copy from
*/
public enableCOW(parent: TypeAwareHNSWIndex): void {
// Shallow copy indexes Map
this.indexes = new Map(parent.indexes)
// Enable COW on each underlying type-specific index
for (const [type, parentIndex] of parent.indexes.entries()) {
const childIndex = new JsHnswVectorIndex(this.config, this.distanceFunction, {
useParallelization: this.useParallelization,
storage: this.storage || undefined,
persistMode: this.persistMode
})
childIndex.enableCOW(parentIndex)
this.indexes.set(type, childIndex)
}
prodLog.info(`TypeAwareHNSWIndex COW enabled: ${parent.indexes.size} type-specific indexes shallow copied`)
}
/**
* Flush dirty HNSW data to storage for all type-specific indexes
*
* In deferred persistence mode, HNSW connections are tracked as dirty but not
* immediately persisted. Call flush() to persist all pending changes across
* all type-specific indexes.
*
* @returns Total number of nodes flushed across all indexes
*/
public async flush(): Promise<number> {
if (this.indexes.size === 0) {
return 0
}
const flushPromises = Array.from(this.indexes.values()).map(index =>
index.flush()
)
const results = await Promise.all(flushPromises)
const totalFlushed = results.reduce((sum, count) => sum + count, 0)
if (totalFlushed > 0) {
prodLog.info(`[TypeAwareHNSW] Flushed ${totalFlushed} dirty nodes across ${this.indexes.size} type indexes`)
}
return totalFlushed
}
/**
* Get the total number of dirty (unpersisted) nodes across all type-specific indexes
*/
public getDirtyNodeCount(): number {
let total = 0
for (const index of this.indexes.values()) {
total += index.getDirtyNodeCount()
}
return total
}
/**
* Get the current persist mode
*/
public getPersistMode(): 'immediate' | 'deferred' {
return this.persistMode
}
/**
* Switch persist mode at runtime. Propagates to all existing type-specific indexes.
* Used by addMany() to defer persistence during batch operations.
*/
public setPersistMode(mode: 'immediate' | 'deferred'): void {
this.persistMode = mode
for (const index of this.indexes.values()) {
if (typeof (index as any).setPersistMode === 'function') {
(index as any).setPersistMode(mode)
}
}
}
/**
* Get or create HNSW index for a specific type (lazy initialization)
*
* Indexes are created on-demand to save memory.
* Only types with entities get an index.
*
* @param type The noun type
* @returns JsHnswVectorIndex for this type
*/
private getIndexForType(type: NounType): JsHnswVectorIndex {
// Validate type is a valid NounType
const typeIndex = TypeUtils.getNounIndex(type)
if (typeIndex === undefined || typeIndex === null || typeIndex < 0) {
throw new Error(
`Invalid NounType: ${type}. Must be one of the 42 defined types.`
)
}
if (!this.indexes.has(type)) {
prodLog.info(`Creating HNSW index for type: ${type}`)
const index = new JsHnswVectorIndex(this.config, this.distanceFunction, {
useParallelization: this.useParallelization,
storage: this.storage || undefined,
persistMode: this.persistMode
})
this.indexes.set(type, index)
}
const index = this.indexes.get(type)
if (!index) {
throw new Error(
`Unexpected: Index for type ${type} not found after creation`
)
}
return index
}
/**
* Add a vector to the type-aware index
*
* Routes to the correct type's HNSW graph.
*
* @param item Vector document to add
* @param type The noun type (required for routing)
* @returns The item ID
*/
public async addItem(item: VectorDocument, type: NounType): Promise<string> {
if (!item || !item.vector) {
throw new Error(
'Invalid VectorDocument: item or vector is null/undefined'
)
}
if (!type) {
throw new Error('Type is required for type-aware indexing')
}
const index = this.getIndexForType(type)
return await index.addItem(item)
}
/**
* Search for nearest neighbors (type-aware)
*
* **Single-type search** (fast path):
* ```typescript
* await index.search(queryVector, 10, 'person')
* // Searches only person graph (100M nodes instead of 1B)
* ```
*
* **Multi-type search**:
* ```typescript
* await index.search(queryVector, 10, ['person', 'organization'])
* // Searches person + organization, merges results
* ```
*
* **All-types search** (fallback):
* ```typescript
* await index.search(queryVector, 10)
* // Searches all 42 graphs (slower but comprehensive)
* ```
*
* @param queryVector Query vector
* @param k Number of results
* @param type Type or types to search (undefined = all types)
* @param filter Optional filter function
* @returns Array of [id, distance] tuples sorted by distance
*/
public async search(
queryVector: Vector,
k: number = 10,
type?: NounType | NounType[],
filter?: (id: string) => Promise<boolean>,
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
): Promise<Array<[string, number]>> {
// Single-type search (fast path)
if (type && typeof type === 'string') {
const index = this.getIndexForType(type)
return await index.search(queryVector, k, filter, options)
}
// Multi-type search (handle empty array edge case)
if (type && Array.isArray(type) && type.length > 0) {
return await this.searchMultipleTypes(queryVector, k, type, filter, options)
}
// All-types search (slowest path + empty array fallback)
return await this.searchAllTypes(queryVector, k, filter, options)
}
/**
* Search across multiple specific types
*
* @param queryVector Query vector
* @param k Number of results
* @param types Array of types to search
* @param filter Optional filter function
* @returns Merged and sorted results
*/
private async searchMultipleTypes(
queryVector: Vector,
k: number,
types: NounType[],
filter?: (id: string) => Promise<boolean>,
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
): Promise<Array<[string, number]>> {
const allResults: Array<[string, number]> = []
// Search each specified type
for (const type of types) {
if (this.indexes.has(type)) {
const index = this.indexes.get(type)!
const results = await index.search(queryVector, k, filter, options)
allResults.push(...results)
}
}
// Merge and sort by distance
allResults.sort((a, b) => a[1] - b[1])
// Return top k
return allResults.slice(0, k)
}
/**
* Search across all types (fallback for type-agnostic queries)
*
* This is the slowest path, but provides comprehensive results.
* Used when type cannot be inferred from query.
*
* @param queryVector Query vector
* @param k Number of results
* @param filter Optional filter function
* @returns Merged and sorted results from all types
*/
private async searchAllTypes(
queryVector: Vector,
k: number,
filter?: (id: string) => Promise<boolean>,
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
): Promise<Array<[string, number]>> {
const allResults: Array<[string, number]> = []
// Search each type's graph
for (const [type, index] of this.indexes.entries()) {
const results = await index.search(queryVector, k, filter, options)
allResults.push(...results)
}
// Merge and sort by distance
allResults.sort((a, b) => a[1] - b[1])
// Return top k
return allResults.slice(0, k)
}
/**
* Remove an item from the index
*
* @param id Item ID to remove
* @param type The noun type (required for routing)
* @returns True if item was removed, false if not found
*/
public async removeItem(id: string, type: NounType): Promise<boolean> {
const index = this.indexes.get(type)
if (!index) {
return false // Type has no index (no items ever added)
}
return await index.removeItem(id)
}
/**
* Get total number of items across all types
*
* @returns Total item count
*/
public size(): number {
let total = 0
for (const index of this.indexes.values()) {
total += index.size()
}
return total
}
/**
* Get number of items for a specific type
*
* @param type The noun type
* @returns Item count for this type
*/
public sizeForType(type: NounType): number {
const index = this.indexes.get(type)
return index ? index.size() : 0
}
/**
* Clear all indexes
*/
public clear(): void {
for (const index of this.indexes.values()) {
index.clear()
}
this.indexes.clear()
}
/**
* Clear index for a specific type
*
* @param type The noun type to clear
*/
public clearType(type: NounType): void {
const index = this.indexes.get(type)
if (index) {
index.clear()
this.indexes.delete(type)
}
}
/**
* Get configuration
*
* @returns HNSW configuration
*/
public getConfig(): HNSWConfig {
return { ...this.config }
}
/**
* Get distance function
*
* @returns Distance function
*/
public getDistanceFunction(): DistanceFunction {
return this.distanceFunction
}
/**
* Set parallelization (applies to all indexes)
*
* @param useParallelization Whether to use parallelization
*/
public setUseParallelization(useParallelization: boolean): void {
this.useParallelization = useParallelization
for (const index of this.indexes.values()) {
index.setUseParallelization(useParallelization)
}
}
/**
* Get parallelization setting
*
* @returns Whether parallelization is enabled
*/
public getUseParallelization(): boolean {
return this.useParallelization
}
/**
* Rebuild HNSW indexes from storage (type-aware)
*
* CRITICAL: This implementation uses type-filtered pagination to avoid
* loading ALL entities for each type (which would be 42 billion reads @ 1B scale).
*
* Can rebuild all types or specific types.
* Much faster than rebuilding a monolithic index.
*
* @param options Rebuild options
*/
public async rebuild(
options: {
types?: NounType[] // Rebuild specific types (undefined = all types)
batchSize?: number // Entities per batch
onProgress?: (type: NounType, loaded: number, total: number) => void
} = {}
): Promise<void> {
if (!this.storage) {
prodLog.warn('TypeAwareHNSW rebuild skipped: no storage adapter')
return
}
const batchSize = options.batchSize || 1000
// Determine which types to rebuild
const typesToRebuild = options.types || this.getAllNounTypes()
prodLog.info(
`Rebuilding ${typesToRebuild.length} type-aware HNSW indexes from persisted data...`
)
// Clear all indexes we're rebuilding
for (const type of typesToRebuild) {
const index = this.getIndexForType(type)
;(index as any).nouns.clear()
}
// Determine preloading strategy (adaptive caching) for entire dataset
const stats = await this.storage.getStatistics()
const entityCount = stats?.totalNodes || 0
const vectorMemory = entityCount * 1536 // 384 dims × 4 bytes
// Use first index's cache (they all share the same UnifiedCache)
const firstIndex = this.getIndexForType(typesToRebuild[0])
const cacheStats = (firstIndex as any).unifiedCache.getStats()
const availableCache = cacheStats.maxSize * 0.80
const shouldPreload = vectorMemory < availableCache
if (shouldPreload) {
prodLog.info(
`HNSW: Preloading ${entityCount.toLocaleString()} vectors at init ` +
`(${(vectorMemory / 1024 / 1024).toFixed(1)}MB < ${(availableCache / 1024 / 1024).toFixed(1)}MB cache)`
)
} else {
prodLog.info(
`HNSW: Adaptive caching for ${entityCount.toLocaleString()} vectors ` +
`(${(vectorMemory / 1024 / 1024).toFixed(1)}MB > ${(availableCache / 1024 / 1024).toFixed(1)}MB cache) - loading on-demand`
)
}
// Load ALL nouns ONCE and route to correct type indexes
// This is O(N) instead of O(42*N) from the previous parallel approach
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
let hasMore = true
let totalLoaded = 0
const loadedByType = new Map<NounType, number>()
while (hasMore) {
const result: {
items: Array<{ id: string; vector: number[]; type?: NounType }>
hasMore: boolean
nextCursor?: string
totalCount?: number
} = await (this.storage as any).getNounsWithPagination({
limit: batchSize,
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
})
// Route each noun to its type index
for (const nounData of result.items) {
try {
// Use 'type' property from HNSWNounWithMetadata (not 'nounType')
// Previously accessed wrong property, causing N+1 metadata fetches
// getNounsWithPagination already includes type in response
const nounType = nounData.type
// Skip if type not in rebuild list
if (!nounType || !typesToRebuild.includes(nounType as NounType)) {
continue
}
// Get the index for this type
const index = this.getIndexForType(nounType as NounType)
// Load HNSW graph data
const hnswData = await (this.storage as any).getVectorIndexData(nounData.id)
if (!hnswData) {
continue // No HNSW data
}
// Create noun with restored connections
const noun = {
id: nounData.id,
vector: shouldPreload ? nounData.vector : [],
connections: new Map(),
level: hnswData.level
}
// Restore connections from storage
for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
const level = parseInt(levelStr, 10)
noun.connections.set(level, new Set<string>(nounIds as string[]))
}
// Add to type-specific index
;(index as any).nouns.set(nounData.id, noun)
// Track high-level nodes
if (noun.level >= 2 && noun.level <= (index as any).MAX_TRACKED_LEVELS) {
if (!(index as any).highLevelNodes.has(noun.level)) {
;(index as any).highLevelNodes.set(noun.level, new Set())
}
;(index as any).highLevelNodes.get(noun.level).add(nounData.id)
}
// Track progress
loadedByType.set(nounType as NounType, (loadedByType.get(nounType as NounType) || 0) + 1)
totalLoaded++
if (options.onProgress && totalLoaded % 100 === 0) {
options.onProgress(nounType as NounType, loadedByType.get(nounType as NounType) || 0, totalLoaded)
}
} catch (error) {
prodLog.error(`Failed to restore HNSW data for ${nounData.id}:`, error)
}
}
hasMore = result.hasMore
offset += batchSize // Increment offset for next page
// Progress logging
if (totalLoaded % 1000 === 0) {
prodLog.info(`Progress: ${totalLoaded.toLocaleString()} entities loaded...`)
}
}
// Restore entry points for each type
for (const type of typesToRebuild) {
const index = this.getIndexForType(type)
let maxLevel = 0
let entryPointId: string | null = null
for (const [id, noun] of (index as any).nouns.entries()) {
if (entryPointId === null || noun.level > maxLevel) {
maxLevel = noun.level
entryPointId = id
}
}
// Recovery: if still null after loop but nouns exist
if ((index as any).nouns.size > 0 && !entryPointId) {
const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1()
entryPointId = recoveredId
maxLevel = recoveredLevel
}
// Validation: if entry point doesn't exist in loaded nouns
if (entryPointId && !(index as any).nouns.has(entryPointId)) {
const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1()
entryPointId = recoveredId
maxLevel = recoveredLevel
}
;(index as any).entryPointId = entryPointId
;(index as any).maxLevel = maxLevel
const loaded = loadedByType.get(type) || 0
const cacheInfo = shouldPreload ? ' (vectors preloaded)' : ' (adaptive caching)'
prodLog.info(
`✅ Rebuilt ${type} index: ${loaded.toLocaleString()} entities, ` +
`${maxLevel + 1} levels, entry point: ${entryPointId || 'none'}${cacheInfo}`
)
}
prodLog.info(
`✅ TypeAwareHNSW rebuild complete: ${this.size().toLocaleString()} total entities across ${this.indexes.size} types (loaded from persisted graph structure)`
)
}
/**
* Get comprehensive statistics
*
* Shows memory reduction compared to monolithic approach.
*
* @returns Type-aware HNSW statistics
*/
public getStats(): TypeAwareHNSWStats {
const typeStats = new Map<
NounType,
{
nodeCount: number
memoryMB: number
maxLevel: number
entryPointId: string | null
}
>()
let totalNodes = 0
let totalMemoryMB = 0
// Collect stats from each type's index
for (const [type, index] of this.indexes.entries()) {
const cacheStats = index.getCacheStats()
const nodeCount = index.size()
const memoryMB = cacheStats.hnswCache.estimatedMemoryMB
typeStats.set(type, {
nodeCount,
memoryMB,
maxLevel: index.getMaxLevel(),
entryPointId: index.getEntryPointId()
})
totalNodes += nodeCount
totalMemoryMB += memoryMB
}
// Estimate monolithic memory (for comparison)
// Monolithic would use ~384 bytes per entity @ 1B scale
const estimatedMonolithicMemoryMB = (totalNodes * 384) / (1024 * 1024)
// Calculate memory reduction
const memoryReductionPercent =
estimatedMonolithicMemoryMB > 0
? ((estimatedMonolithicMemoryMB - totalMemoryMB) /
estimatedMonolithicMemoryMB) *
100
: 0
return {
totalNodes,
totalMemoryMB: parseFloat(totalMemoryMB.toFixed(2)),
typeCount: this.indexes.size,
typeStats,
memoryReductionPercent: parseFloat(memoryReductionPercent.toFixed(2)),
estimatedMonolithicMemoryMB: parseFloat(
estimatedMonolithicMemoryMB.toFixed(2)
)
}
}
/**
* Get statistics for a specific type
*
* @param type The noun type
* @returns Statistics for this type's index (null if no index)
*/
public getStatsForType(
type: NounType
): {
nodeCount: number
memoryMB: number
maxLevel: number
entryPointId: string | null
cacheStats: any
} | null {
const index = this.indexes.get(type)
if (!index) {
return null
}
const cacheStats = index.getCacheStats()
return {
nodeCount: index.size(),
memoryMB: cacheStats.hnswCache.estimatedMemoryMB,
maxLevel: index.getMaxLevel(),
entryPointId: index.getEntryPointId(),
cacheStats
}
}
/**
* Get all noun types (for iteration)
*
* @returns Array of all noun types
*/
private getAllNounTypes(): NounType[] {
const types: NounType[] = []
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
types.push(TypeUtils.getNounFromIndex(i))
}
return types
}
/**
* Get list of types that have indexes (have entities)
*
* @returns Array of types with indexes
*/
public getActiveTypes(): NounType[] {
return Array.from(this.indexes.keys())
}
}

18
src/index.ts
View file

@ -146,6 +146,7 @@ export type {
TxUnrelateOperation, TxUnrelateOperation,
TransactOptions, TransactOptions,
TransactReceipt, TransactReceipt,
TxLogEntry,
CompactHistoryOptions, CompactHistoryOptions,
CompactHistoryResult, CompactHistoryResult,
ChangedIds ChangedIds
@ -221,23 +222,6 @@ export { MemoryStorage, createStorage }
// FileSystemStorage is exported separately to avoid browser build issues. // FileSystemStorage is exported separately to avoid browser build issues.
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js' export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
// Export COW (Copy-on-Write) infrastructure
import { CommitLog } from './storage/cow/CommitLog.js'
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js'
import { BlobStorage } from './storage/cow/BlobStorage.js'
import { RefManager } from './storage/cow/RefManager.js'
import { TreeObject } from './storage/cow/TreeObject.js'
export {
// COW infrastructure
CommitLog,
CommitObject,
CommitBuilder,
BlobStorage,
RefManager,
TreeObject
}
// Export types // Export types
import type { import type {
Vector, Vector,

5
src/indexes/columnStore/ColumnManifest.ts
View file

@ -2,9 +2,8 @@
* @module columnStore/ColumnManifest * @module columnStore/ColumnManifest
* @description Per-field manifest tracking segment files and field metadata. * @description Per-field manifest tracking segment files and field metadata.
* *
* Stored as JSON at `_column_index/{field}/MANIFEST.json` using workspace-global * Stored as JSON at `_column_index/{field}/MANIFEST.json` using storage-root-
* storage paths (not branch-scoped, same as `_cow/` metadata). Rewritten * relative paths. Rewritten atomically on every flush and compaction.
* atomically on every flush and compaction.
* *
* The manifest is the single source of truth for which segments exist for a * The manifest is the single source of truth for which segments exist for a
* field. On startup, the column store loads each field's manifest and * field. On startup, the column store loads each field's manifest and

92
src/migration/MigrationRunner.ts
View file

@ -192,7 +192,7 @@ export class MigrationRunner {
* Entity-level errors are tracked (not thrown). If maxErrors is exceeded, migration * Entity-level errors are tracked (not thrown). If maxErrors is exceeded, migration
* stops early and returns partial results with errors. * stops early and returns partial results with errors.
*/ */
async run(options?: Pick<MigrateOptions, 'onProgress' | 'maxErrors'>): Promise<Omit<MigrationResult, 'backupBranch'>> { async run(options?: Pick<MigrateOptions, 'onProgress' | 'maxErrors'>): Promise<Omit<MigrationResult, 'backupPath'>> {
const state = await this.getState() const state = await this.getState()
const pending = this.getPendingMigrations(state) const pending = this.getPendingMigrations(state)
@ -259,93 +259,6 @@ export class MigrationRunner {
} }
} }
/**
* Run specific migrations without checking completion state.
* Used for branch iterations where the state on main says "completed"
* but branch-local entities still need transforming.
*
* Safe because transforms are idempotent (return null when already applied).
* Does NOT save migration state — the authoritative state lives on main.
*/
async runMigrations(
migrations: Migration[],
options?: Pick<MigrateOptions, 'onProgress' | 'maxErrors'>
): Promise<Omit<MigrationResult, 'backupBranch'>> {
if (migrations.length === 0) {
return { migrationsApplied: [], entitiesProcessed: 0, entitiesModified: 0, errors: [] }
}
let totalProcessed = 0
let totalModified = 0
const appliedMigrations: string[] = []
const errors: MigrationError[] = []
const maxErrors = options?.maxErrors ?? DEFAULT_MAX_ERRORS
const batchConfig = this.storage.getBatchConfig()
const batchSize = batchConfig.maxBatchSize
const batchDelay = batchConfig.batchDelayMs
for (const migration of migrations) {
if (errors.length >= maxErrors) break
let processed = 0
let modified = 0
if (migration.applies === 'nouns' || migration.applies === 'both') {
const result = await this.migrateNouns(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress)
processed += result.processed
modified += result.modified
}
if (migration.applies === 'verbs' || migration.applies === 'both') {
if (errors.length < maxErrors) {
const result = await this.migrateVerbs(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress)
processed += result.processed
modified += result.modified
}
}
totalProcessed += processed
totalModified += modified
if (modified > 0) {
appliedMigrations.push(migration.id)
}
}
return {
migrationsApplied: appliedMigrations,
entitiesProcessed: totalProcessed,
entitiesModified: totalModified,
errors
}
}
/**
* Clean up old system:backup branches created by previous migrations.
* Identifies backups via ref metadata (not by name prefix).
*/
async cleanupOldBackups(): Promise<void> {
const refManager = this.storage.refManager
if (!refManager) return
const refs = await refManager.listRefs()
const currentBranch = this.storage.currentBranch || 'main'
for (const ref of refs) {
if (
ref.metadata?.type === 'system:backup' &&
ref.name.startsWith('refs/heads/') &&
ref.name !== `refs/heads/${currentBranch}`
) {
const branchName = ref.name.replace('refs/heads/', '')
try {
await refManager.deleteRef(branchName)
} catch {
// Ignore — branch may be current or protected
}
}
}
}
// ─── Private helpers ─────────────────────────────────────────────── // ─── Private helpers ───────────────────────────────────────────────
private async migrateNouns( private async migrateNouns(
@ -524,12 +437,11 @@ export class MigrationRunner {
private async saveResumeState(migrationId: string, offset: number): Promise<void> { private async saveResumeState(migrationId: string, offset: number): Promise<void> {
const state = await this.getState() const state = await this.getState()
const branch = this.storage.currentBranch || 'main'
await this.saveState({ await this.saveState({
completedVersion: state?.completedVersion || '', completedVersion: state?.completedVersion || '',
completedAt: state?.completedAt || 0, completedAt: state?.completedAt || 0,
completedMigrations: state?.completedMigrations || [], completedMigrations: state?.completedMigrations || [],
resumeState: { migrationId, lastProcessedOffset: offset, branch } resumeState: { migrationId, lastProcessedOffset: offset }
}) })
} }

12
src/migration/types.ts
View file

@ -29,7 +29,6 @@ export interface MigrationState {
resumeState?: { resumeState?: {
migrationId: string migrationId: string
lastProcessedOffset: number lastProcessedOffset: number
branch: string
} }
} }
@ -56,8 +55,8 @@ export interface MigrationError {
} }
export interface MigrationResult { export interface MigrationResult {
/** Backup branch name, or null if no changes were needed */ /** Path of the pre-migration snapshot, or null when no `backupTo` was supplied */
backupBranch: string | null backupPath: string | null
/** IDs of migrations that were applied */ /** IDs of migrations that were applied */
migrationsApplied: string[] migrationsApplied: string[]
/** Total entities processed (scanned) */ /** Total entities processed (scanned) */
@ -71,6 +70,13 @@ export interface MigrationResult {
export interface MigrateOptions { export interface MigrateOptions {
/** Preview what would change without writing */ /** Preview what would change without writing */
dryRun?: boolean dryRun?: boolean
/**
* Directory to persist a pre-migration snapshot into (created; must be
* empty or absent). The snapshot is cut via the Db API (hard-link
* snapshot of the current generation) BEFORE any transform runs; restore
* it wholesale with `brain.restore(path, { confirm: true })`.
*/
backupTo?: string
/** Progress callback for long-running migrations */ /** Progress callback for long-running migrations */
onProgress?: (progress: { onProgress?: (progress: {
migrationId: string migrationId: string

5
src/storage/adapters/baseStorageAdapter.ts
View file

@ -152,9 +152,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
// "graph-lsm/source/sstable-123" "<root>/_blobs/graph-lsm/source/sstable-123.bin" // "graph-lsm/source/sstable-123" "<root>/_blobs/graph-lsm/source/sstable-123.bin"
// //
// i.e. the key's "/"-separated segments are nested under a `_blobs/` prefix and // i.e. the key's "/"-separated segments are nested under a `_blobs/` prefix and
// suffixed with `.bin`. Blobs are deliberately NOT branch-scoped (COW): they // suffixed with `.bin`. Blobs are immutable, content-addressed segments
// are immutable, content-addressed segments managed by their producer, // managed by their producer.
// mirroring how COW metadata under `_cow/` is also kept global.
/** /**
* Persist a raw binary blob under `key`, writing the bytes verbatim (no JSON * Persist a raw binary blob under `key`, writing the bytes verbatim (no JSON

46
src/storage/adapters/fileSystemStorage.ts
View file

@ -912,7 +912,7 @@ export class FileSystemStorage extends BaseStorage {
if (entry.isFile()) { if (entry.isFile()) {
// Handle multiple compression formats for broad compatibility // Handle multiple compression formats for broad compatibility
// - .json.gz: Standard entity/metadata files (JSON compressed) // - .json.gz: Standard entity/metadata files (JSON compressed)
// - .gz: COW files (refs, blobs, commits - raw compressed) // - .gz: Raw compressed payloads (e.g. blob-store binary values)
// - .json: Uncompressed JSON files // - .json: Uncompressed JSON files
if (entry.name.endsWith('.json.gz')) { if (entry.name.endsWith('.json.gz')) {
// Strip .gz extension and add the .json path // Strip .gz extension and add the .json path
@ -923,7 +923,7 @@ export class FileSystemStorage extends BaseStorage {
seen.add(normalizedPath) seen.add(normalizedPath)
} }
} else if (entry.name.endsWith('.gz')) { } else if (entry.name.endsWith('.gz')) {
// COW files stored as .gz (not .json.gz) // Raw payloads stored as .gz (not .json.gz)
// Strip .gz extension and return path // Strip .gz extension and return path
const normalizedName = entry.name.slice(0, -3) // Remove .gz const normalizedName = entry.name.slice(0, -3) // Remove .gz
const normalizedPath = path.join(prefix, normalizedName) const normalizedPath = path.join(prefix, normalizedName)
@ -1387,12 +1387,10 @@ export class FileSystemStorage extends BaseStorage {
} }
} }
// Clear the entire branches/ directory (branch-based storage) // Clear the canonical entity/verb data area
// Bug fix: Data is stored in branches/main/entities/, not just entities/ const entitiesDir = path.join(this.rootDir, 'entities')
// The branch-based structure was introduced for COW support if (await this.directoryExists(entitiesDir)) {
const branchesDir = path.join(this.rootDir, 'branches') await removeDirectoryContents(entitiesDir)
if (await this.directoryExists(branchesDir)) {
await removeDirectoryContents(branchesDir)
} }
// Remove all files in both system directories // Remove all files in both system directories
@ -1401,19 +1399,16 @@ export class FileSystemStorage extends BaseStorage {
await removeDirectoryContents(this.indexDir) await removeDirectoryContents(this.indexDir)
} }
// Remove COW (copy-on-write) version control data // Remove the content-addressed blob store (VFS file content)
// This directory stores all git-like versioning data (commits, trees, blobs, refs) const casDir = path.join(this.rootDir, '_cas')
// Must be deleted to fully clear all data including version history if (await this.directoryExists(casDir)) {
const cowDir = path.join(this.rootDir, '_cow') // Delete the entire _cas/ directory (not just contents)
if (await this.directoryExists(cowDir)) { await fs.promises.rm(casDir, { recursive: true, force: true })
// Delete the entire _cow/ directory (not just contents)
await fs.promises.rm(cowDir, { recursive: true, force: true })
// Reset COW managers (but don't disable COW - it's always enabled) // Drop the BlobStorage instance — its LRU cache holds deleted blobs.
// COW will re-initialize automatically on next use // initializeBlobStorage() re-creates it (brain.clear() does this before
this.refManager = undefined // the VFS is rebuilt).
this.blobStorage = undefined this.blobStorage = undefined
this.commitLog = undefined
} }
// Clear the statistics cache // Clear the statistics cache
@ -1426,7 +1421,7 @@ export class FileSystemStorage extends BaseStorage {
;(this as any).totalVerbCount = 0 ;(this as any).totalVerbCount = 0
// Clear write-through cache (inherited from BaseStorage) // Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data // Without this, readCanonicalObject() would return stale cached data
// after clear(), causing "ghost" entities to appear // after clear(), causing "ghost" entities to appear
this.clearWriteCache() this.clearWriteCache()
} }
@ -1457,17 +1452,6 @@ export class FileSystemStorage extends BaseStorage {
return result return result
} }
/**
* Check if COW has been explicitly disabled via clear()
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker file exists, false otherwise
* @protected
*/
/**
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
/** /**
* Get information about storage usage and capacity * Get information about storage usage and capacity
*/ */

571
src/storage/adapters/historicalStorageAdapter.ts
View file

@ -1,571 +0,0 @@
/**
* Historical Storage Adapter
*
* Provides lazy-loading read-only access to a historical commit state.
* Uses LRU cache to bound memory usage and prevent eager-loading of entire history.
*
* Architecture:
* - Extends BaseStorage to inherit all storage infrastructure
* - Wraps an underlying storage adapter to access commit state
* - Implements lazy-loading with LRU cache (bounded memory)
* - All writes throw read-only errors
* - All reads load from historical commit state on-demand
*
* Usage:
* const historical = new HistoricalStorageAdapter({
* underlyingStorage: brain.storage as BaseStorage,
* commitId: 'abc123...',
* cacheSize: 10000 // LRU cache size
* })
* await historical.init()
*
* Performance:
* - O(1) cache lookups for frequently accessed entities
* - Bounded memory: max cacheSize entities in memory
* - Lazy loading: only loads entities when accessed
* - No eager-loading of entire commit state
*
* Production-ready, billion-scale historical queries
*/
import { BaseStorage } from '../baseStorage.js'
import { CommitLog } from '../cow/CommitLog.js'
import { BlobStorage } from '../cow/BlobStorage.js'
import {
HNSWNoun,
HNSWVerb,
NounMetadata,
VerbMetadata,
StatisticsData
} from '../../coreTypes.js'
/**
* Simple LRU Cache implementation
* Bounds memory usage by evicting least-recently-used items
*/
class LRUCache<T> {
private cache = new Map<string, T>()
private accessOrder: string[] = []
private maxSize: number
constructor(maxSize: number = 10000) {
this.maxSize = maxSize
}
get(key: string): T | undefined {
const value = this.cache.get(key)
if (value !== undefined) {
// Move to end (most recently used)
this.accessOrder = this.accessOrder.filter(k => k !== key)
this.accessOrder.push(key)
}
return value
}
set(key: string, value: T): void {
// Remove if already exists
if (this.cache.has(key)) {
this.accessOrder = this.accessOrder.filter(k => k !== key)
}
// Add to cache
this.cache.set(key, value)
this.accessOrder.push(key)
// Evict oldest if over capacity
if (this.cache.size > this.maxSize) {
const oldest = this.accessOrder.shift()
if (oldest) {
this.cache.delete(oldest)
}
}
}
has(key: string): boolean {
return this.cache.has(key)
}
clear(): void {
this.cache.clear()
this.accessOrder = []
}
get size(): number {
return this.cache.size
}
}
export interface HistoricalStorageAdapterOptions {
/** Underlying storage to access commit state from */
underlyingStorage: BaseStorage
/** Commit ID to load historical state from */
commitId: string
/** Max number of entities to cache (default: 10000) */
cacheSize?: number
/** Branch containing the commit (default: 'main') */
branch?: string
}
/**
* Historical Storage Adapter
*
* Lazy-loading, read-only storage adapter for historical commit state.
* Implements billion-scale time-travel queries with bounded memory.
*/
export class HistoricalStorageAdapter extends BaseStorage {
private underlyingStorage: BaseStorage
private commitId: string
private branch: string
private cacheSize: number
// LRU caches for lazy-loaded entities
private cache: LRUCache<any>
// Historical commit state (loaded lazily) - must match BaseStorage visibility
public commitLog?: CommitLog
public blobStorage?: BlobStorage
constructor(options: HistoricalStorageAdapterOptions) {
super()
this.underlyingStorage = options.underlyingStorage
this.commitId = options.commitId
this.branch = options.branch || 'main'
this.cacheSize = options.cacheSize || 10000
this.cache = new LRUCache(this.cacheSize)
}
/**
* Initialize historical storage adapter
* Loads commit metadata but NOT entity data (lazy loading)
*/
public async init(): Promise<void> {
// Get COW components from underlying storage
// Fixed property names - use public properties without underscore prefix
this.commitLog = this.underlyingStorage.commitLog
this.blobStorage = this.underlyingStorage.blobStorage
if (!this.commitLog || !this.blobStorage) {
throw new Error(
'Historical storage requires underlying storage to have COW enabled. ' +
'Call brain.init() first to initialize COW.'
)
}
// Verify commit exists
const commit = await this.commitLog.getCommit(this.commitId)
if (!commit) {
throw new Error(`Commit not found: ${this.commitId}`)
}
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
}
// ============= Abstract Method Implementations =============
/**
* Read object from historical commit state
* Uses LRU cache to avoid repeated blob reads
*/
protected async readObjectFromPath(path: string): Promise<any | null> {
// Check cache first
if (this.cache.has(path)) {
return this.cache.get(path) || null
}
try {
// Import COW classes
const { CommitObject } = await import('../cow/CommitObject.js')
const { TreeObject } = await import('../cow/TreeObject.js')
const { isNullHash } = await import('../cow/constants.js')
// Read commit
const commit = await CommitObject.read(this.blobStorage!, this.commitId)
if (isNullHash(commit.tree)) {
return null
}
// Read tree
const tree = await TreeObject.read(this.blobStorage!, commit.tree)
// Walk tree to find matching path
for await (const entry of TreeObject.walk(this.blobStorage!, tree)) {
if (entry.type === 'blob' && entry.name === path) {
// Read blob data
const blobData = await this.blobStorage!.read(entry.hash)
const data = JSON.parse(blobData.toString())
// Cache the result
this.cache.set(path, data)
return data
}
}
return null
} catch (error) {
// Path doesn't exist in historical state
return null
}
}
/**
* List objects under path in historical commit state
*/
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
try {
// Import COW classes
const { CommitObject } = await import('../cow/CommitObject.js')
const { TreeObject } = await import('../cow/TreeObject.js')
const { isNullHash } = await import('../cow/constants.js')
// Read commit
const commit = await CommitObject.read(this.blobStorage!, this.commitId)
if (isNullHash(commit.tree)) {
return []
}
// Read tree
const tree = await TreeObject.read(this.blobStorage!, commit.tree)
// Walk tree to find all paths matching prefix
const paths: string[] = []
for await (const entry of TreeObject.walk(this.blobStorage!, tree)) {
if (entry.name.startsWith(prefix)) {
paths.push(entry.name)
}
}
return paths
} catch (error) {
return []
}
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot write to path: ${path}`
)
}
/**
* DELETE BLOCKED: Historical storage is read-only
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot delete path: ${path}`
)
}
// ===========================================================================
// Raw binary-blob primitive (read-only)
// ===========================================================================
/**
* WRITE BLOCKED: Historical storage is read-only.
*
* @param key - The blob key (unused; included for the error message).
* @throws Always — historical state is immutable.
*/
public async saveBinaryBlob(key: string, _data: Buffer): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot save binary blob: ${key}`
)
}
/**
* Load the raw bytes of a binary blob from the historical commit state by
* walking the commit tree for the blob entry at `_blobs/<key>.bin` and reading
* its content-addressed bytes verbatim (no JSON decode). Returns `null` if the
* blob was not present in this commit.
*
* @param key - The blob key (same convention as the live adapters).
* @returns The blob bytes as stored at this commit, or `null` if absent.
*/
public async loadBinaryBlob(key: string): Promise<Buffer | null> {
try {
const { CommitObject } = await import('../cow/CommitObject.js')
const { TreeObject } = await import('../cow/TreeObject.js')
const { isNullHash } = await import('../cow/constants.js')
const commit = await CommitObject.read(this.blobStorage!, this.commitId)
if (isNullHash(commit.tree)) {
return null
}
const tree = await TreeObject.read(this.blobStorage!, commit.tree)
const blobName = `_blobs/${key}.bin`
for await (const entry of TreeObject.walk(this.blobStorage!, tree)) {
if (entry.type === 'blob' && entry.name === blobName) {
return await this.blobStorage!.read(entry.hash)
}
}
return null
} catch (error) {
// Blob not present in historical state
return null
}
}
/**
* DELETE BLOCKED: Historical storage is read-only.
*
* @param key - The blob key (unused; included for the error message).
* @throws Always — historical state is immutable.
*/
public async deleteBinaryBlob(key: string): Promise<void> {
throw new Error(
`Historical storage is read-only. Cannot delete binary blob: ${key}`
)
}
/**
* Historical state lives in the content-addressed commit store, not on the
* local filesystem, so there is no mmap-able path. Always returns `null`.
*
* @param _key - The blob key (unused).
* @returns Always `null`.
*/
public getBinaryBlobPath(_key: string): string | null {
return null
}
/**
* Get storage statistics from historical commit
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
return await this.readObjectFromPath('_system/statistics.json')
}
/**
* WRITE BLOCKED: Cannot save statistics to historical storage
*/
protected async saveStatisticsData(data: StatisticsData): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save statistics.')
}
/**
* Clear cache (does not affect historical data)
*/
public async clear(): Promise<void> {
this.cache.clear()
}
/**
* Get storage status
*/
public async getStorageStatus(): Promise<{
type: string
used: number
quota: number | null
details?: Record<string, any>
}> {
return {
type: 'historical',
used: this.cache.size,
quota: this.cacheSize,
details: {
commitId: this.commitId,
branch: this.branch,
cached: this.cache.size,
maxCache: this.cacheSize,
readOnly: true
}
}
}
/**
* Check if COW has been explicitly disabled via clear()
* No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
* @returns Always false (read-only adapter doesn't manage COW state)
* @protected
*/
/**
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
// ============= Override Write Methods (Read-Only) =============
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveNoun(noun: HNSWNoun): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save noun.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveNounMetadata(id: string, metadata: NounMetadata): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save noun metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteNoun(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete noun.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteNounMetadata(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete noun metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveVerb(verb: HNSWVerb): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save verb.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveVerbMetadata(id: string, metadata: VerbMetadata): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save verb metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteVerb(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete verb.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async deleteVerbMetadata(id: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot delete verb metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save metadata.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveVectorIndexData(nounId: string, hnswData: {
level: number
connections: Record<string, string[]>
}): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save HNSW data.')
}
/**
* WRITE BLOCKED: Historical storage is read-only
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
maxLevel: number
}): Promise<void> {
throw new Error('Historical storage is read-only. Cannot save HNSW system data.')
}
// ============= Additional Abstract Methods =============
/**
* Get noun vector from historical state
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
return noun?.vector || null
}
/**
* Get HNSW data from historical state
*/
public async getVectorIndexData(nounId: string): Promise<{
level: number
connections: Record<string, string[]>
} | null> {
const path = `_system/hnsw/nodes/${nounId}.json`
return await this.readObjectFromPath(path)
}
/**
* Get HNSW system data from historical state
*/
public async getHNSWSystem(): Promise<{
entryPointId: string | null
maxLevel: number
} | null> {
return await this.readObjectFromPath('_system/hnsw/system.json')
}
/**
* Initialize counts (no-op for historical storage)
* Counts are loaded from historical state metadata
*/
protected async initializeCounts(): Promise<void> {
// No-op: Historical storage doesn't need to initialize counts
// They're read from commit state metadata
}
/**
* WRITE BLOCKED: Cannot persist counts to historical storage
*/
protected async persistCounts(): Promise<void> {
// No-op: Historical storage is read-only
}
// ===========================================================================
// Generational record layer (8.0 MVCC) — not applicable to this adapter
//
// HistoricalStorageAdapter is the pre-8.0 commit-based time-travel view and
// is read-only by construction. The generation store never registers its
// write hooks on reader-mode instances, so these methods are unreachable in
// normal operation; they throw explicitly rather than fake behavior.
// ===========================================================================
/**
* WRITE BLOCKED: the tx-log belongs to the live store, not a historical view.
*/
public async appendTxLogLine(_line: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot append to the transaction log.')
}
/**
* Not supported: historical commit views predate the generational tx-log.
*/
public async readTxLogLines(): Promise<string[]> {
throw new Error(
'Transaction-log reads are not supported on a historical commit view. ' +
'Use the live store (or a Db from brain.now()/brain.asOf()).'
)
}
/**
* Not supported: snapshot a live store via db.persist(path) instead.
*/
public async snapshotToDirectory(_targetPath: string): Promise<void> {
throw new Error(
'snapshotToDirectory is not supported on a historical commit view. ' +
'Call db.persist(path) on the live store instead.'
)
}
/**
* WRITE BLOCKED: cannot restore into a read-only historical view.
*/
public async restoreFromDirectory(_sourcePath: string): Promise<void> {
throw new Error('Historical storage is read-only. Cannot restore from a snapshot.')
}
}

18
src/storage/adapters/memoryStorage.ts
View file

@ -362,8 +362,13 @@ export class MemoryStorage extends BaseStorage {
this.statisticsCache = null this.statisticsCache = null
this.statisticsModified = false this.statisticsModified = false
// Drop the BlobStorage instance — its LRU cache holds deleted blobs.
// initializeBlobStorage() re-creates it (brain.clear() does this before
// the VFS is rebuilt).
this.blobStorage = undefined
// Clear write-through cache (inherited from BaseStorage) // Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data // Without this, readCanonicalObject() would return stale cached data
// after clear(), causing "ghost" entities to appear // after clear(), causing "ghost" entities to appear
this.clearWriteCache() this.clearWriteCache()
} }
@ -390,17 +395,6 @@ export class MemoryStorage extends BaseStorage {
} }
} }
/**
* Check if COW has been explicitly disabled via clear()
* No-op for MemoryStorage (doesn't persist)
* @returns Always false (marker doesn't persist in memory)
* @protected
*/
/**
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
/** /**
* Save statistics data to storage * Save statistics data to storage
* @param statistics The statistics data to save * @param statistics The statistics data to save

596
src/storage/baseStorage.ts
View file

File diff suppressed because it is too large Load diff

77
src/storage/binaryDataCodec.ts Normal file
View file

@ -0,0 +1,77 @@
/**
* @module storage/binaryDataCodec
* @description Single source of truth for wrapping/unwrapping binary payloads
* stored in JSON-based object storage. Binary bytes (compressed blobs, raw
* buffers) are persisted as `{ _binary: true, data: "<base64>" }` so they can
* travel through adapters whose object primitive is JSON. `unwrapBinaryData`
* reverses that wrapping — and MUST be used everywhere bytes come back out,
* because content hashes are computed over the original bytes, not the
* wrapper.
*
* Used by `BaseStorage`'s blob-store bridge and by `BlobStorage`'s
* defense-in-depth verification path.
*/
/**
* @description Wrapped binary data format used when storing binary data in
* JSON-based storage.
*/
export interface WrappedBinaryData {
_binary: true
data: string // base64-encoded
}
/**
* @description Type guard for the wrapped binary format.
* @param data - Candidate value.
* @returns True when `data` is a `{ _binary: true, data: string }` wrapper.
*/
export function isWrappedBinary(data: any): data is WrappedBinaryData {
return (
typeof data === 'object' &&
data !== null &&
data._binary === true &&
typeof data.data === 'string'
)
}
/**
* @description Unwrap binary data from its JSON wrapper.
*
* Handles:
* - `Buffer` → `Buffer` (pass-through)
* - `{ _binary: true, data: "<base64>" }` → `Buffer` (unwrap)
* - Plain object → `Buffer` (JSON stringify — defensive)
* - String → `Buffer`
*
* @param data - Data to unwrap (Buffer, wrapped object, plain object, or string).
* @returns The unwrapped bytes.
* @throws Error when the value cannot be interpreted as binary data.
*/
export function unwrapBinaryData(data: any): Buffer {
// Case 1: Already a Buffer (no unwrapping needed)
if (Buffer.isBuffer(data)) {
return data
}
// Case 2: Wrapped binary data {_binary: true, data: "base64..."}
if (isWrappedBinary(data)) {
return Buffer.from(data.data, 'base64')
}
// Case 3: Plain object (shouldn't happen for binary blobs, but handle gracefully)
if (typeof data === 'object' && data !== null) {
return Buffer.from(JSON.stringify(data))
}
// Case 4: String (convert to Buffer)
if (typeof data === 'string') {
return Buffer.from(data)
}
// Case 5: Invalid type
throw new Error(
`Invalid data type for unwrap: ${typeof data}. ` +
`Expected Buffer or {_binary: true, data: "base64..."}`
)
}

498
src/storage/blobStorage.ts Normal file
View file

@ -0,0 +1,498 @@
/**
* @module storage/blobStorage
* @description Content-addressed blob store. Backs VFS file content with
* SHA-256 addressing (automatic deduplication), reference counting, zstd
* compression where it pays (MIME-aware: already-compressed media is stored
* raw), and an LRU read cache.
*
* The store persists through a narrow key-value bridge
* ({@link BlobStoreAdapter}) provided by `BaseStorage`, which roots all keys
* under the `_cas/` storage area. Key naming is an explicit type contract:
* `blob:<hash>` keys hold binary bytes, `blob-meta:<hash>` keys hold JSON
* metadata — the key format decides how bytes are encoded, never content
* sniffing.
*/
import { createHash } from 'crypto'
import { unwrapBinaryData } from './binaryDataCodec.js'
/**
* @description Key-value bridge the blob store persists through. Implemented
* by `BaseStorage.initializeBlobStorage()` over the adapter's raw object
* primitives.
*/
export interface BlobStoreAdapter {
/** Read the bytes stored under `key`, or `undefined` when absent. */
get(key: string): Promise<Buffer | undefined>
/** Persist `data` under `key` (overwrites). */
put(key: string, data: Buffer): Promise<void>
/** Delete the value under `key`. Missing keys are ignored. */
delete(key: string): Promise<void>
/** List all keys starting with `prefix`. */
list(prefix: string): Promise<string[]>
}
/**
* @description Metadata persisted alongside each blob (under
* `blob-meta:<hash>`).
*/
export interface BlobMetadata {
/** SHA-256 content hash (the blob's identity). */
hash: string
/** Original (uncompressed) size in bytes. */
size: number
/** Stored size in bytes (after compression, if any). */
compressedSize: number
/** Compression applied to the stored bytes. */
compression: 'none' | 'zstd'
/** Creation timestamp (epoch ms). */
createdAt: number
/** Number of logical references to this blob (deduplicated writes). */
refCount: number
}
/**
* @description Options for {@link BlobStorage.write}.
*/
export interface BlobWriteOptions {
/**
* Compression strategy. `'auto'` (default) compresses payloads above 1 KB
* with zstd unless the MIME type says the bytes are already compressed.
* Explicit `'none'`/`'zstd'` is honoured as asserted by the caller.
*/
compression?: 'none' | 'zstd' | 'auto'
/**
* Content type of the payload (e.g. `image/jpeg`, `video/mp4`).
*
* When set on an `auto`-compression write, the store skips zstd for MIME
* types that are already heavily compressed (JPEG, PNG, WebP, MP4, WebM,
* MP3, ZIP, PDF, etc.). zstd over these formats wastes CPU and rarely
* shaves more than a single-digit percent — usually it actually grows the
* payload because the entropy is already maximised by the format itself.
*
* Has no effect when `compression` is `'none'` or `'zstd'` explicitly —
* the caller is asserting the choice and the store honours it.
*/
mimeType?: string
}
/**
* MIME types whose payload is already heavily compressed. zstd over these is
* almost always a CPU-only loss — the bytes are already near entropy-maximal,
* so the output is the same size or slightly larger plus the cost of running
* the compressor. Used by `BlobStorage.selectCompression()` in `auto` mode.
*
* Conservative denylist (well-known formats only). Anything not in this set
* goes through the size heuristic. False negatives (compressing something we
* should have skipped) waste CPU; false positives (skipping something we
* could have compressed) waste a few percent of bytes. The denylist favours
* CPU-cycle safety because the formats listed here are the ones where
* gzip/zstd is reliably a net loss.
*/
const ALREADY_COMPRESSED_MIME_TYPES = new Set<string>([
// Images
'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp',
'image/avif', 'image/heic', 'image/heif', 'image/jp2',
// Video
'video/mp4', 'video/webm', 'video/x-matroska', 'video/quicktime',
'video/x-msvideo', 'video/mpeg', 'video/3gpp', 'video/x-ms-wmv',
// Audio
'audio/mpeg', 'audio/mp4', 'audio/aac', 'audio/ogg', 'audio/webm',
'audio/opus', 'audio/flac', 'audio/x-ms-wma',
// Archives
'application/zip', 'application/gzip', 'application/x-gzip',
'application/x-bzip2', 'application/x-7z-compressed',
'application/x-rar-compressed', 'application/x-xz', 'application/x-zstd',
'application/x-compress', 'application/vnd.rar',
// Documents with internal compression
'application/pdf', 'application/epub+zip',
// Office formats (zip-based)
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/vnd.oasis.opendocument.text',
'application/vnd.oasis.opendocument.spreadsheet',
'application/vnd.oasis.opendocument.presentation'
])
/**
* @description True when the MIME type names a payload format known to be
* already heavily compressed. Strips any `;charset=…` / `;boundary=…`
* parameters and lowercases the bare type/subtype before lookup, so
* `'IMAGE/JPEG; charset=binary'` and `'image/jpeg'` resolve identically.
* @param mimeType - MIME type string, or `undefined`.
* @returns Whether `auto` compression should skip zstd for this payload.
*/
export function isAlreadyCompressedMimeType(mimeType: string | undefined): boolean {
if (!mimeType) return false
const bare = mimeType.split(';', 1)[0].trim().toLowerCase()
return ALREADY_COMPRESSED_MIME_TYPES.has(bare)
}
/**
* LRU cache entry.
*/
interface CacheEntry {
data: Buffer
metadata: BlobMetadata
lastAccess: number
size: number
}
/**
* @description Content-addressed, deduplicating, reference-counted blob
* store with MIME-aware zstd compression and an LRU read cache. See the
* module doc for the persistence contract.
*
* @example
* const hash = await blobStorage.write(buffer, { mimeType: 'image/png' })
* const bytes = await blobStorage.read(hash) // verified against the hash
* await blobStorage.delete(hash) // decrements refCount first
*/
export class BlobStorage {
private adapter: BlobStoreAdapter
private cache: Map<string, CacheEntry>
private cacheMaxSize: number
private currentCacheSize: number
// Compression (lazily loaded)
private zstdCompress?: (data: Buffer) => Promise<Buffer>
private zstdDecompress?: (data: Buffer) => Promise<Buffer>
private compressionReady = false
// Configuration
private readonly CACHE_MAX_SIZE = 100 * 1024 * 1024 // 100MB default
private readonly COMPRESSION_THRESHOLD = 1024 // 1KB - don't compress smaller
/**
* @param adapter - Key-value bridge to persist through.
* @param options - `cacheMaxSize` bounds the LRU read cache (bytes,
* default 100 MB).
*/
constructor(adapter: BlobStoreAdapter, options?: { cacheMaxSize?: number }) {
this.adapter = adapter
this.cache = new Map()
this.cacheMaxSize = options?.cacheMaxSize ?? this.CACHE_MAX_SIZE
this.currentCacheSize = 0
}
/**
* Lazy-load the zstd compression module. Falls back to uncompressed
* storage when the optional dependency is unavailable.
*/
private async ensureCompressionReady(): Promise<void> {
if (this.compressionReady) return
try {
// Dynamic import to avoid loading if not needed
// @ts-ignore - Optional dependency, gracefully handled if missing
const zstd = await import('@mongodb-js/zstd')
this.zstdCompress = async (data: Buffer) => {
return Buffer.from(await zstd.compress(data, 3)) // Level 3 = fast
}
this.zstdDecompress = async (data: Buffer) => {
return Buffer.from(await zstd.decompress(data))
}
} catch (error) {
console.warn('zstd compression not available, falling back to uncompressed')
this.zstdCompress = undefined
this.zstdDecompress = undefined
}
this.compressionReady = true
}
/**
* @description Compute the SHA-256 content hash of `data`.
* @param data - Bytes to hash.
* @returns Hex-encoded SHA-256 hash.
*/
static hash(data: Buffer): string {
return createHash('sha256').update(data).digest('hex')
}
/**
* @description Write a blob. Content-addressed: the SHA-256 hash of the
* bytes is the storage key, so identical payloads deduplicate (the
* existing blob's reference count is incremented instead of rewriting).
*
* @param data - Blob bytes.
* @param options - Compression strategy and MIME hint (see
* {@link BlobWriteOptions}).
* @returns The blob's SHA-256 hash.
*/
async write(data: Buffer, options: BlobWriteOptions = {}): Promise<string> {
const hash = BlobStorage.hash(data)
// Deduplication: identical content already stored — just add a reference.
if (await this.has(hash)) {
await this.incrementRefCount(hash)
return hash
}
await this.ensureCompressionReady()
// Determine compression strategy
const compression = this.selectCompression(data, options)
// Compress if needed
let finalData = data
let compressedSize = data.length
if (compression === 'zstd' && this.zstdCompress) {
finalData = await this.zstdCompress(data)
compressedSize = finalData.length
}
// Record the ACTUAL compression state, not the intended one — prevents
// corruption if compression failed to initialize.
const actualCompression = finalData === data ? 'none' : compression
const metadata: BlobMetadata = {
hash,
size: data.length,
compressedSize,
compression: actualCompression,
createdAt: Date.now(),
refCount: 1
}
await this.adapter.put(`blob:${hash}`, finalData)
await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata)))
// Write-through cache (caches the ORIGINAL bytes, not the compressed form)
this.addToCache(hash, data, metadata)
return hash
}
/**
* @description Read a blob: LRU cache first, then storage with
* decompression and integrity verification (the bytes are re-hashed and
* compared against the requested hash).
*
* @param hash - The blob's SHA-256 hash.
* @returns The original (decompressed) blob bytes.
* @throws Error when the blob is missing or fails integrity verification.
*/
async read(hash: string): Promise<Buffer> {
// Check cache first
const cached = this.getFromCache(hash)
if (cached) {
return cached.data
}
const metadataBuffer = await this.adapter.get(`blob-meta:${hash}`)
if (!metadataBuffer) {
throw new Error(`Blob metadata not found: ${hash}`)
}
// Unwrap before parsing (defense-in-depth): metadata should come back as
// JSON bytes, but an adapter might return the wrapped binary format.
const metadata: BlobMetadata = JSON.parse(unwrapBinaryData(metadataBuffer).toString())
const data = await this.adapter.get(`blob:${hash}`)
if (!data) {
throw new Error(`Blob not found: ${hash}`)
}
// Decompress if needed
let finalData = data
if (metadata.compression === 'zstd') {
if (!this.zstdDecompress) {
await this.ensureCompressionReady()
}
if (!this.zstdDecompress) {
throw new Error('zstd decompression not available')
}
finalData = await this.zstdDecompress(data)
}
// Defense-in-depth unwrap: even though the bridge unwraps, verify it
// happened and re-unwrap if needed. Hash verification must run on the
// original content bytes.
const unwrappedData = unwrapBinaryData(finalData)
// Integrity verification (always on — a content-addressed store that
// returns bytes not matching the address is corruption, not a result)
if (BlobStorage.hash(unwrappedData) !== hash) {
throw new Error(`Blob integrity check failed: ${hash}`)
}
this.addToCache(hash, unwrappedData, metadata)
return unwrappedData
}
/**
* @description Whether a blob with this hash exists (cache or storage).
* @param hash - The blob's SHA-256 hash.
* @returns True when the blob exists.
*/
async has(hash: string): Promise<boolean> {
if (this.cache.has(hash)) {
return true
}
const exists = await this.adapter.get(`blob:${hash}`)
return exists !== undefined
}
/**
* @description Drop one reference to the blob. The stored bytes and
* metadata are physically deleted only when the reference count reaches
* zero — deduplicated content shared by other writers survives.
*
* @param hash - The blob's SHA-256 hash.
*/
async delete(hash: string): Promise<void> {
const refCount = await this.decrementRefCount(hash)
// Only delete if no references remain
if (refCount > 0) {
return
}
await this.adapter.delete(`blob:${hash}`)
await this.adapter.delete(`blob-meta:${hash}`)
this.removeFromCache(hash)
}
/**
* @description Read a blob's metadata without reading its bytes.
* @param hash - The blob's SHA-256 hash.
* @returns The metadata, or `undefined` when the blob does not exist.
*/
async getMetadata(hash: string): Promise<BlobMetadata | undefined> {
const data = await this.adapter.get(`blob-meta:${hash}`)
if (data) {
return JSON.parse(unwrapBinaryData(data).toString())
}
return undefined
}
// ========== PRIVATE METHODS ==========
/**
* Select the compression strategy for a write (see
* {@link BlobWriteOptions.compression}).
*/
private selectCompression(
data: Buffer,
options: BlobWriteOptions
): 'none' | 'zstd' {
if (options.compression === 'none') {
return 'none'
}
if (options.compression === 'zstd') {
return this.zstdCompress ? 'zstd' : 'none'
}
// Auto mode
if (data.length < this.COMPRESSION_THRESHOLD) {
return 'none' // Too small to benefit
}
// Content-type policy: skip already-compressed media. zstd over
// JPEG / MP4 / ZIP etc. is a CPU loss for no measurable byte savings,
// and on hot save paths (image / video uploads) it's the difference
// between fast and slow. Applies only to `auto`; explicit `'zstd'` is
// honoured because the caller is asserting the choice.
if (isAlreadyCompressedMimeType(options.mimeType)) {
return 'none'
}
return this.zstdCompress ? 'zstd' : 'none'
}
/**
* Increment the reference count for an existing blob.
*/
private async incrementRefCount(hash: string): Promise<number> {
const metadata = await this.getMetadata(hash)
if (!metadata) {
throw new Error(`Cannot increment ref count, blob not found: ${hash}`)
}
metadata.refCount++
await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata)))
return metadata.refCount
}
/**
* Decrement the reference count for a blob (floored at zero).
*/
private async decrementRefCount(hash: string): Promise<number> {
const metadata = await this.getMetadata(hash)
if (!metadata) {
return 0
}
metadata.refCount = Math.max(0, metadata.refCount - 1)
await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata)))
return metadata.refCount
}
/**
* Add a blob to the LRU cache (evicting least-recently-used entries to
* stay under the size bound).
*/
private addToCache(hash: string, data: Buffer, metadata: BlobMetadata): void {
if (data.length > this.cacheMaxSize) {
return // Blob too large for cache
}
while (
this.currentCacheSize + data.length > this.cacheMaxSize &&
this.cache.size > 0
) {
this.evictLRU()
}
this.cache.set(hash, {
data,
metadata,
lastAccess: Date.now(),
size: data.length
})
this.currentCacheSize += data.length
}
/**
* Get a blob from the cache, refreshing its LRU position.
*/
private getFromCache(hash: string): CacheEntry | undefined {
const entry = this.cache.get(hash)
if (entry) {
entry.lastAccess = Date.now() // Update LRU
}
return entry
}
/**
* Remove a blob from the cache.
*/
private removeFromCache(hash: string): void {
const entry = this.cache.get(hash)
if (entry) {
this.cache.delete(hash)
this.currentCacheSize -= entry.size
}
}
/**
* Evict the least-recently-used cache entry.
*/
private evictLRU(): void {
let oldestHash: string | null = null
let oldestTime = Infinity
for (const [hash, entry] of this.cache.entries()) {
if (entry.lastAccess < oldestTime) {
oldestTime = entry.lastAccess
oldestHash = hash
}
}
if (oldestHash) {
this.removeFromCache(oldestHash)
}
}
}

763
src/storage/cow/BlobStorage.ts
View file

@ -1,763 +0,0 @@
/**
* BlobStorage: Content-Addressable Blob Storage for COW (Copy-on-Write)
*
* State-of-the-art implementation featuring:
* - Content-addressable: SHA-256 hashing
* - Type-aware chunking: Separate vectors, metadata, relationships
* - Compression: zstd for JSON, optimized for vectors
* - LRU caching: Hot blob performance
* - Streaming: Multipart upload for large blobs
* - Batch operations: Parallel I/O
* - Integrity: Cryptographic verification
* - Observability: Metrics and tracing
*
* @module storage/cow/BlobStorage
*/
import { createHash } from 'crypto'
import { NULL_HASH, isNullHash } from './constants.js'
import { unwrapBinaryData } from './binaryDataCodec.js'
/**
* Simple key-value storage interface for COW primitives
* This will be implemented by BaseStorage when COW is integrated
*/
export interface COWStorageAdapter {
get(key: string): Promise<Buffer | undefined>
put(key: string, data: Buffer): Promise<void>
delete(key: string): Promise<void>
list(prefix: string): Promise<string[]>
}
/**
* Blob metadata stored alongside blob data
*/
export interface BlobMetadata {
hash: string // SHA-256 hash
size: number // Original size in bytes
compressedSize: number // Compressed size in bytes
compression: 'none' | 'zstd'
type: 'vector' | 'metadata' | 'tree' | 'commit' | 'blob' | 'raw'
createdAt: number // Timestamp
refCount: number // How many objects reference this blob
}
/**
* Blob write options
*/
export interface BlobWriteOptions {
compression?: 'none' | 'zstd' | 'auto' // Auto chooses based on type
type?: 'vector' | 'metadata' | 'tree' | 'commit' | 'blob' | 'raw'
/**
* Content type of the payload (e.g. `image/jpeg`, `video/mp4`).
*
* When set on an `auto`-compression write, BlobStorage skips zstd for MIME
* types that are already heavily compressed (JPEG, PNG, WebP, MP4, WebM,
* MP3, ZIP, PDF, etc.). Gzip/zstd over these formats wastes CPU and rarely
* shaves more than a single-digit percent — usually it actually grows the
* payload because the entropy is already maximised by the format itself.
*
* Has no effect when `compression` is set to `'none'` or `'zstd'` explicitly
* — the caller is asserting the choice and BlobStorage honours it.
*/
mimeType?: string
skipVerification?: boolean // Skip hash verification (faster, less safe)
}
/**
* MIME types whose payload is already heavily compressed. zstd over these is
* almost always a CPU-only loss — the bytes are already near entropy-maximal,
* so the output is the same size or slightly larger plus the cost of running
* the compressor. Used by `BlobStorage.selectCompression()` in `auto` mode.
*
* Conservative denylist (well-known formats only). Anything not in this set
* goes through the existing per-type heuristic. False negatives (compressing
* something we should have skipped) waste CPU; false positives (skipping
* something we could have compressed) waste a few percent of bytes. The
* denylist favours CPU-cycle safety because the formats listed here are the
* ones where gzip/zstd is reliably a net loss.
*/
const ALREADY_COMPRESSED_MIME_TYPES = new Set<string>([
// Images
'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp',
'image/avif', 'image/heic', 'image/heif', 'image/jp2',
// Video
'video/mp4', 'video/webm', 'video/x-matroska', 'video/quicktime',
'video/x-msvideo', 'video/mpeg', 'video/3gpp', 'video/x-ms-wmv',
// Audio
'audio/mpeg', 'audio/mp4', 'audio/aac', 'audio/ogg', 'audio/webm',
'audio/opus', 'audio/flac', 'audio/x-ms-wma',
// Archives
'application/zip', 'application/gzip', 'application/x-gzip',
'application/x-bzip2', 'application/x-7z-compressed',
'application/x-rar-compressed', 'application/x-xz', 'application/x-zstd',
'application/x-compress', 'application/vnd.rar',
// Documents with internal compression
'application/pdf', 'application/epub+zip',
// Office formats (zip-based)
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/vnd.oasis.opendocument.text',
'application/vnd.oasis.opendocument.spreadsheet',
'application/vnd.oasis.opendocument.presentation'
])
/**
* @description True when the MIME type names a payload format known to be
* already heavily compressed. Strips any `;charset=…` / `;boundary=…`
* parameters and lowercases the bare type/subtype before lookup, so
* `'IMAGE/JPEG; charset=binary'` and `'image/jpeg'` resolve identically.
*/
export function isAlreadyCompressedMimeType(mimeType: string | undefined): boolean {
if (!mimeType) return false
const bare = mimeType.split(';', 1)[0].trim().toLowerCase()
return ALREADY_COMPRESSED_MIME_TYPES.has(bare)
}
/**
* Blob read options
*/
export interface BlobReadOptions {
skipDecompression?: boolean // Return compressed data
skipCache?: boolean // Don't use cache
skipVerification?: boolean // Skip hash verification (faster, less safe)
}
/**
* Blob statistics for observability
*/
export interface BlobStats {
totalBlobs: number
totalSize: number
compressedSize: number
cacheHits: number
cacheMisses: number
compressionRatio: number
avgBlobSize: number
dedupSavings: number // Bytes saved from deduplication
}
/**
* LRU Cache entry
*/
interface CacheEntry {
data: Buffer
metadata: BlobMetadata
lastAccess: number
size: number
}
/**
* State-of-the-art content-addressable blob storage
*
* Features:
* - Content addressing via SHA-256
* - Type-aware compression (zstd, vector-optimized)
* - LRU caching with memory limits
* - Streaming for large blobs
* - Batch operations
* - Integrity verification
* - Observability metrics
*/
export class BlobStorage {
private adapter: COWStorageAdapter
private cache: Map<string, CacheEntry>
private cacheMaxSize: number
private currentCacheSize: number
private stats: BlobStats
// Compression (lazily loaded)
private zstdCompress?: (data: Buffer) => Promise<Buffer>
private zstdDecompress?: (data: Buffer) => Promise<Buffer>
private compressionReady = false
// Configuration
private readonly CACHE_MAX_SIZE = 100 * 1024 * 1024 // 100MB default
private readonly MULTIPART_THRESHOLD = 5 * 1024 * 1024 // 5MB
private readonly COMPRESSION_THRESHOLD = 1024 // 1KB - don't compress smaller
constructor(adapter: COWStorageAdapter, options?: {
cacheMaxSize?: number
enableCompression?: boolean
}) {
this.adapter = adapter
this.cache = new Map()
this.cacheMaxSize = options?.cacheMaxSize ?? this.CACHE_MAX_SIZE
this.currentCacheSize = 0
this.stats = {
totalBlobs: 0,
totalSize: 0,
compressedSize: 0,
cacheHits: 0,
cacheMisses: 0,
compressionRatio: 1.0,
avgBlobSize: 0,
dedupSavings: 0
}
// Lazy load compression (only if needed)
if (options?.enableCompression !== false) {
this.initCompression()
}
}
/**
* Lazy load zstd compression module
* (Avoids loading if not needed)
*/
private async initCompression(): Promise<void> {
try {
// Dynamic import to avoid loading if not needed
// @ts-ignore - Optional dependency, gracefully handled if missing
const zstd = await import('@mongodb-js/zstd')
this.zstdCompress = async (data: Buffer) => {
return Buffer.from(await zstd.compress(data, 3)) // Level 3 = fast
}
this.zstdDecompress = async (data: Buffer) => {
return Buffer.from(await zstd.decompress(data))
}
} catch (error) {
console.warn('zstd compression not available, falling back to uncompressed')
this.zstdCompress = undefined
this.zstdDecompress = undefined
}
}
/**
* Ensure compression is ready before write operations
* Fixes race condition where write happens before async compression init completes
*/
private async ensureCompressionReady(): Promise<void> {
if (this.compressionReady) return
await this.initCompression()
this.compressionReady = true
}
/**
* Compute SHA-256 hash of data
*
* @param data - Data to hash
* @returns SHA-256 hash as hex string
*/
static hash(data: Buffer): string {
return createHash('sha256').update(data).digest('hex')
}
/**
* Write a blob to storage
*
* Features:
* - Content-addressable: hash determines storage key
* - Deduplication: existing blob not rewritten
* - Compression: auto-compress based on type
* - Multipart: for large blobs (>5MB)
* - Verification: hash verification
* - Caching: write-through cache
*
* @param data - Blob data to write
* @param options - Write options
* @returns Blob hash
*/
async write(data: Buffer, options: BlobWriteOptions = {}): Promise<string> {
const hash = BlobStorage.hash(data)
// Deduplication: Check if blob already exists
if (await this.has(hash)) {
// Update ref count
await this.incrementRefCount(hash)
this.stats.dedupSavings += data.length
return hash
}
// Ensure compression is initialized before writing
// Fixes race condition where write happens before async init completes
await this.ensureCompressionReady()
// Determine compression strategy
const compression = this.selectCompression(data, options)
// Compress if needed
let finalData = data
let compressedSize = data.length
if (compression === 'zstd' && this.zstdCompress) {
finalData = await this.zstdCompress(data)
compressedSize = finalData.length
}
// Create metadata
// Store ACTUAL compression state, not intended
// Prevents corruption if compression failed to initialize
const actualCompression = finalData === data ? 'none' : compression
const metadata: BlobMetadata = {
hash,
size: data.length,
compressedSize,
compression: actualCompression,
type: options.type || 'blob', // CRITICAL FIX: Use 'blob' default to match storage prefix
createdAt: Date.now(),
refCount: 1
}
// Write blob data
if (finalData.length > this.MULTIPART_THRESHOLD) {
// Large blob: use streaming/multipart
await this.writeMultipart(hash, finalData, metadata)
} else {
// Small blob: single write
const prefix = options.type || 'blob'
await this.adapter.put(`${prefix}:${hash}`, finalData)
}
// Write metadata
const prefix = options.type || 'blob'
await this.adapter.put(`${prefix}-meta:${hash}`, Buffer.from(JSON.stringify(metadata)))
// Update cache (write-through)
this.addToCache(hash, data, metadata)
// Update stats
this.stats.totalBlobs++
this.stats.totalSize += data.length
this.stats.compressedSize += compressedSize
this.stats.compressionRatio = this.stats.totalSize / (this.stats.compressedSize || 1)
this.stats.avgBlobSize = this.stats.totalSize / this.stats.totalBlobs
return hash
}
/**
* Read a blob from storage
*
* Features:
* - Cache lookup first (LRU)
* - Decompression (if compressed)
* - Verification (optional hash check)
* - Streaming for large blobs
*
* @param hash - Blob hash
* @param options - Read options
* @returns Blob data
*/
async read(hash: string, options: BlobReadOptions = {}): Promise<Buffer> {
// Guard against NULL hash (sentinel value)
// NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree"
// It should NEVER be read as actual blob data
if (isNullHash(hash)) {
throw new Error(
`Cannot read NULL hash (${NULL_HASH}): ` +
`This is a sentinel value indicating "no parent commit" or "empty tree". ` +
`If you're seeing this error from CommitObject.walk(), there's a bug in commit traversal logic. ` +
`If you're seeing this from TreeObject operations, there's a bug in tree handling.`
)
}
// Check cache first
if (!options.skipCache) {
const cached = this.getFromCache(hash)
if (cached) {
this.stats.cacheHits++
return cached.data
}
this.stats.cacheMisses++
}
// Try to read metadata to determine type (for backward compatibility)
// Try commit, tree, then blob prefixes
let prefix: string | null = null
let metadataBuffer: Buffer | undefined
let metadata: BlobMetadata | undefined
for (const tryPrefix of ['commit', 'tree', 'blob']) {
metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`)
if (metadataBuffer) {
prefix = tryPrefix
// Unwrap metadata before parsing (defense-in-depth)
// Metadata should be JSON, but adapter might return wrapped format
const unwrappedMetadata = unwrapBinaryData(metadataBuffer)
metadata = JSON.parse(unwrappedMetadata.toString())
break
}
}
if (!prefix || !metadata) {
throw new Error(`Blob metadata not found: ${hash}`)
}
// Read from storage using determined prefix
const data = await this.adapter.get(`${prefix}:${hash}`)
if (!data) {
throw new Error(`Blob not found: ${hash}`)
}
// Decompress if needed
let finalData = data
if (metadata.compression === 'zstd' && !options.skipDecompression) {
if (!this.zstdDecompress) {
throw new Error('zstd decompression not available')
}
finalData = await this.zstdDecompress(data)
}
// Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
// Even though COW adapter should unwrap, verify it happened and re-unwrap if needed
// This prevents "Blob integrity check failed" errors if adapter returns wrapped data
// Uses shared binaryDataCodec utility (single source of truth for unwrap logic)
const unwrappedData = unwrapBinaryData(finalData)
// Verify hash (on unwrapped data)
if (!options.skipVerification && BlobStorage.hash(unwrappedData) !== hash) {
throw new Error(`Blob integrity check failed: ${hash}`)
}
// Add to cache (only if not skipped)
if (!options.skipCache) {
this.addToCache(hash, unwrappedData, metadata)
}
return unwrappedData
}
/**
* Check if blob exists
*
* @param hash - Blob hash
* @returns True if blob exists
*/
async has(hash: string): Promise<boolean> {
// Check cache first
if (this.cache.has(hash)) {
return true
}
// Check storage - try all prefixes for backward compatibility
for (const prefix of ['commit', 'tree', 'blob']) {
const exists = await this.adapter.get(`${prefix}:${hash}`)
if (exists !== undefined) {
return true
}
}
return false
}
/**
* Delete a blob from storage
*
* Features:
* - Reference counting: only delete if refCount = 0
* - Cascade: delete metadata too
* - Cache invalidation
*
* @param hash - Blob hash
*/
async delete(hash: string): Promise<void> {
// Decrement ref count
const refCount = await this.decrementRefCount(hash)
// Only delete if no references remain
if (refCount > 0) {
return
}
// Determine prefix by checking which one exists
let prefix = 'blob'
for (const tryPrefix of ['commit', 'tree', 'blob', 'metadata', 'vector', 'raw']) {
const exists = await this.adapter.get(`${tryPrefix}:${hash}`)
if (exists !== undefined) {
prefix = tryPrefix
break
}
}
// Delete blob data
await this.adapter.delete(`${prefix}:${hash}`)
// Delete metadata
await this.adapter.delete(`${prefix}-meta:${hash}`)
// Remove from cache
this.removeFromCache(hash)
// Update stats
this.stats.totalBlobs--
}
/**
* Get blob metadata without reading full blob
*
* @param hash - Blob hash
* @returns Blob metadata
*/
async getMetadata(hash: string): Promise<BlobMetadata | undefined> {
// Try to read metadata with type-aware prefix (backward compatible)
// Check all valid blob types: commit, tree, blob, metadata, vector, raw
for (const prefix of ['commit', 'tree', 'blob', 'metadata', 'vector', 'raw']) {
const data = await this.adapter.get(`${prefix}-meta:${hash}`)
if (data) {
return JSON.parse(data.toString())
}
}
return undefined
}
/**
* Batch write multiple blobs in parallel
*
* @param blobs - Array of [data, options] tuples
* @returns Array of blob hashes
*/
async writeBatch(blobs: Array<[Buffer, BlobWriteOptions?]>): Promise<string[]> {
return Promise.all(
blobs.map(([data, options]) => this.write(data, options))
)
}
/**
* Batch read multiple blobs in parallel
*
* @param hashes - Array of blob hashes
* @param options - Read options
* @returns Array of blob data
*/
async readBatch(hashes: string[], options?: BlobReadOptions): Promise<Buffer[]> {
return Promise.all(
hashes.map(hash => this.read(hash, options))
)
}
/**
* List all blobs (for garbage collection, debugging)
*
* @returns Array of blob hashes
*/
async listBlobs(): Promise<string[]> {
// List all types of blobs
const hashes = new Set<string>()
for (const prefix of ['commit', 'tree', 'blob']) {
const keys = await this.adapter.list(`${prefix}:`)
keys.forEach((key: string) => {
const hash = key.replace(new RegExp(`^${prefix}:`), '')
hashes.add(hash)
})
}
return Array.from(hashes)
}
/**
* Get storage statistics
*
* @returns Blob statistics
*/
getStats(): BlobStats {
return { ...this.stats }
}
/**
* Clear cache (useful for testing, memory pressure)
*/
clearCache(): void {
this.cache.clear()
this.currentCacheSize = 0
}
/**
* Garbage collect unreferenced blobs
*
* @param referencedHashes - Set of hashes that should be kept
* @returns Number of blobs deleted
*/
async garbageCollect(referencedHashes: Set<string>): Promise<number> {
const allBlobs = await this.listBlobs()
let deleted = 0
for (const hash of allBlobs) {
if (!referencedHashes.has(hash)) {
// Check ref count
const metadata = await this.getMetadata(hash)
if (metadata && metadata.refCount === 0) {
await this.delete(hash)
deleted++
}
}
}
return deleted
}
// ========== PRIVATE METHODS ==========
/**
* Select compression strategy based on data and options
*/
private selectCompression(
data: Buffer,
options: BlobWriteOptions
): 'none' | 'zstd' {
if (options.compression === 'none') {
return 'none'
}
if (options.compression === 'zstd') {
return this.zstdCompress ? 'zstd' : 'none'
}
// Auto mode
if (data.length < this.COMPRESSION_THRESHOLD) {
return 'none' // Too small to benefit
}
// Content-type policy (2.5.0 #32): skip already-compressed media. zstd
// over JPEG / MP4 / ZIP etc. is a CPU loss for no measurable byte savings,
// and on hot save paths (image / video uploads) it's the difference
// between fast and slow. Applies only to `auto`; explicit `'zstd'` is
// honoured because the caller is asserting the choice.
if (isAlreadyCompressedMimeType(options.mimeType)) {
return 'none'
}
// Compress metadata, trees, commits (text/JSON)
if (options.type === 'metadata' || options.type === 'tree' || options.type === 'commit') {
return this.zstdCompress ? 'zstd' : 'none'
}
// Don't compress vectors (already dense)
if (options.type === 'vector') {
return 'none'
}
// Default: compress
return this.zstdCompress ? 'zstd' : 'none'
}
/**
* Write large blob using multipart upload
* (Future enhancement: stream to adapter if supported)
*/
private async writeMultipart(
hash: string,
data: Buffer,
metadata: BlobMetadata
): Promise<void> {
// Single-blob write. Brainy 8.0 ships filesystem + memory only; the
// multipart-upload concern that motivated this method (S3/R2/GCS) is gone
// with the cloud adapters per BR-BRAINY-80-STORAGE-SIMPLIFY.
const prefix = metadata.type || 'blob'
await this.adapter.put(`${prefix}:${hash}`, data)
}
/**
* Increment reference count for a blob
*/
private async incrementRefCount(hash: string): Promise<number> {
const metadata = await this.getMetadata(hash)
if (!metadata) {
throw new Error(`Cannot increment ref count, blob not found: ${hash}`)
}
metadata.refCount++
const prefix = metadata.type || 'blob'
await this.adapter.put(
`${prefix}-meta:${hash}`,
Buffer.from(JSON.stringify(metadata))
)
return metadata.refCount
}
/**
* Decrement reference count for a blob
*/
private async decrementRefCount(hash: string): Promise<number> {
const metadata = await this.getMetadata(hash)
if (!metadata) {
return 0
}
metadata.refCount = Math.max(0, metadata.refCount - 1)
const prefix = metadata.type || 'blob'
await this.adapter.put(
`${prefix}-meta:${hash}`,
Buffer.from(JSON.stringify(metadata))
)
return metadata.refCount
}
/**
* Add blob to LRU cache
*/
private addToCache(hash: string, data: Buffer, metadata: BlobMetadata): void {
// Check if adding would exceed cache size
if (data.length > this.cacheMaxSize) {
return // Blob too large for cache
}
// Evict old entries if needed
while (
this.currentCacheSize + data.length > this.cacheMaxSize &&
this.cache.size > 0
) {
this.evictLRU()
}
// Add to cache
this.cache.set(hash, {
data,
metadata,
lastAccess: Date.now(),
size: data.length
})
this.currentCacheSize += data.length
}
/**
* Get blob from cache
*/
private getFromCache(hash: string): CacheEntry | undefined {
const entry = this.cache.get(hash)
if (entry) {
entry.lastAccess = Date.now() // Update LRU
}
return entry
}
/**
* Remove blob from cache
*/
private removeFromCache(hash: string): void {
const entry = this.cache.get(hash)
if (entry) {
this.cache.delete(hash)
this.currentCacheSize -= entry.size
}
}
/**
* Evict least recently used entry from cache
*/
private evictLRU(): void {
let oldestHash: string | null = null
let oldestTime = Infinity
for (const [hash, entry] of this.cache.entries()) {
if (entry.lastAccess < oldestTime) {
oldestTime = entry.lastAccess
oldestHash = hash
}
}
if (oldestHash) {
this.removeFromCache(oldestHash)
}
}
}

531
src/storage/cow/CommitLog.ts
View file

@ -1,531 +0,0 @@
/**
* CommitLog: Commit history traversal and querying for COW (Copy-on-Write)
*
* Provides efficient commit history operations:
* - Walk commit graph (DAG traversal)
* - Find commits by time, author, operation
* - Time-travel queries (asOf)
* - Commit statistics and analytics
*
* Optimizations:
* - Commit index for fast timestamp lookups
* - Parent cache for efficient traversal
* - Lazy loading (only read commits when needed)
*
* @module storage/cow/CommitLog
*/
import { BlobStorage } from './BlobStorage.js'
import { CommitObject } from './CommitObject.js'
import { RefManager } from './RefManager.js'
/**
* Commit index entry (for fast lookups)
*/
interface CommitIndexEntry {
hash: string
timestamp: number
parentHash: string | null
}
/**
* Commit log statistics
*/
export interface CommitLogStats {
totalCommits: number
oldestCommit: number // Timestamp
newestCommit: number // Timestamp
authors: Set<string>
operations: Set<string>
avgCommitInterval: number // Average time between commits (ms)
}
/**
* CommitLog: Efficient commit history traversal and querying
*
* Pure implementation - modern, clean, fast
*/
export class CommitLog {
private blobStorage: BlobStorage
private refManager: RefManager
private index: Map<string, CommitIndexEntry>
private indexValid: boolean
constructor(blobStorage: BlobStorage, refManager: RefManager) {
this.blobStorage = blobStorage
this.refManager = refManager
this.index = new Map()
this.indexValid = false
}
/**
* Walk commit history from a starting point
*
* Yields commits in reverse chronological order (newest first)
*
* @param startRef - Starting ref/commit (e.g., 'main', commit hash)
* @param options - Walk options
*/
async *walk(
startRef: string = 'main',
options?: {
maxDepth?: number
until?: number
stopAt?: string
filter?: (commit: CommitObject) => boolean
}
): AsyncIterableIterator<CommitObject> {
// Resolve ref to commit hash
let startHash: string
if (/^[a-f0-9]{64}$/.test(startRef)) {
// Already a commit hash
startHash = startRef
} else {
// Resolve ref
const commitHash = await this.refManager.resolveRef(startRef)
if (!commitHash) {
throw new Error(`Ref not found: ${startRef}`)
}
startHash = commitHash
}
// Walk using CommitObject (delegates to it)
yield* CommitObject.walk(this.blobStorage, startHash, options)
}
/**
* Find commit at or before a specific timestamp
*
* Uses index for fast O(log n) lookup
*
* @param ref - Starting ref (e.g., 'main')
* @param timestamp - Target timestamp
* @returns Commit at or before timestamp, or null
*/
async findAtTime(ref: string, timestamp: number): Promise<CommitObject | null> {
// Build index if needed
await this.buildIndex(ref)
// Binary search in index
const entries = Array.from(this.index.values()).sort(
(a, b) => b.timestamp - a.timestamp // Newest first
)
let left = 0
let right = entries.length - 1
let result: CommitIndexEntry | null = null
while (left <= right) {
const mid = Math.floor((left + right) / 2)
const entry = entries[mid]
if (entry.timestamp <= timestamp) {
result = entry
right = mid - 1 // Look for newer commit
} else {
left = mid + 1 // Look for older commit
}
}
if (!result) {
return null
}
// Read full commit
return CommitObject.read(this.blobStorage, result.hash)
}
/**
* Get commit by hash
*
* @param hash - Commit hash
* @returns Commit object
*/
async getCommit(hash: string): Promise<CommitObject> {
return CommitObject.read(this.blobStorage, hash)
}
/**
* Get commits in time range
*
* @param ref - Starting ref
* @param startTime - Start of time range
* @param endTime - End of time range
* @returns Array of commits in range (newest first)
*/
async getInTimeRange(
ref: string,
startTime: number,
endTime: number
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
for await (const commit of this.walk(ref, { until: startTime })) {
if (commit.timestamp >= startTime && commit.timestamp <= endTime) {
commits.push(commit)
}
}
return commits
}
/**
* Get commits by author
*
* @param ref - Starting ref
* @param author - Author name
* @param options - Additional options
* @returns Array of commits by author
*/
async getByAuthor(
ref: string,
author: string,
options?: { maxCount?: number; since?: number }
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
let count = 0
for await (const commit of this.walk(ref, { until: options?.since })) {
if (commit.author === author) {
commits.push(commit)
count++
if (options?.maxCount && count >= options.maxCount) {
break
}
}
}
return commits
}
/**
* Get commits by operation type
*
* @param ref - Starting ref
* @param operation - Operation type (e.g., 'add', 'update', 'delete')
* @param options - Additional options
* @returns Array of commits by operation
*/
async getByOperation(
ref: string,
operation: string,
options?: { maxCount?: number; since?: number }
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
let count = 0
for await (const commit of this.walk(ref, { until: options?.since })) {
if (commit.metadata?.operation === operation) {
commits.push(commit)
count++
if (options?.maxCount && count >= options.maxCount) {
break
}
}
}
return commits
}
/**
* Get commit history as array
*
* @param ref - Starting ref
* @param options - Walk options
* @returns Array of commits (newest first)
*/
async getHistory(
ref: string,
options?: {
maxCount?: number
since?: number
until?: number
}
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
let count = 0
for await (const commit of this.walk(ref, {
maxDepth: options?.maxCount,
until: options?.until
})) {
if (options?.since && commit.timestamp < options.since) {
continue
}
commits.push(commit)
count++
if (options?.maxCount && count >= options.maxCount) {
break
}
}
return commits
}
/**
* Stream commit history (memory-efficient for large histories)
*
* Yields commits one at a time without accumulating in memory.
* Use this for large commit histories (1000s of commits) where
* memory efficiency is important.
*
* @param ref - Starting ref
* @param options - Walk options
* @yields Commits in reverse chronological order (newest first)
*
* @example
* ```typescript
* // Stream all commits without memory accumulation
* for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) {
* console.log(commit.message)
* }
* ```
*/
async *streamHistory(
ref: string,
options?: {
maxCount?: number
since?: number
until?: number
}
): AsyncIterableIterator<CommitObject> {
let count = 0
for await (const commit of this.walk(ref, {
maxDepth: options?.maxCount,
until: options?.until
})) {
// Filter by since timestamp if provided
if (options?.since && commit.timestamp < options.since) {
continue
}
yield commit
count++
// Stop after maxCount commits
if (options?.maxCount && count >= options.maxCount) {
break
}
}
}
/**
* Count commits between two commits
*
* @param fromRef - Starting ref/commit
* @param toRef - Ending ref/commit (optional, defaults to fromRef's parent)
* @returns Number of commits between
*/
async countBetween(fromRef: string, toRef?: string): Promise<number> {
const fromHash = await this.resolveToHash(fromRef)
const toHash = toRef ? await this.resolveToHash(toRef) : null
return CommitObject.countBetween(this.blobStorage, fromHash, toHash!)
}
/**
* Find common ancestor of two commits (merge base)
*
* @param ref1 - First ref/commit
* @param ref2 - Second ref/commit
* @returns Common ancestor commit or null
*/
async findCommonAncestor(
ref1: string,
ref2: string
): Promise<CommitObject | null> {
const hash1 = await this.resolveToHash(ref1)
const hash2 = await this.resolveToHash(ref2)
return CommitObject.findCommonAncestor(this.blobStorage, hash1, hash2)
}
/**
* Get commit log statistics
*
* @param ref - Starting ref
* @param options - Options
* @returns Commit log statistics
*/
async getStats(
ref: string = 'main',
options?: { maxDepth?: number }
): Promise<CommitLogStats> {
const authors = new Set<string>()
const operations = new Set<string>()
const timestamps: number[] = []
let totalCommits = 0
for await (const commit of this.walk(ref, options)) {
totalCommits++
authors.add(commit.author)
timestamps.push(commit.timestamp)
if (commit.metadata?.operation) {
operations.add(commit.metadata.operation)
}
}
// Calculate average interval
let avgInterval = 0
if (timestamps.length > 1) {
const intervals: number[] = []
for (let i = 0; i < timestamps.length - 1; i++) {
intervals.push(timestamps[i] - timestamps[i + 1])
}
avgInterval = intervals.reduce((a, b) => a + b, 0) / intervals.length
}
return {
totalCommits,
oldestCommit: timestamps[timestamps.length - 1] ?? 0,
newestCommit: timestamps[0] ?? 0,
authors,
operations,
avgCommitInterval: avgInterval
}
}
/**
* Check if commit is ancestor of another commit
*
* @param ancestorRef - Potential ancestor ref/commit
* @param descendantRef - Descendant ref/commit
* @returns True if ancestor is in descendant's history
*/
async isAncestor(ancestorRef: string, descendantRef: string): Promise<boolean> {
const ancestorHash = await this.resolveToHash(ancestorRef)
const descendantHash = await this.resolveToHash(descendantRef)
// Walk from descendant, check if we encounter ancestor
for await (const commit of this.walk(descendantHash)) {
const commitHash = CommitObject.hash(commit)
if (commitHash === ancestorHash) {
return true
}
}
return false
}
/**
* Get recent commits (last N)
*
* @param ref - Starting ref
* @param count - Number of commits to retrieve
* @returns Array of recent commits
*/
async getRecent(ref: string, count: number = 10): Promise<CommitObject[]> {
return this.getHistory(ref, { maxCount: count })
}
/**
* Find commits with tag
*
* @param ref - Starting ref
* @param tag - Tag to search for
* @returns Array of commits with tag
*/
async findWithTag(ref: string, tag: string): Promise<CommitObject[]> {
const commits: CommitObject[] = []
for await (const commit of this.walk(ref)) {
if (CommitObject.hasTag(commit, tag)) {
commits.push(commit)
}
}
return commits
}
/**
* Get first (oldest) commit
*
* @param ref - Starting ref
* @returns Oldest commit
*/
async getFirstCommit(ref: string): Promise<CommitObject | null> {
let oldest: CommitObject | null = null
for await (const commit of this.walk(ref)) {
oldest = commit
}
return oldest
}
/**
* Get latest commit
*
* @param ref - Starting ref
* @returns Latest commit
*/
async getLatestCommit(ref: string): Promise<CommitObject | null> {
for await (const commit of this.walk(ref, { maxDepth: 1 })) {
return commit
}
return null
}
/**
* Clear index (useful for testing, after new commits)
*/
clearIndex(): void {
this.index.clear()
this.indexValid = false
}
// ========== PRIVATE METHODS ==========
/**
* Build commit index for fast lookups
*
* @param ref - Starting ref
*/
private async buildIndex(ref: string): Promise<void> {
if (this.indexValid) {
return // Already built
}
this.index.clear()
for await (const commit of this.walk(ref)) {
const hash = CommitObject.hash(commit)
this.index.set(hash, {
hash,
timestamp: commit.timestamp,
parentHash: commit.parent
})
}
this.indexValid = true
}
/**
* Resolve ref or hash to commit hash
*
* @param refOrHash - Ref name or commit hash
* @returns Commit hash
*/
private async resolveToHash(refOrHash: string): Promise<string> {
if (/^[a-f0-9]{64}$/.test(refOrHash)) {
return refOrHash // Already a hash
}
const hash = await this.refManager.resolveRef(refOrHash)
if (!hash) {
throw new Error(`Ref not found: ${refOrHash}`)
}
return hash
}
}

563
src/storage/cow/CommitObject.ts
View file

@ -1,563 +0,0 @@
/**
* CommitObject: Snapshot metadata for COW (Copy-on-Write)
*
* Similar to Git commits, represents a point-in-time snapshot of a Brainy instance.
* Commits reference tree objects and parent commits, forming a directed acyclic graph (DAG).
*
* Structure:
* - tree: hash of root tree object (contains all data)
* - parent: hash of parent commit (null for initial commit)
* - message: human-readable commit message
* - author: who/what created this commit
* - timestamp: when this commit was created
* - metadata: additional commit metadata (tags, etc.)
*
* @module storage/cow/CommitObject
*/
import { BlobStorage } from './BlobStorage.js'
import { isNullHash } from './constants.js'
/**
* Commit object structure
*/
export interface CommitObject {
tree: string // Root tree hash
parent: string | null // Parent commit hash (null for initial commit)
message: string // Commit message
author: string // Author (user, system, augmentation)
timestamp: number // Unix timestamp (milliseconds)
metadata?: {
tags?: string[] // Tags (e.g., ['v1.0.0', 'production'])
branch?: string // Branch name (e.g., 'main', 'experiment')
operation?: string // Operation type (e.g., 'add', 'update', 'delete', 'merge')
entityCount?: number // Number of entities at this commit
relationshipCount?: number // Number of relationships at this commit
[key: string]: any // Custom metadata
}
}
/**
* CommitBuilder: Fluent API for building commit objects
*
* Example:
* ```typescript
* const commit = await CommitBuilder.create(blobStorage)
* .tree(treeHash)
* .parent(parentHash)
* .message('Add user entities')
* .author('system')
* .tag('v1.0.0')
* .build()
* ```
*/
export class CommitBuilder {
private _tree?: string
private _parent?: string | null
private _message: string = 'Auto-commit'
private _author: string = 'system'
private _timestamp: number = Date.now()
private _metadata: NonNullable<CommitObject['metadata']> = {}
private blobStorage: BlobStorage
constructor(blobStorage: BlobStorage) {
this.blobStorage = blobStorage
}
static create(blobStorage: BlobStorage): CommitBuilder {
return new CommitBuilder(blobStorage)
}
/**
* Set tree hash
*/
tree(hash: string): this {
this._tree = hash
return this
}
/**
* Set parent commit hash (null for initial commit)
*/
parent(hash: string | null): this {
this._parent = hash
return this
}
/**
* Set commit message
*/
message(message: string): this {
this._message = message
return this
}
/**
* Set commit author
*/
author(author: string): this {
this._author = author
return this
}
/**
* Set commit timestamp (defaults to now)
*/
timestamp(timestamp: number | Date): this {
this._timestamp = typeof timestamp === 'number' ? timestamp : timestamp.getTime()
return this
}
/**
* Add tag to commit
*/
tag(tag: string): this {
if (!this._metadata.tags) {
this._metadata.tags = []
}
this._metadata.tags.push(tag)
return this
}
/**
* Set branch name
*/
branch(branch: string): this {
this._metadata.branch = branch
return this
}
/**
* Set operation type
*/
operation(operation: string): this {
this._metadata.operation = operation
return this
}
/**
* Set entity count
*/
entityCount(count: number): this {
this._metadata.entityCount = count
return this
}
/**
* Set relationship count
*/
relationshipCount(count: number): this {
this._metadata.relationshipCount = count
return this
}
/**
* Set custom metadata
*/
meta(key: string, value: any): this {
this._metadata[key] = value
return this
}
/**
* Build and persist the commit object
*
* @returns Commit hash
*/
async build(): Promise<string> {
if (!this._tree) {
throw new Error('CommitBuilder: tree hash is required')
}
const commit: CommitObject = {
tree: this._tree,
parent: this._parent ?? null,
message: this._message,
author: this._author,
timestamp: this._timestamp,
metadata: Object.keys(this._metadata).length > 0 ? this._metadata : undefined
}
return CommitObject.write(this.blobStorage, commit)
}
}
/**
* CommitObject: Represents a point-in-time snapshot in COW storage
*/
export class CommitObject {
/**
* Serialize commit object to Buffer
*
* Format: JSON (simple, debuggable)
* Future: Could use protobuf for efficiency
*
* @param commit - Commit object
* @returns Serialized commit
*/
static serialize(commit: CommitObject): Buffer {
return Buffer.from(JSON.stringify(commit, null, 0)) // Compact JSON
}
/**
* Deserialize commit object from Buffer
*
* @param data - Serialized commit
* @returns Commit object
*/
static deserialize(data: Buffer): CommitObject {
const commit = JSON.parse(data.toString())
// Validate structure
if (typeof commit.tree !== 'string' || commit.tree.length !== 64) {
throw new Error('Invalid commit object: tree must be 64-char SHA-256')
}
if (commit.parent !== null && (typeof commit.parent !== 'string' || commit.parent.length !== 64)) {
throw new Error('Invalid commit object: parent must be 64-char SHA-256 or null')
}
if (typeof commit.message !== 'string') {
throw new Error('Invalid commit object: message must be string')
}
if (typeof commit.author !== 'string') {
throw new Error('Invalid commit object: author must be string')
}
if (typeof commit.timestamp !== 'number') {
throw new Error('Invalid commit object: timestamp must be number')
}
if (commit.metadata !== undefined && typeof commit.metadata !== 'object') {
throw new Error('Invalid commit object: metadata must be object')
}
return commit
}
/**
* Compute hash of commit object
*
* @param commit - Commit object
* @returns SHA-256 hash
*/
static hash(commit: CommitObject): string {
const data = CommitObject.serialize(commit)
return BlobStorage.hash(data)
}
/**
* Write commit object to blob storage
*
* @param blobStorage - Blob storage instance
* @param commit - Commit object
* @returns Commit hash
*/
static async write(blobStorage: BlobStorage, commit: CommitObject): Promise<string> {
const data = CommitObject.serialize(commit)
return blobStorage.write(data, { type: 'commit', compression: 'auto' })
}
/**
* Read commit object from blob storage
*
* @param blobStorage - Blob storage instance
* @param hash - Commit hash
* @returns Commit object
*/
static async read(blobStorage: BlobStorage, hash: string): Promise<CommitObject> {
const data = await blobStorage.read(hash)
return CommitObject.deserialize(data)
}
/**
* Check if commit is initial commit (has no parent)
*
* @param commit - Commit object
* @returns True if initial commit
*/
static isInitial(commit: CommitObject): boolean {
return commit.parent === null
}
/**
* Check if commit is merge commit (has multiple parents)
* (Future enhancement: support merge commits with multiple parents)
*
* @param commit - Commit object
* @returns True if merge commit
*/
static isMerge(commit: CommitObject): boolean {
// For now, we only support single-parent commits
// Future: extend to support multiple parents
return commit.metadata?.merge !== undefined
}
/**
* Check if commit has a specific tag
*
* @param commit - Commit object
* @param tag - Tag to check
* @returns True if commit has tag
*/
static hasTag(commit: CommitObject, tag: string): boolean {
return commit.metadata?.tags?.includes(tag) ?? false
}
/**
* Get all tags from commit
*
* @param commit - Commit object
* @returns Array of tags
*/
static getTags(commit: CommitObject): string[] {
return commit.metadata?.tags ?? []
}
/**
* Walk commit history (traverse DAG)
*
* Yields commits in reverse chronological order (newest first)
*
* @param blobStorage - Blob storage instance
* @param startHash - Starting commit hash
* @param options - Walk options
*/
static async *walk(
blobStorage: BlobStorage,
startHash: string,
options?: {
maxDepth?: number // Maximum depth to walk
until?: number // Stop at this timestamp
stopAt?: string // Stop at this commit hash
filter?: (commit: CommitObject) => boolean
}
): AsyncIterableIterator<CommitObject> {
let currentHash: string | null = startHash
let depth = 0
// Guard against NULL hash (sentinel for "no parent")
// The initial commit has parent = null or NULL_HASH ('0000...0000')
// We must stop walking when we reach it, not try to read it
while (currentHash && !isNullHash(currentHash)) {
// Check max depth
if (options?.maxDepth && depth >= options.maxDepth) {
break
}
// Read commit
const commit = await CommitObject.read(blobStorage, currentHash)
// Check filter
if (options?.filter && !options.filter(commit)) {
currentHash = commit.parent
depth++
continue
}
// Yield commit
yield commit
// Check stop conditions
if (options?.until && commit.timestamp < options.until) {
break
}
if (options?.stopAt && currentHash === options.stopAt) {
break
}
// Move to parent (can be null or NULL_HASH for initial commit)
currentHash = commit.parent
depth++
}
}
/**
* Find commit at or before a specific timestamp
*
* @param blobStorage - Blob storage instance
* @param startHash - Starting commit hash (e.g., 'main' ref)
* @param timestamp - Target timestamp
* @returns Commit at or before timestamp, or null if not found
*/
static async findAtTime(
blobStorage: BlobStorage,
startHash: string,
timestamp: number
): Promise<CommitObject | null> {
for await (const commit of CommitObject.walk(blobStorage, startHash)) {
if (commit.timestamp <= timestamp) {
return commit
}
}
return null
}
/**
* Get commit history as array (newest first)
*
* @param blobStorage - Blob storage instance
* @param startHash - Starting commit hash
* @param options - Walk options
* @returns Array of commits
*/
static async getHistory(
blobStorage: BlobStorage,
startHash: string,
options?: {
maxDepth?: number
until?: number
stopAt?: string
}
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
for await (const commit of CommitObject.walk(blobStorage, startHash, options)) {
commits.push(commit)
}
return commits
}
/**
* Find common ancestor of two commits (merge base)
*
* Useful for diff/merge operations
*
* @param blobStorage - Blob storage instance
* @param hash1 - First commit hash
* @param hash2 - Second commit hash
* @returns Common ancestor commit, or null if not found
*/
static async findCommonAncestor(
blobStorage: BlobStorage,
hash1: string,
hash2: string
): Promise<CommitObject | null> {
// Build set of all ancestors of commit1
const ancestors1 = new Set<string>()
for await (const commit of CommitObject.walk(blobStorage, hash1)) {
ancestors1.add(CommitObject.hash(commit))
}
// Walk commit2 history, find first commit in ancestors1
for await (const commit of CommitObject.walk(blobStorage, hash2)) {
const commitHash = CommitObject.hash(commit)
if (ancestors1.has(commitHash)) {
return commit
}
}
return null
}
/**
* Count commits between two commits
*
* @param blobStorage - Blob storage instance
* @param fromHash - Starting commit hash
* @param toHash - Ending commit hash
* @returns Number of commits between (inclusive)
*/
static async countBetween(
blobStorage: BlobStorage,
fromHash: string,
toHash: string
): Promise<number> {
let count = 0
for await (const commit of CommitObject.walk(blobStorage, fromHash, {
stopAt: toHash
})) {
count++
}
return count
}
/**
* Get commits in time range
*
* @param blobStorage - Blob storage instance
* @param startHash - Starting commit hash
* @param startTime - Start of time range
* @param endTime - End of time range
* @returns Array of commits in range
*/
static async getInTimeRange(
blobStorage: BlobStorage,
startHash: string,
startTime: number,
endTime: number
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
for await (const commit of CommitObject.walk(blobStorage, startHash, {
until: startTime
})) {
if (commit.timestamp >= startTime && commit.timestamp <= endTime) {
commits.push(commit)
}
}
return commits
}
/**
* Get commits by author
*
* @param blobStorage - Blob storage instance
* @param startHash - Starting commit hash
* @param author - Author name
* @param options - Walk options
* @returns Array of commits by author
*/
static async getByAuthor(
blobStorage: BlobStorage,
startHash: string,
author: string,
options?: { maxDepth?: number }
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
for await (const commit of CommitObject.walk(blobStorage, startHash, {
...options,
filter: c => c.author === author
})) {
commits.push(commit)
}
return commits
}
/**
* Get commits by operation type
*
* @param blobStorage - Blob storage instance
* @param startHash - Starting commit hash
* @param operation - Operation type (e.g., 'add', 'update', 'delete')
* @param options - Walk options
* @returns Array of commits by operation
*/
static async getByOperation(
blobStorage: BlobStorage,
startHash: string,
operation: string,
options?: { maxDepth?: number }
): Promise<CommitObject[]> {
const commits: CommitObject[] = []
for await (const commit of CommitObject.walk(blobStorage, startHash, {
...options,
filter: c => c.metadata?.operation === operation
})) {
commits.push(commit)
}
return commits
}
}

547
src/storage/cow/RefManager.ts
View file

@ -1,547 +0,0 @@
/**
* RefManager: Branch and reference management for COW (Copy-on-Write)
*
* Similar to Git refs, manages symbolic names (branches, tags) that point to commits.
*
* Structure:
* - refs/heads/main → commit hash (main branch)
* - refs/heads/experiment → commit hash (experiment branch)
* - refs/tags/v1.0.0 → commit hash (version tag)
* - HEAD → ref name (current branch)
*
* Features:
* - Branch management (create, delete, list)
* - Tag management
* - HEAD pointer (current branch)
* - Fast-forward and force updates
* - Atomic operations
*
* @module storage/cow/RefManager
*/
import type { COWStorageAdapter } from './BlobStorage.js'
import { compareCodePoints } from '../../utils/collation.js'
/**
* Reference type
*/
export type RefType = 'branch' | 'tag' | 'remote'
/**
* Reference object
*/
export interface Ref {
name: string // Full ref name (e.g., 'refs/heads/main')
commitHash: string // Commit hash this ref points to
type: RefType // Reference type
createdAt: number // When ref was created
updatedAt: number // When ref was last updated
metadata?: {
description?: string
author?: string
[key: string]: any
}
}
/**
* Ref update options
*/
export interface RefUpdateOptions {
force?: boolean // Force update (allow non-fast-forward)
createOnly?: boolean // Only create, fail if exists
updateOnly?: boolean // Only update, fail if doesn't exist
expectedOldValue?: string // CAS: only update if current value matches
}
/**
* RefManager: Manages branches, tags, and HEAD pointer
*
* Pure implementation - no backward compatibility
*/
export class RefManager {
private adapter: COWStorageAdapter
private cache: Map<string, Ref>
private cacheValid: boolean
constructor(adapter: COWStorageAdapter) {
this.adapter = adapter
this.cache = new Map()
this.cacheValid = false
}
/**
* Get reference by name
*
* @param name - Reference name (e.g., 'main', 'refs/heads/main')
* @returns Reference object or undefined
*/
async getRef(name: string): Promise<Ref | undefined> {
const fullName = this.normalizeRefName(name)
// Check cache
if (this.cacheValid && this.cache.has(fullName)) {
return this.cache.get(fullName)
}
// Read from storage
const data = await this.adapter.get(`ref:${fullName}`)
if (!data) {
return undefined
}
const ref = JSON.parse(data.toString()) as Ref
// Update cache
this.cache.set(fullName, ref)
return ref
}
/**
* Set reference to point to commit
*
* @param name - Reference name
* @param commitHash - Commit hash to point to
* @param options - Update options
*/
async setRef(
name: string,
commitHash: string,
options: RefUpdateOptions = {}
): Promise<void> {
const fullName = this.normalizeRefName(name)
// Validate commit hash format
if (!/^[a-f0-9]{64}$/.test(commitHash)) {
throw new Error(`Invalid commit hash: ${commitHash}`)
}
// Check if ref exists
const existing = await this.getRef(fullName)
// Handle createOnly
if (options.createOnly && existing) {
throw new Error(`Ref already exists: ${fullName}`)
}
// Handle updateOnly
if (options.updateOnly && !existing) {
throw new Error(`Ref does not exist: ${fullName}`)
}
// Handle CAS (Compare-And-Swap)
if (options.expectedOldValue !== undefined) {
if (!existing || existing.commitHash !== options.expectedOldValue) {
throw new Error(
`Ref update failed: expected ${options.expectedOldValue}, ` +
`got ${existing?.commitHash ?? 'none'}`
)
}
}
// Fast-forward verification is intentionally permissive: COW is a
// single-writer subsystem locked by the storage adapter's writer lock,
// so out-of-order updates can't reach this point. If multi-writer COW
// is ever introduced, replace with a commit-ancestry walk.
// Create/update ref
const ref: Ref = {
name: fullName,
commitHash,
type: this.getRefType(fullName),
createdAt: existing?.createdAt ?? Date.now(),
updatedAt: Date.now(),
metadata: existing?.metadata
}
// Write to storage
await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref)))
// Update cache
this.cache.set(fullName, ref)
this.cacheValid = false // Invalidate for listRefs
}
/**
* Delete reference
*
* @param name - Reference name
*/
async deleteRef(name: string): Promise<void> {
const fullName = this.normalizeRefName(name)
// Don't allow deleting HEAD
if (fullName === 'HEAD') {
throw new Error('Cannot delete HEAD')
}
// Don't allow deleting main if it's the only branch
if (fullName === 'refs/heads/main') {
const branches = await this.listRefs('branch')
if (branches.length === 1) {
throw new Error('Cannot delete last branch')
}
}
// Delete from storage
await this.adapter.delete(`ref:${fullName}`)
// Update cache
this.cache.delete(fullName)
this.cacheValid = false
}
/**
* List all references
*
* @param type - Filter by type (optional)
* @returns Array of references
*/
async listRefs(type?: RefType): Promise<Ref[]> {
// Get all ref keys
const keys = await this.adapter.list('ref:')
const refs: Ref[] = []
for (const key of keys) {
const refName = key.replace(/^ref:/, '')
// Skip HEAD in listings (it's special)
if (refName === 'HEAD') {
continue
}
const ref = await this.getRef(refName)
if (ref) {
// Filter by type if requested
if (!type || ref.type === type) {
refs.push(ref)
}
}
}
// Mark cache as valid
this.cacheValid = true
return refs.sort((a, b) => compareCodePoints(a.name, b.name))
}
/**
* Copy reference (create branch from existing ref)
*
* @param sourceName - Source reference name
* @param targetName - Target reference name
* @param options - Update options
*/
async copyRef(
sourceName: string,
targetName: string,
options: RefUpdateOptions = {}
): Promise<void> {
const sourceRef = await this.getRef(sourceName)
if (!sourceRef) {
throw new Error(`Source ref not found: ${sourceName}`)
}
// Set target ref to same commit as source
await this.setRef(targetName, sourceRef.commitHash, options)
}
/**
* Get current HEAD (current branch)
*
* @returns HEAD reference or undefined
*/
async getHead(): Promise<Ref | undefined> {
const data = await this.adapter.get('ref:HEAD')
if (!data) {
return undefined
}
const head = JSON.parse(data.toString()) as { ref: string }
// Resolve symbolic ref
return this.getRef(head.ref)
}
/**
* Set HEAD to point to branch
*
* @param branchName - Branch name (e.g., 'main', 'refs/heads/experiment')
*/
async setHead(branchName: string): Promise<void> {
const fullName = this.normalizeRefName(branchName)
// Verify branch exists
const branch = await this.getRef(fullName)
if (!branch) {
throw new Error(`Branch not found: ${fullName}`)
}
if (branch.type !== 'branch') {
throw new Error(`Cannot set HEAD to non-branch ref: ${fullName}`)
}
// Set HEAD (symbolic ref)
const head = { ref: fullName }
await this.adapter.put('ref:HEAD', Buffer.from(JSON.stringify(head)))
}
/**
* Get current commit hash (resolves HEAD)
*
* @returns Current commit hash or undefined
*/
async getCurrentCommit(): Promise<string | undefined> {
const head = await this.getHead()
return head?.commitHash
}
/**
* Create branch
*
* @param name - Branch name (e.g., 'experiment')
* @param commitHash - Commit hash to point to
* @param options - Create options
*/
async createBranch(
name: string,
commitHash: string,
options?: { description?: string; author?: string }
): Promise<void> {
const fullName = this.normalizeRefName(name, 'branch')
await this.setRef(fullName, commitHash, { createOnly: true })
// Update metadata if provided
if (options?.description || options?.author) {
const ref = await this.getRef(fullName)
if (ref) {
ref.metadata = {
...ref.metadata,
description: options.description,
author: options.author
}
await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref)))
}
}
}
/**
* Delete branch
*
* @param name - Branch name
*/
async deleteBranch(name: string): Promise<void> {
const fullName = this.normalizeRefName(name, 'branch')
await this.deleteRef(fullName)
}
/**
* List all branches
*
* @returns Array of branch references
*/
async listBranches(): Promise<Ref[]> {
return this.listRefs('branch')
}
/**
* Create tag
*
* @param name - Tag name (e.g., 'v1.0.0')
* @param commitHash - Commit hash to point to
* @param options - Create options
*/
async createTag(
name: string,
commitHash: string,
options?: { description?: string; author?: string }
): Promise<void> {
const fullName = this.normalizeRefName(name, 'tag')
await this.setRef(fullName, commitHash, { createOnly: true })
// Update metadata if provided
if (options?.description || options?.author) {
const ref = await this.getRef(fullName)
if (ref) {
ref.metadata = {
...ref.metadata,
description: options.description,
author: options.author
}
await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref)))
}
}
}
/**
* Delete tag
*
* @param name - Tag name
*/
async deleteTag(name: string): Promise<void> {
const fullName = this.normalizeRefName(name, 'tag')
await this.deleteRef(fullName)
}
/**
* List all tags
*
* @returns Array of tag references
*/
async listTags(): Promise<Ref[]> {
return this.listRefs('tag')
}
/**
* Check if reference exists
*
* @param name - Reference name
* @returns True if reference exists
*/
async hasRef(name: string): Promise<boolean> {
const ref = await this.getRef(name)
return ref !== undefined
}
/**
* Update reference to new commit (with validation)
*
* @param name - Reference name
* @param newCommitHash - New commit hash
* @param oldCommitHash - Expected old commit hash (for safety)
*/
async updateRef(
name: string,
newCommitHash: string,
oldCommitHash?: string
): Promise<void> {
const options: RefUpdateOptions = {}
if (oldCommitHash) {
options.expectedOldValue = oldCommitHash
}
await this.setRef(name, newCommitHash, options)
}
/**
* Update metadata on an existing ref (merge semantics).
*
* @param name - Reference name
* @param metadata - Metadata fields to merge into the ref
*/
async updateRefMetadata(name: string, metadata: Record<string, unknown>): Promise<void> {
const fullName = this.normalizeRefName(name)
const ref = await this.getRef(fullName)
if (!ref) throw new Error(`Ref not found: ${fullName}`)
ref.metadata = { ...ref.metadata, ...metadata }
ref.updatedAt = Date.now()
await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref)))
this.cache.set(fullName, ref)
}
/**
* Get commit hash for reference
*
* @param name - Reference name
* @returns Commit hash or undefined
*/
async resolveRef(name: string): Promise<string | undefined> {
const ref = await this.getRef(name)
return ref?.commitHash
}
/**
* Find references pointing to commit
*
* @param commitHash - Commit hash
* @returns Array of references pointing to this commit
*/
async findRefsPointingTo(commitHash: string): Promise<Ref[]> {
const allRefs = await this.listRefs()
return allRefs.filter(ref => ref.commitHash === commitHash)
}
/**
* Clear cache (useful for testing)
*/
clearCache(): void {
this.cache.clear()
this.cacheValid = false
}
// ========== PRIVATE METHODS ==========
/**
* Normalize reference name to full format
*
* Examples:
* - 'main' → 'refs/heads/main'
* - 'v1.0.0' (with type='tag') → 'refs/tags/v1.0.0'
* - 'refs/heads/experiment' → 'refs/heads/experiment'
*
* @param name - Reference name
* @param type - Reference type hint
* @returns Full reference name
*/
private normalizeRefName(name: string, type?: RefType): string {
// Already full format
if (name.startsWith('refs/')) {
return name
}
// HEAD is special
if (name === 'HEAD') {
return 'HEAD'
}
// Infer type from name if not provided
if (!type) {
// Tags usually start with 'v' or contain dots
if (name.startsWith('v') && /\d/.test(name)) {
type = 'tag'
} else {
type = 'branch' // Default to branch
}
}
// Add prefix
switch (type) {
case 'branch':
return `refs/heads/${name}`
case 'tag':
return `refs/tags/${name}`
case 'remote':
return `refs/remotes/${name}`
default:
return name
}
}
/**
* Get reference type from full name
*
* @param fullName - Full reference name
* @returns Reference type
*/
private getRefType(fullName: string): RefType {
if (fullName.startsWith('refs/heads/')) {
return 'branch'
} else if (fullName.startsWith('refs/tags/')) {
return 'tag'
} else if (fullName.startsWith('refs/remotes/')) {
return 'remote'
} else {
return 'branch' // Default
}
}
}

359
src/storage/cow/TreeObject.ts
View file

@ -1,359 +0,0 @@
/**
* TreeObject: Directory structure for COW (Copy-on-Write)
*
* Similar to Git trees, represents the structure of a Brainy instance at a point in time.
* Trees contain entries mapping names to blob hashes.
*
* Structure:
* - entities/ → tree hash (entity blobs)
* - indexes/nouns → blob hash (HNSW noun index)
* - indexes/metadata → blob hash (metadata index)
* - indexes/graph → blob hash (graph adjacency index)
* - indexes/deleted → blob hash (deleted items index)
*
* @module storage/cow/TreeObject
*/
import { BlobStorage } from './BlobStorage.js'
import { compareCodePoints } from '../../utils/collation.js'
/**
* Tree entry: name → blob hash mapping
*/
export interface TreeEntry {
name: string
hash: string
type: 'blob' | 'tree' // blob = leaf, tree = subtree
size: number // Original size (uncompressed)
}
/**
* Tree object structure
*/
export interface TreeObject {
entries: TreeEntry[]
createdAt: number
}
/**
* TreeBuilder: Fluent API for building tree objects
*
* Example:
* ```typescript
* const tree = await TreeBuilder.create(blobStorage)
* .addBlob('entities/abc123', entityHash, size)
* .addBlob('indexes/nouns', nounsHash, size)
* .build()
* ```
*/
export class TreeBuilder {
private entries: TreeEntry[] = []
private blobStorage: BlobStorage
constructor(blobStorage: BlobStorage) {
this.blobStorage = blobStorage
}
static create(blobStorage: BlobStorage): TreeBuilder {
return new TreeBuilder(blobStorage)
}
/**
* Add a blob entry to the tree
*
* @param name - Entry name (e.g., 'entities/abc123')
* @param hash - Blob hash
* @param size - Original blob size
*/
addBlob(name: string, hash: string, size: number): this {
this.entries.push({
name,
hash,
type: 'blob',
size
})
return this
}
/**
* Add a subtree entry to the tree
*
* @param name - Subtree name (e.g., 'entities/')
* @param treeHash - Tree hash
* @param size - Total size of subtree
*/
addTree(name: string, treeHash: string, size: number): this {
this.entries.push({
name,
hash: treeHash,
type: 'tree',
size
})
return this
}
/**
* Build and persist the tree object
*
* @returns Tree hash
*/
async build(): Promise<string> {
const tree: TreeObject = {
// Code-point (UTF-8 byte) order so the serialized tree — and therefore its
// content hash — is reproducible across environments (localeCompare's default
// locale varies by OS/Node/ICU). Sorting happens only here at write time;
// deserialize() preserves stored order, so existing trees keep their hashes
// and stay valid — no migration needed.
entries: this.entries.sort((a, b) => compareCodePoints(a.name, b.name)),
createdAt: Date.now()
}
return TreeObject.write(this.blobStorage, tree)
}
}
/**
* TreeObject: Represents directory structure in COW storage
*/
export class TreeObject {
/**
* Serialize tree object to Buffer
*
* Format: JSON (simple, debuggable)
* Future: Could use protobuf for efficiency
*
* @param tree - Tree object
* @returns Serialized tree
*/
static serialize(tree: TreeObject): Buffer {
return Buffer.from(JSON.stringify(tree, null, 0)) // Compact JSON
}
/**
* Deserialize tree object from Buffer
*
* @param data - Serialized tree
* @returns Tree object
*/
static deserialize(data: Buffer): TreeObject {
const tree = JSON.parse(data.toString())
// Validate structure
if (!tree.entries || !Array.isArray(tree.entries)) {
throw new Error('Invalid tree object: missing entries array')
}
if (typeof tree.createdAt !== 'number') {
throw new Error('Invalid tree object: missing or invalid createdAt')
}
// Validate entries
for (const entry of tree.entries) {
if (typeof entry.name !== 'string') {
throw new Error(`Invalid tree entry: name must be string`)
}
if (typeof entry.hash !== 'string' || entry.hash.length !== 64) {
throw new Error(`Invalid tree entry: hash must be 64-char SHA-256`)
}
if (entry.type !== 'blob' && entry.type !== 'tree') {
throw new Error(`Invalid tree entry: type must be 'blob' or 'tree'`)
}
if (typeof entry.size !== 'number' || entry.size < 0) {
throw new Error(`Invalid tree entry: size must be non-negative number`)
}
}
return tree
}
/**
* Compute hash of tree object
*
* @param tree - Tree object
* @returns SHA-256 hash
*/
static hash(tree: TreeObject): string {
const data = TreeObject.serialize(tree)
return BlobStorage.hash(data)
}
/**
* Write tree object to blob storage
*
* @param blobStorage - Blob storage instance
* @param tree - Tree object
* @returns Tree hash
*/
static async write(blobStorage: BlobStorage, tree: TreeObject): Promise<string> {
const data = TreeObject.serialize(tree)
return blobStorage.write(data, { type: 'tree', compression: 'auto' })
}
/**
* Read tree object from blob storage
*
* @param blobStorage - Blob storage instance
* @param hash - Tree hash
* @returns Tree object
*/
static async read(blobStorage: BlobStorage, hash: string): Promise<TreeObject> {
const data = await blobStorage.read(hash)
return TreeObject.deserialize(data)
}
/**
* Get specific entry from tree
*
* @param tree - Tree object
* @param name - Entry name
* @returns Tree entry or undefined
*/
static getEntry(tree: TreeObject, name: string): TreeEntry | undefined {
return tree.entries.find(e => e.name === name)
}
/**
* Get all blob entries from tree (non-recursive)
*
* @param tree - Tree object
* @returns Array of blob entries
*/
static getBlobs(tree: TreeObject): TreeEntry[] {
return tree.entries.filter(e => e.type === 'blob')
}
/**
* Get all subtree entries from tree (non-recursive)
*
* @param tree - Tree object
* @returns Array of tree entries
*/
static getSubtrees(tree: TreeObject): TreeEntry[] {
return tree.entries.filter(e => e.type === 'tree')
}
/**
* Walk tree recursively, yielding all blob entries
*
* @param blobStorage - Blob storage instance
* @param tree - Tree object
*/
static async *walk(
blobStorage: BlobStorage,
tree: TreeObject
): AsyncIterableIterator<TreeEntry> {
for (const entry of tree.entries) {
if (entry.type === 'blob') {
yield entry
} else {
// Recurse into subtree
const subtree = await TreeObject.read(blobStorage, entry.hash)
yield* TreeObject.walk(blobStorage, subtree)
}
}
}
/**
* Compute total size of tree (recursive)
*
* @param blobStorage - Blob storage instance
* @param tree - Tree object
* @returns Total size in bytes
*/
static async getTotalSize(blobStorage: BlobStorage, tree: TreeObject): Promise<number> {
let totalSize = 0
for await (const entry of TreeObject.walk(blobStorage, tree)) {
totalSize += entry.size
}
return totalSize
}
/**
* Create a new tree by updating a single entry
* (Copy-on-write: creates new tree, doesn't modify original)
*
* @param blobStorage - Blob storage instance
* @param tree - Original tree
* @param name - Entry name to update
* @param hash - New blob/tree hash
* @param size - New size
* @returns New tree hash
*/
static async updateEntry(
blobStorage: BlobStorage,
tree: TreeObject,
name: string,
hash: string,
size: number
): Promise<string> {
const builder = TreeBuilder.create(blobStorage)
// Copy all entries except the one being updated
for (const entry of tree.entries) {
if (entry.name === name) {
// Replace with new entry
if (entry.type === 'blob') {
builder.addBlob(name, hash, size)
} else {
builder.addTree(name, hash, size)
}
} else {
// Keep existing entry
if (entry.type === 'blob') {
builder.addBlob(entry.name, entry.hash, entry.size)
} else {
builder.addTree(entry.name, entry.hash, entry.size)
}
}
}
// If entry didn't exist, add it
if (!TreeObject.getEntry(tree, name)) {
builder.addBlob(name, hash, size)
}
return builder.build()
}
/**
* Diff two trees, return changed/added/deleted entries
*
* @param tree1 - First tree (base)
* @param tree2 - Second tree (comparison)
* @returns Diff result
*/
static diff(tree1: TreeObject, tree2: TreeObject): {
added: TreeEntry[]
modified: TreeEntry[]
deleted: TreeEntry[]
} {
const entries1 = new Map(tree1.entries.map(e => [e.name, e]))
const entries2 = new Map(tree2.entries.map(e => [e.name, e]))
const added: TreeEntry[] = []
const modified: TreeEntry[] = []
const deleted: TreeEntry[] = []
// Find added and modified
for (const [name, entry2] of entries2) {
const entry1 = entries1.get(name)
if (!entry1) {
added.push(entry2)
} else if (entry1.hash !== entry2.hash) {
modified.push(entry2)
}
}
// Find deleted
for (const [name, entry1] of entries1) {
if (!entries2.has(name)) {
deleted.push(entry1)
}
}
return { added, modified, deleted }
}
}

100
src/storage/cow/binaryDataCodec.ts
View file

@ -1,100 +0,0 @@
/**
* Binary Data Codec: Single Source of Truth for Wrap/Unwrap Operations
*
* This module provides the ONLY implementation of binary data encoding/decoding
* used across all storage adapters and blob storage.
*
* Design Principles:
* - DRY: One implementation, used everywhere
* - Single Responsibility: Only handles binary ↔ JSON conversion
* - Type-Safe: Proper TypeScript types
* - Defensive: Handles all edge cases
*
* Used by:
* - BaseStorage COW adapter (write/read operations)
* - BlobStorage (defense-in-depth verification)
* - All storage adapters (via BaseStorage)
*
* @module storage/cow/binaryDataCodec
*/
/**
* Wrapped binary data format
* Used when storing binary data in JSON-based storage
*/
export interface WrappedBinaryData {
_binary: true
data: string // base64-encoded
}
/**
* Check if data is wrapped binary format
*/
export function isWrappedBinary(data: any): data is WrappedBinaryData {
return (
typeof data === 'object' &&
data !== null &&
data._binary === true &&
typeof data.data === 'string'
)
}
/**
* Unwrap binary data from JSON wrapper
*
* This is the SINGLE SOURCE OF TRUTH for unwrapping binary data.
* All storage operations MUST use this function.
*
* Handles:
* - Buffer → Buffer (pass-through)
* - {_binary: true, data: "base64..."} → Buffer (unwrap)
* - Plain object → Buffer (JSON stringify)
* - Other types → Error
*
* @param data - Data to unwrap (may be Buffer, wrapped object, or plain object)
* @returns Unwrapped Buffer
* @throws Error if data type is invalid
*/
export function unwrapBinaryData(data: any): Buffer {
// Case 1: Already a Buffer (no unwrapping needed)
if (Buffer.isBuffer(data)) {
return data
}
// Case 2: Wrapped binary data {_binary: true, data: "base64..."}
if (isWrappedBinary(data)) {
return Buffer.from(data.data, 'base64')
}
// Case 3: Plain object (shouldn't happen for binary blobs, but handle gracefully)
if (typeof data === 'object' && data !== null) {
return Buffer.from(JSON.stringify(data))
}
// Case 4: String (convert to Buffer)
if (typeof data === 'string') {
return Buffer.from(data)
}
// Case 5: Invalid type
throw new Error(
`Invalid data type for unwrap: ${typeof data}. ` +
`Expected Buffer or {_binary: true, data: "base64..."}`
)
}
/**
* Ensure data is a Buffer
*
* Convenience function that combines type checking and unwrapping.
* Use this when you need to ensure you have a Buffer.
*
* @param data - Data that should be or can be converted to Buffer
* @returns Buffer
*/
export function ensureBuffer(data: any): Buffer {
if (Buffer.isBuffer(data)) {
return data
}
return unwrapBinaryData(data)
}

59
src/storage/cow/constants.ts
View file

@ -1,59 +0,0 @@
/**
* COW Storage Constants
*
* Sentinel values and utilities for Copy-On-Write storage system.
*
* @module storage/cow/constants
*/
/**
* NULL_HASH - Sentinel value for "no parent commit" or "empty tree"
*
* In Git-like COW systems, we need a way to represent:
* - Initial commit (has no parent)
* - Empty tree (contains no files)
*
* We use a 64-character zero hash as a sentinel value.
* This should NEVER be used as an actual content hash.
*
* @constant
* @example
* ```typescript
* const builder = CommitBuilder.create(storage)
* .tree(NULL_HASH) // Empty tree
* .parent(null) // No parent (use null, not NULL_HASH)
* .build()
* ```
*/
export const NULL_HASH = '0000000000000000000000000000000000000000000000000000000000000000'
/**
* Check if a hash is the NULL sentinel value
*
* @param hash - Hash to check (can be string or null)
* @returns true if hash is null or NULL_HASH
*
* @example
* ```typescript
* if (isNullHash(commit.parent)) {
* console.log('This is the initial commit')
* }
* ```
*/
export function isNullHash(hash: string | null | undefined): boolean {
return hash === null || hash === undefined || hash === NULL_HASH
}
/**
* Check if a hash is valid (non-null, non-empty, proper format)
*
* @param hash - Hash to check
* @returns true if hash is a valid SHA-256 hash
*/
export function isValidHash(hash: string | null | undefined): boolean {
if (isNullHash(hash)) {
return false
}
// SHA-256 hash must be exactly 64 hexadecimal characters
return typeof hash === 'string' && /^[a-f0-9]{64}$/.test(hash)
}

24
src/storage/storageFactory.ts
View file

@ -52,30 +52,10 @@ export interface StorageOptions {
[key: string]: any [key: string]: any
} }
/** Branch name for COW storage (filesystem only). */
branch?: string
/** Whether to enable COW blob compression. Default `true`. */
enableCompression?: boolean
/** Operation tuning (timeouts, retry budgets) passed through to the adapter. */ /** Operation tuning (timeouts, retry budgets) passed through to the adapter. */
operationConfig?: OperationConfig operationConfig?: OperationConfig
} }
/**
* Attach the COW (copy-on-write) options to the adapter so the storage's
* branch + compression settings are honored when the adapter's
* `initializeCOW()` hook fires later in `brain.init()`.
*/
function configureCOW(storage: any, options?: StorageOptions): void {
if (typeof storage?.initializeCOW === 'function') {
storage._cowOptions = {
branch: options?.branch || 'main',
enableCompression: options?.enableCompression !== false
}
}
}
/** /**
* Resolve `StorageOptions` to a concrete storage adapter. * Resolve `StorageOptions` to a concrete storage adapter.
* *
@ -85,9 +65,7 @@ function configureCOW(storage: any, options?: StorageOptions): void {
* - `forceFileSystemStorage: true` returns `FileSystemStorage`. * - `forceFileSystemStorage: true` returns `FileSystemStorage`.
*/ */
export async function createStorage(options: StorageOptions = {}): Promise<StorageAdapter> { export async function createStorage(options: StorageOptions = {}): Promise<StorageAdapter> {
const storage = await pickAdapter(options) return pickAdapter(options)
configureCOW(storage, options)
return storage
} }
async function pickAdapter(options: StorageOptions): Promise<StorageAdapter> { async function pickAdapter(options: StorageOptions): Promise<StorageAdapter> {

3
src/transaction/operations/IndexOperations.ts
View file

@ -20,8 +20,7 @@ import type { Operation, RollbackAction } from '../types.js'
* *
* Rollback strategy: * Rollback strategy:
* - Remove item from index * - Remove item from index
*
* Note: Works with both JsHnswVectorIndex and TypeAwareHNSWIndex
*/ */
export class AddToHNSWOperation implements Operation { export class AddToHNSWOperation implements Operation {
readonly name = 'AddToHNSW' readonly name = 'AddToHNSW'

3
src/transaction/operations/StorageOperations.ts
View file

@ -5,10 +5,9 @@
* Each operation can be executed and rolled back atomically. * Each operation can be executed and rolled back atomically.
* *
* Supports: * Supports:
* - All 8 storage adapters (FileSystem, S3, Memory, OPFS, GCS, Azure, R2, TypeAware) * - Both storage adapters (FileSystem, Memory)
* - Nouns (entities) and Verbs (relationships) * - Nouns (entities) and Verbs (relationships)
* - Metadata and vector data * - Metadata and vector data
* - COW (Copy-on-Write) storage
*/ */
import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js' import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js'

1
src/types/brainy.types.ts
View file

@ -1099,7 +1099,6 @@ export interface BrainyConfig {
* including plugin-provided factories (e.g. Cortex mmap). */ * including plugin-provided factories (e.g. Cortex mmap). */
rootDirectory?: string rootDirectory?: string
options?: any options?: any
branch?: string // COW branch name (default: 'main')
} }
// Index configuration // Index configuration

9
src/types/brainyInterface.ts
View file

@ -164,7 +164,8 @@ export interface BrainyInterface<T = unknown> {
/** /**
* Run pending data migrations, or preview what would change. * Run pending data migrations, or preview what would change.
* *
* @param options - Pass { dryRun: true } to preview without writing * @param options - Pass `{ dryRun: true }` to preview without writing;
* pass `{ backupTo: path }` to persist a pre-migration snapshot.
* @returns Migration result or preview depending on options * @returns Migration result or preview depending on options
* *
* @example * @example
@ -173,9 +174,9 @@ export interface BrainyInterface<T = unknown> {
* const preview = await brain.migrate({ dryRun: true }) * const preview = await brain.migrate({ dryRun: true })
* console.log(preview.affectedEntities) * console.log(preview.affectedEntities)
* *
* // Apply * // Apply with a restore point
* const result = await brain.migrate() * const result = await brain.migrate({ backupTo: '/backups/pre-migration' })
* console.log(result.backupBranch) // 'pre-migration-7.17.0' * console.log(result.backupPath) // '/backups/pre-migration'
* ``` * ```
*/ */
migrate(options?: MigrateOptions): Promise<MigrationResult | MigrationPreview> migrate(options?: MigrateOptions): Promise<MigrationResult | MigrationPreview>

459
src/versioning/VersionDiff.ts
View file

@ -1,459 +0,0 @@
/**
* VersionDiff - Deep Object Comparison for Entity Versions
*
* Provides deep diff between entity versions:
* - Field-level change detection
* - Nested object comparison
* - Array diffing
* - Type change detection
* - Human-readable diff output
*
* NO MOCKS - Production implementation
*/
import type { NounMetadata } from '../coreTypes.js'
/**
* Types of changes in a diff
*/
export type ChangeType = 'added' | 'removed' | 'modified' | 'type-changed'
/**
* A single field change in a diff
*/
export interface FieldChange {
/** Path to the field (e.g., 'metadata.user.name') */
path: string
/** Type of change */
type: ChangeType
/** Old value (undefined for 'added') */
oldValue?: any
/** New value (undefined for 'removed') */
newValue?: any
/** Old type (for 'type-changed') */
oldType?: string
/** New type (for 'type-changed') */
newType?: string
}
/**
* Complete diff between two versions
*/
export interface VersionDiff {
/** Entity ID being compared */
entityId: string
/** From version number */
fromVersion: number
/** To version number */
toVersion: number
/** Fields that were added */
added: FieldChange[]
/** Fields that were removed */
removed: FieldChange[]
/** Fields that were modified */
modified: FieldChange[]
/** Fields whose type changed */
typeChanged: FieldChange[]
/** Total number of changes */
totalChanges: number
/** Whether versions are identical */
identical: boolean
}
/**
* Options for diff comparison
*/
export interface DiffOptions {
/** Entity ID (for context in output) */
entityId: string
/** From version number */
fromVersion: number
/** To version number */
toVersion: number
/** Ignore these fields in comparison */
ignoreFields?: string[]
/** Maximum depth for nested object comparison (default: 10) */
maxDepth?: number
/** Include unchanged fields in output (default: false) */
includeUnchanged?: boolean
}
/**
* Compare two entity versions and generate diff
*
* @param from Old version entity
* @param to New version entity
* @param options Diff options
* @returns Diff between versions
*/
export function compareEntityVersions(
from: NounMetadata,
to: NounMetadata,
options: DiffOptions
): VersionDiff {
const added: FieldChange[] = []
const removed: FieldChange[] = []
const modified: FieldChange[] = []
const typeChanged: FieldChange[] = []
const ignoreFields = new Set(options.ignoreFields || [])
const maxDepth = options.maxDepth ?? 10
// Compare objects recursively
compareObjects(from, to, '', added, removed, modified, typeChanged, ignoreFields, 0, maxDepth)
const totalChanges = added.length + removed.length + modified.length + typeChanged.length
const identical = totalChanges === 0
return {
entityId: options.entityId,
fromVersion: options.fromVersion,
toVersion: options.toVersion,
added,
removed,
modified,
typeChanged,
totalChanges,
identical
}
}
/**
* Recursively compare two objects
*/
function compareObjects(
from: any,
to: any,
path: string,
added: FieldChange[],
removed: FieldChange[],
modified: FieldChange[],
typeChanged: FieldChange[],
ignoreFields: Set<string>,
depth: number,
maxDepth: number
): void {
if (depth >= maxDepth) {
// Hit max depth - treat as single value
if (!deepEqual(from, to)) {
modified.push({
path,
type: 'modified',
oldValue: from,
newValue: to
})
}
return
}
// Get all keys from both objects
const fromKeys = new Set(Object.keys(from || {}))
const toKeys = new Set(Object.keys(to || {}))
const allKeys = new Set([...fromKeys, ...toKeys])
for (const key of allKeys) {
const fieldPath = path ? `${path}.${key}` : key
// Skip ignored fields
if (ignoreFields.has(fieldPath) || ignoreFields.has(key)) {
continue
}
const fromHas = fromKeys.has(key)
const toHas = toKeys.has(key)
if (!fromHas && toHas) {
// Field added
added.push({
path: fieldPath,
type: 'added',
newValue: to[key]
})
} else if (fromHas && !toHas) {
// Field removed
removed.push({
path: fieldPath,
type: 'removed',
oldValue: from[key]
})
} else {
// Field exists in both - check for changes
const fromValue = from[key]
const toValue = to[key]
const fromType = getValueType(fromValue)
const toType = getValueType(toValue)
if (fromType !== toType) {
// Type changed
typeChanged.push({
path: fieldPath,
type: 'type-changed',
oldValue: fromValue,
newValue: toValue,
oldType: fromType,
newType: toType
})
} else if (fromType === 'object' && toType === 'object') {
// Recursively compare nested objects
compareObjects(
fromValue,
toValue,
fieldPath,
added,
removed,
modified,
typeChanged,
ignoreFields,
depth + 1,
maxDepth
)
} else if (fromType === 'array' && toType === 'array') {
// Compare arrays
if (!arraysEqual(fromValue, toValue)) {
modified.push({
path: fieldPath,
type: 'modified',
oldValue: fromValue,
newValue: toValue
})
}
} else {
// Primitive value comparison
if (!deepEqual(fromValue, toValue)) {
modified.push({
path: fieldPath,
type: 'modified',
oldValue: fromValue,
newValue: toValue
})
}
}
}
}
}
/**
* Get human-readable type of a value
*/
function getValueType(value: any): string {
if (value === null) return 'null'
if (value === undefined) return 'undefined'
if (Array.isArray(value)) return 'array'
return typeof value
}
/**
* Deep equality check
*/
function deepEqual(a: any, b: any): boolean {
if (a === b) return true
if (a === null || b === null) return false
if (a === undefined || b === undefined) return false
const typeA = getValueType(a)
const typeB = getValueType(b)
if (typeA !== typeB) return false
if (typeA === 'array') {
return arraysEqual(a, b)
}
if (typeA === 'object') {
return objectsEqual(a, b)
}
// Primitive comparison
return a === b
}
/**
* Compare arrays for equality
*/
function arraysEqual(a: any[], b: any[]): boolean {
if (a.length !== b.length) return false
for (let i = 0; i < a.length; i++) {
if (!deepEqual(a[i], b[i])) {
return false
}
}
return true
}
/**
* Compare objects for equality
*/
function objectsEqual(a: any, b: any): boolean {
const keysA = Object.keys(a)
const keysB = Object.keys(b)
if (keysA.length !== keysB.length) return false
for (const key of keysA) {
if (!keysB.includes(key)) return false
if (!deepEqual(a[key], b[key])) return false
}
return true
}
/**
* Format diff as human-readable string
*
* @param diff Diff to format
* @returns Formatted string
*/
export function formatDiff(diff: VersionDiff): string {
const lines: string[] = []
lines.push(`Diff: ${diff.entityId} v${diff.fromVersion} → v${diff.toVersion}`)
lines.push('')
if (diff.identical) {
lines.push('No changes')
return lines.join('\n')
}
lines.push(`Total changes: ${diff.totalChanges}`)
lines.push('')
if (diff.added.length > 0) {
lines.push(`Added (${diff.added.length}):`)
for (const change of diff.added) {
lines.push(` + ${change.path}: ${formatValue(change.newValue)}`)
}
lines.push('')
}
if (diff.removed.length > 0) {
lines.push(`Removed (${diff.removed.length}):`)
for (const change of diff.removed) {
lines.push(` - ${change.path}: ${formatValue(change.oldValue)}`)
}
lines.push('')
}
if (diff.modified.length > 0) {
lines.push(`Modified (${diff.modified.length}):`)
for (const change of diff.modified) {
lines.push(` ~ ${change.path}:`)
lines.push(` ${formatValue(change.oldValue)}`)
lines.push(` → ${formatValue(change.newValue)}`)
}
lines.push('')
}
if (diff.typeChanged.length > 0) {
lines.push(`Type Changed (${diff.typeChanged.length}):`)
for (const change of diff.typeChanged) {
lines.push(` ! ${change.path}: ${change.oldType} → ${change.newType}`)
lines.push(` ${formatValue(change.oldValue)}`)
lines.push(` → ${formatValue(change.newValue)}`)
}
}
return lines.join('\n')
}
/**
* Format value for display
*/
function formatValue(value: any): string {
if (value === null) return 'null'
if (value === undefined) return 'undefined'
if (typeof value === 'string') return `"${value}"`
if (typeof value === 'object') {
try {
return JSON.stringify(value)
} catch {
return '[Object]'
}
}
return String(value)
}
/**
* Get summary statistics about a diff
*/
export function getDiffStats(diff: VersionDiff): {
changedFields: number
addedFields: number
removedFields: number
modifiedFields: number
typeChangedFields: number
} {
return {
changedFields: diff.totalChanges,
addedFields: diff.added.length,
removedFields: diff.removed.length,
modifiedFields: diff.modified.length,
typeChangedFields: diff.typeChanged.length
}
}
/**
* Check if diff has any changes
*/
export function hasChanges(diff: VersionDiff): boolean {
return !diff.identical
}
/**
* Get all changed field paths
*/
export function getChangedPaths(diff: VersionDiff): string[] {
const paths = new Set<string>()
for (const change of diff.added) paths.add(change.path)
for (const change of diff.removed) paths.add(change.path)
for (const change of diff.modified) paths.add(change.path)
for (const change of diff.typeChanged) paths.add(change.path)
return Array.from(paths).sort()
}
/**
* Filter diff to only include specific paths
*/
export function filterDiff(diff: VersionDiff, paths: string[]): VersionDiff {
const pathSet = new Set(paths)
const filterChanges = (changes: FieldChange[]) =>
changes.filter((c) => pathSet.has(c.path) || paths.some((p) => c.path.startsWith(p + '.')))
const added = filterChanges(diff.added)
const removed = filterChanges(diff.removed)
const modified = filterChanges(diff.modified)
const typeChanged = filterChanges(diff.typeChanged)
return {
...diff,
added,
removed,
modified,
typeChanged,
totalChanges: added.length + removed.length + modified.length + typeChanged.length,
identical: added.length + removed.length + modified.length + typeChanged.length === 0
}
}

310
src/versioning/VersionIndex.ts
View file

@ -1,310 +0,0 @@
/**
* VersionIndex - Pure Key-Value Version Storage
*
* Stores version metadata using simple key-value storage:
* - Version list per entity: __version_meta_{entityId}_{branch}
* - Content stored separately by VersionStorage
*
* Key Design Decisions:
* - Versions are NOT entities (no brain.add())
* - Versions do NOT pollute find() results
* - Simple O(1) lookups per entity
* - Versions per entity is typically small (10-1000)
*
* NO MOCKS - Production implementation
*/
import type { EntityVersion, VersionQuery } from './VersionManager.js'
/**
* Internal storage structure for version metadata
*/
interface VersionMetadataStore {
entityId: string
branch: string
versions: VersionEntry[]
}
/**
* Individual version entry in the store
*/
interface VersionEntry {
version: number
timestamp: number
contentHash: string
commitHash?: string
tag?: string
description?: string
author?: string
}
/**
* VersionIndex - Pure key-value version metadata storage
*
* Uses simple JSON storage instead of creating entities.
* This ensures versions never appear in find() results.
*/
export class VersionIndex {
private brain: any // Brainy instance
private initialized: boolean = false
constructor(brain: any) {
this.brain = brain
}
/**
* Initialize version index
*/
async initialize(): Promise<void> {
if (this.initialized) return
this.initialized = true
}
/**
* Add version to index
*
* Stores version entry in key-value storage.
* Handles deduplication by content hash.
*
* @param version Version metadata to store
*/
async addVersion(version: EntityVersion): Promise<void> {
await this.initialize()
const key = this.getMetaKey(version.entityId, version.branch)
const store = await this.loadStore(key) || {
entityId: version.entityId,
branch: version.branch,
versions: []
}
// Check for duplicate content hash (deduplication)
const existing = store.versions.find(v => v.contentHash === version.contentHash)
if (existing) {
// Update tag/description if provided on duplicate save
let updated = false
if (version.tag && version.tag !== existing.tag) {
existing.tag = version.tag
updated = true
}
if (version.description && version.description !== existing.description) {
existing.description = version.description
updated = true
}
if (updated) {
await this.saveStore(key, store)
}
return
}
// Add new version entry
store.versions.push({
version: version.version,
timestamp: version.timestamp,
contentHash: version.contentHash,
commitHash: version.commitHash,
tag: version.tag,
description: version.description,
author: version.author
})
await this.saveStore(key, store)
}
/**
* Get versions for an entity
*
* @param query Version query with filters
* @returns List of versions (newest first)
*/
async getVersions(query: VersionQuery): Promise<EntityVersion[]> {
await this.initialize()
const branch = query.branch || this.brain.currentBranch
const key = this.getMetaKey(query.entityId, branch)
const store = await this.loadStore(key)
if (!store) return []
// Convert entries to EntityVersion format
let versions: EntityVersion[] = store.versions.map(entry => ({
version: entry.version,
entityId: store.entityId,
branch: store.branch,
timestamp: entry.timestamp,
contentHash: entry.contentHash,
commitHash: entry.commitHash || '',
tag: entry.tag,
description: entry.description,
author: entry.author
}))
// Apply filters
if (query.tag) {
versions = versions.filter(v => v.tag === query.tag)
}
if (query.startDate) {
versions = versions.filter(v => v.timestamp >= query.startDate!)
}
if (query.endDate) {
versions = versions.filter(v => v.timestamp <= query.endDate!)
}
// Sort newest first (highest version number first)
versions.sort((a, b) => b.version - a.version)
// Apply pagination
const start = query.offset || 0
const end = query.limit ? start + query.limit : undefined
return versions.slice(start, end)
}
/**
* Get specific version
*
* @param entityId Entity ID
* @param version Version number
* @param branch Branch name
* @returns Version metadata or null
*/
async getVersion(
entityId: string,
version: number,
branch: string
): Promise<EntityVersion | null> {
const versions = await this.getVersions({ entityId, branch })
return versions.find(v => v.version === version) || null
}
/**
* Get version by tag
*
* @param entityId Entity ID
* @param tag Version tag
* @param branch Branch name
* @returns Version metadata or null
*/
async getVersionByTag(
entityId: string,
tag: string,
branch: string
): Promise<EntityVersion | null> {
const versions = await this.getVersions({ entityId, branch, tag })
return versions[0] || null
}
/**
* Get version count for entity
*
* @param entityId Entity ID
* @param branch Branch name
* @returns Number of versions
*/
async getVersionCount(entityId: string, branch: string): Promise<number> {
const key = this.getMetaKey(entityId, branch)
const store = await this.loadStore(key)
return store?.versions.length || 0
}
/**
* Remove version from index
*
* @param entityId Entity ID
* @param version Version number
* @param branch Branch name
*/
async removeVersion(
entityId: string,
version: number,
branch: string
): Promise<void> {
await this.initialize()
const key = this.getMetaKey(entityId, branch)
const store = await this.loadStore(key)
if (!store) return
const initialLength = store.versions.length
store.versions = store.versions.filter(v => v.version !== version)
// Only save if something was removed
if (store.versions.length < initialLength) {
await this.saveStore(key, store)
}
}
/**
* Clear all versions for an entity
*
* @param entityId Entity ID
* @param branch Branch name
* @returns Number of versions deleted
*/
async clearVersions(entityId: string, branch: string): Promise<number> {
const key = this.getMetaKey(entityId, branch)
const store = await this.loadStore(key)
if (!store) return 0
const count = store.versions.length
// Delete the store by saving null/empty
await this.saveStore(key, { entityId, branch, versions: [] })
return count
}
/**
* Get all versioned entities (for cleanup/debugging)
*
* Note: This is an expensive operation that requires scanning.
* In the simple key-value approach, we don't maintain a global index.
* This method returns an empty array - use storage-level scanning if needed.
*
* @returns Empty array (not supported in simple approach)
*/
async getVersionedEntities(): Promise<string[]> {
// In the simple key-value approach, we don't maintain a global index
// of all versioned entities. This would require scanning storage.
// For most use cases, you know which entities you've versioned.
return []
}
// ============= Private Helpers =============
/**
* Generate storage key for version metadata
*
* @param entityId Entity ID
* @param branch Branch name
* @returns Storage key
*/
private getMetaKey(entityId: string, branch: string): string {
return `__version_meta_${entityId}_${branch}`
}
/**
* Load version store from storage
*
* @param key Storage key
* @returns Version store or null
*/
private async loadStore(key: string): Promise<VersionMetadataStore | null> {
try {
const store = await this.brain.storageAdapter.getMetadata(key)
// Handle empty store
if (!store || !store.versions) return null
return store as VersionMetadataStore
} catch {
return null
}
}
/**
* Save version store to storage
*
* @param key Storage key
* @param store Version store
*/
private async saveStore(key: string, store: VersionMetadataStore): Promise<void> {
await this.brain.storageAdapter.saveMetadata(key, store)
}
}

667
src/versioning/VersionManager.ts
View file

@ -1,667 +0,0 @@
/**
* VersionManager - Entity-Level Versioning Engine
*
* Provides entity-level version control with:
* - save() - Create entity version
* - restore() - Restore entity to specific version (now updates all indexes)
* - list() - List all versions of an entity
* - compare() - Deep diff between versions
* - prune() - Remove old versions (retention policies)
*
* Architecture:
* - Versions stored as key-value pairs, NOT as entities (no index pollution)
* - Content-addressable: SHA-256 hashing for deduplication
* - Space-efficient: Only stores unique content
* - Branch-aware: Versions tied to current branch
* - restore() uses brain.update() to refresh ALL indexes (HNSW, metadata, graph)
*
* Storage keys:
* - Version metadata: __version_meta_{entityId}_{branch}
* - Version content: __system_version_{entityId}_{contentHash}
*
* ZERO-CONFIG - Works automatically with existing storage infrastructure.
* NO MOCKS - Production implementation.
*/
import { BaseStorage } from '../storage/baseStorage.js'
import { VersionStorage } from './VersionStorage.js'
import { VersionIndex } from './VersionIndex.js'
import { VersionDiff, compareEntityVersions } from './VersionDiff.js'
import type { NounMetadata } from '../coreTypes.js'
export interface EntityVersion {
/** Version number (1-indexed, sequential per entity) */
version: number
/** Entity ID */
entityId: string
/** Branch this version was created on */
branch: string
/** Commit hash containing this version */
commitHash: string
/** Timestamp of version creation */
timestamp: number
/** Optional user-provided tag (e.g., 'v1.0', 'before-refactor') */
tag?: string
/** Optional description */
description?: string
/** Content hash (SHA-256 of entity data) */
contentHash: string
/** Author of this version */
author?: string
/** Metadata about the version */
metadata?: Record<string, any>
}
export interface VersionQuery {
/** Entity ID to query versions for */
entityId: string
/** Optional: Filter by branch (default: current branch) */
branch?: string
/** Optional: Limit number of versions returned */
limit?: number
/** Optional: Skip first N versions */
offset?: number
/** Optional: Filter by tag */
tag?: string
/** Optional: Filter by date range */
startDate?: number
endDate?: number
}
export interface SaveVersionOptions {
/** Optional tag for this version (e.g., 'v1.0', 'milestone-1') */
tag?: string
/** Optional description */
description?: string
/** Optional author name */
author?: string
/** Optional custom metadata */
metadata?: Record<string, any>
/** Optional: Create commit automatically (default: false) */
createCommit?: boolean
/** Optional: Commit message if createCommit is true */
commitMessage?: string
}
export interface RestoreOptions {
/** Optional: Create version before restoring (for undo) */
createSnapshot?: boolean
/** Optional: Tag for snapshot before restore */
snapshotTag?: string
}
export interface PruneOptions {
/** Keep N most recent versions (required if keepVersions not set) */
keepRecent?: number
/** Keep versions newer than timestamp (optional) */
keepAfter?: number
/** Keep tagged versions (default: true) */
keepTagged?: boolean
/** Dry run - don't actually delete (default: false) */
dryRun?: boolean
}
/**
* VersionManager - Core versioning engine
*/
export class VersionManager {
private brain: any // Brainy instance
private versionStorage: VersionStorage
private versionIndex: VersionIndex
private initialized: boolean = false
constructor(brain: any) {
this.brain = brain
this.versionStorage = new VersionStorage(brain)
this.versionIndex = new VersionIndex(brain)
}
/**
* Check if an entity is a VFS file
* VFS files store content in BlobStorage, not in entity.data
*
* @param entity Entity metadata object
* @returns True if entity is a VFS file
*/
private isVFSFile(entity: any): boolean {
return (
entity?.isVFS === true &&
entity?.vfsType === 'file' &&
typeof entity?.path === 'string'
)
}
/**
* Check if content is text-based for encoding decisions
* @param mimeType MIME type of the content
* @returns True if content should be stored as UTF-8 string
*/
private isTextContent(mimeType?: string): boolean {
if (!mimeType) return false
return (
mimeType.startsWith('text/') ||
mimeType === 'application/json' ||
mimeType === 'application/javascript' ||
mimeType === 'application/typescript' ||
mimeType === 'application/xml' ||
mimeType.includes('+xml') ||
mimeType.includes('+json')
)
}
/**
* Initialize versioning system (lazy)
*/
async initialize(): Promise<void> {
if (this.initialized) return
await this.versionStorage.initialize()
await this.versionIndex.initialize()
this.initialized = true
}
/**
* Save a version of an entity
*
* Creates a version snapshot of the current entity state.
* If createCommit is true, also creates a commit containing this version.
*
* @param entityId Entity ID to version
* @param options Save options
* @returns Created version metadata
*/
async save(
entityId: string,
options: SaveVersionOptions = {}
): Promise<EntityVersion> {
await this.initialize()
// Get current entity state
const entity = await this.brain.getNounMetadata(entityId)
if (!entity) {
throw new Error(`Entity ${entityId} not found`)
}
// FIX: For VFS file entities, fetch current content from blob storage
// The entity.data field contains stale embedding text, not actual file content
// VFS files store their real content in BlobStorage (content-addressable)
if (this.isVFSFile(entity)) {
if (!this.brain.vfs) {
throw new Error(
`Cannot version VFS file ${entityId}: VFS not initialized. ` +
`Ensure brain.vfs is available before versioning VFS files.`
)
}
// Read fresh content from blob storage via VFS
const freshContent: Buffer = await this.brain.vfs.readFile(entity.path)
// Store content with appropriate encoding
// Text files as UTF-8 string (readable, smaller)
// Binary files as base64 (safe for JSON serialization)
if (this.isTextContent(entity.mimeType)) {
entity.data = freshContent.toString('utf-8')
} else {
entity.data = freshContent.toString('base64')
entity._vfsEncoding = 'base64' // Flag for restore to decode
}
}
// Get current branch
const currentBranch = this.brain.currentBranch
// Get next version number
const existingVersions = await this.versionIndex.getVersions({
entityId,
branch: currentBranch
})
const nextVersion = existingVersions.length + 1
// Calculate content hash
const contentHash = this.versionStorage.hashEntity(entity)
// Check for duplicate (same content as last version)
if (existingVersions.length > 0) {
const lastVersion = existingVersions[existingVersions.length - 1]
if (lastVersion.contentHash === contentHash) {
// Content unchanged - return last version instead of creating duplicate
return lastVersion
}
}
// Create commit if requested
let commitHash: string
if (options.createCommit) {
const commitMessage =
options.commitMessage || `Version ${nextVersion} of entity ${entityId}`
// Use brain's commit method (note: single options object)
await this.brain.commit({
message: commitMessage,
author: options.author,
metadata: {
versionedEntity: entityId,
version: nextVersion,
...options.metadata
}
})
// Get the commit hash that was just created
const refManager = this.brain.refManager
const ref = await refManager.getRef(currentBranch)
commitHash = ref.commitHash
} else {
// Use current HEAD commit
const refManager = this.brain.refManager
const ref = await refManager.getRef(currentBranch)
if (!ref) {
throw new Error(
`No commit exists on branch ${currentBranch}. Create a commit first or use createCommit: true`
)
}
commitHash = ref.commitHash
}
// Create version metadata
const version: EntityVersion = {
version: nextVersion,
entityId,
branch: currentBranch,
commitHash,
timestamp: Date.now(),
contentHash,
tag: options.tag,
description: options.description,
author: options.author,
metadata: options.metadata
}
// Store version
await this.versionStorage.saveVersion(version, entity)
await this.versionIndex.addVersion(version)
return version
}
/**
* Get all versions of an entity
*
* @param entityId Entity ID
* @param query Optional query filters
* @returns List of versions (newest first)
*/
async list(
entityId: string,
query: Partial<VersionQuery> = {}
): Promise<EntityVersion[]> {
await this.initialize()
const currentBranch = this.brain.currentBranch
return this.versionIndex.getVersions({
entityId,
branch: query.branch || currentBranch,
limit: query.limit,
offset: query.offset,
tag: query.tag,
startDate: query.startDate,
endDate: query.endDate
})
}
/**
* Get a specific version of an entity
*
* @param entityId Entity ID
* @param version Version number (1-indexed)
* @returns Version metadata
*/
async getVersion(
entityId: string,
version: number
): Promise<EntityVersion | null> {
await this.initialize()
const currentBranch = this.brain.currentBranch
return this.versionIndex.getVersion(entityId, version, currentBranch)
}
/**
* Get version by tag
*
* @param entityId Entity ID
* @param tag Version tag
* @returns Version metadata
*/
async getVersionByTag(
entityId: string,
tag: string
): Promise<EntityVersion | null> {
await this.initialize()
const currentBranch = this.brain.currentBranch
return this.versionIndex.getVersionByTag(entityId, tag, currentBranch)
}
/**
* Restore entity to a specific version
*
* Overwrites current entity state with the specified version.
* Optionally creates a snapshot before restoring for undo capability.
*
* @param entityId Entity ID
* @param version Version number or tag
* @param options Restore options
* @returns Restored version metadata
*/
async restore(
entityId: string,
version: number | string,
options: RestoreOptions = {}
): Promise<EntityVersion> {
await this.initialize()
// Create snapshot before restoring (for undo)
if (options.createSnapshot) {
await this.save(entityId, {
tag: options.snapshotTag || 'before-restore',
description: `Snapshot before restoring to version ${version}`,
metadata: { restoringTo: version }
})
}
// Get target version
let targetVersion: EntityVersion | null
if (typeof version === 'number') {
targetVersion = await this.getVersion(entityId, version)
} else {
targetVersion = await this.getVersionByTag(entityId, version)
}
if (!targetVersion) {
throw new Error(
`Version ${version} not found for entity ${entityId}`
)
}
// Load versioned entity data
const versionedEntity = await this.versionStorage.loadVersion(targetVersion)
if (!versionedEntity) {
throw new Error(
`Version data not found for entity ${entityId} version ${version}`
)
}
// FIX: For VFS file entities, write content back to blob storage
// The versioned data contains the actual file content (not stale embedding text)
// Using vfs.writeFile() ensures proper blob creation and metadata update
if (this.isVFSFile(versionedEntity)) {
if (!this.brain.vfs) {
throw new Error(
`Cannot restore VFS file ${entityId}: VFS not initialized. ` +
`Ensure brain.vfs is available before restoring VFS files.`
)
}
// Decode content based on how it was stored
let content: Buffer
if (versionedEntity._vfsEncoding === 'base64') {
// Binary file stored as base64
content = Buffer.from(versionedEntity.data as string, 'base64')
} else {
// Text file stored as UTF-8 string
content = Buffer.from(versionedEntity.data as string, 'utf-8')
}
// Write content back to VFS - this handles:
// - BlobStorage write (new hash)
// - Entity metadata update
// - Path resolver cache update
await this.brain.vfs.writeFile(versionedEntity.path, content)
return targetVersion
}
// For non-VFS entities, use existing brain.update() logic
// Extract standard fields vs custom metadata
// NounMetadata has: noun, data, createdAt, updatedAt, createdBy, service, confidence, weight
const {
noun,
data,
createdAt,
updatedAt,
createdBy,
service,
confidence,
weight,
...customMetadata
} = versionedEntity
// Use brain.update() to restore - this updates ALL indexes (HNSW, metadata, graph)
// This is critical: saveNounMetadata() only saves to storage without updating indexes
await this.brain.update({
id: entityId,
data: data,
type: noun,
metadata: customMetadata,
confidence: confidence,
weight: weight,
merge: false // Replace entirely, don't merge with existing metadata
})
return targetVersion
}
/**
* Compare two versions of an entity
*
* @param entityId Entity ID
* @param fromVersion Version number or tag (older)
* @param toVersion Version number or tag (newer)
* @returns Diff between versions
*/
async compare(
entityId: string,
fromVersion: number | string,
toVersion: number | string
): Promise<VersionDiff> {
await this.initialize()
// Get versions
const fromVer =
typeof fromVersion === 'number'
? await this.getVersion(entityId, fromVersion)
: await this.getVersionByTag(entityId, fromVersion)
const toVer =
typeof toVersion === 'number'
? await this.getVersion(entityId, toVersion)
: await this.getVersionByTag(entityId, toVersion)
if (!fromVer) {
throw new Error(
`Version ${fromVersion} not found for entity ${entityId}`
)
}
if (!toVer) {
throw new Error(
`Version ${toVersion} not found for entity ${entityId}`
)
}
// Load entity data
const fromEntity = await this.versionStorage.loadVersion(fromVer)
const toEntity = await this.versionStorage.loadVersion(toVer)
if (!fromEntity || !toEntity) {
throw new Error('Failed to load version data for comparison')
}
// Compare versions
return compareEntityVersions(fromEntity, toEntity, {
fromVersion: fromVer.version,
toVersion: toVer.version,
entityId
})
}
/**
* Prune old versions based on retention policy
*
* @param entityId Entity ID (or '*' for all entities)
* @param options Prune options
* @returns Number of versions deleted
*/
async prune(
entityId: string,
options: PruneOptions
): Promise<{ deleted: number; kept: number }> {
await this.initialize()
if (!options.keepRecent && !options.keepAfter) {
throw new Error(
'Must specify either keepRecent or keepAfter in prune options'
)
}
const currentBranch = this.brain.currentBranch
// Get all versions
const versions = await this.versionIndex.getVersions({
entityId,
branch: currentBranch
})
// Determine which versions to keep
const toKeep = new Set<number>()
const toDelete: EntityVersion[] = []
// Keep recent versions
if (options.keepRecent) {
const recentVersions = versions.slice(0, options.keepRecent)
recentVersions.forEach((v) => toKeep.add(v.version))
}
// Keep versions after timestamp
if (options.keepAfter) {
versions
.filter((v) => v.timestamp >= options.keepAfter!)
.forEach((v) => toKeep.add(v.version))
}
// Keep tagged versions
if (options.keepTagged !== false) {
versions
.filter((v) => v.tag !== undefined)
.forEach((v) => toKeep.add(v.version))
}
// Build delete list
for (const version of versions) {
if (!toKeep.has(version.version)) {
toDelete.push(version)
}
}
// Dry run - just return counts
if (options.dryRun) {
return {
deleted: toDelete.length,
kept: toKeep.size
}
}
// Delete versions
for (const version of toDelete) {
await this.versionStorage.deleteVersion(version)
await this.versionIndex.removeVersion(
entityId,
version.version,
currentBranch
)
}
return {
deleted: toDelete.length,
kept: toKeep.size
}
}
/**
* Get version count for an entity
*
* @param entityId Entity ID
* @returns Number of versions
*/
async getVersionCount(entityId: string): Promise<number> {
await this.initialize()
const currentBranch = this.brain.currentBranch
return this.versionIndex.getVersionCount(entityId, currentBranch)
}
/**
* Check if entity has versions
*
* @param entityId Entity ID
* @returns True if entity has versions
*/
async hasVersions(entityId: string): Promise<boolean> {
const count = await this.getVersionCount(entityId)
return count > 0
}
/**
* Get latest version of an entity
*
* @param entityId Entity ID
* @returns Latest version metadata or null
*/
async getLatest(entityId: string): Promise<EntityVersion | null> {
await this.initialize()
const versions = await this.list(entityId, { limit: 1 })
return versions[0] || null
}
/**
* Clear all versions for an entity
*
* @param entityId Entity ID
* @returns Number of versions deleted
*/
async clear(entityId: string): Promise<number> {
await this.initialize()
const result = await this.prune(entityId, {
keepRecent: 0,
keepTagged: false
})
return result.deleted
}
}

250
src/versioning/VersionStorage.ts
View file

@ -1,250 +0,0 @@
/**
* VersionStorage - Hybrid Storage for Entity Versions
*
* Implements content-addressable storage for entity versions:
* - SHA-256 content hashing for deduplication
* - Uses BaseStorage.saveMetadata/getMetadata for storage
* - Integrates with COW commit system
* - Space-efficient: Only stores unique content
*
* Storage structure:
* Version content is stored using system metadata keys:
* __system_version_{entityId}_{contentHash}
*
* This integrates with BaseStorage's routing which places system keys
* in the _system/ directory, keeping version data separate from entities.
*
* NO MOCKS - Production implementation
*/
import { createHash } from 'crypto'
import { BaseStorage } from '../storage/baseStorage.js'
import type { NounMetadata } from '../coreTypes.js'
import type { EntityVersion } from './VersionManager.js'
/**
* VersionStorage - Content-addressable version storage
*/
export class VersionStorage {
private brain: any // Brainy instance
private initialized: boolean = false
constructor(brain: any) {
this.brain = brain
}
/**
* Initialize version storage directories
*/
async initialize(): Promise<void> {
if (this.initialized) return
// Version storage uses the same storage adapter as the main database
// Directories are created automatically by the storage adapter
this.initialized = true
}
/**
* Calculate SHA-256 hash of entity content
*
* Used for content-addressable storage and deduplication
*
* @param entity Entity to hash
* @returns SHA-256 hash (hex string)
*/
hashEntity(entity: NounMetadata): string {
// Create stable JSON representation (sorted keys)
const stableJson = this.toStableJson(entity)
return createHash('sha256').update(stableJson).digest('hex')
}
/**
* Convert entity to stable JSON (sorted keys for consistent hashing)
*/
private toStableJson(obj: any): string {
if (obj === null) return 'null'
if (obj === undefined) return 'undefined'
if (typeof obj !== 'object') return JSON.stringify(obj)
if (Array.isArray(obj)) {
const items = obj.map((item) => this.toStableJson(item))
return `[${items.join(',') }]`
}
// Sort object keys for stable hashing
const sortedKeys = Object.keys(obj).sort()
const pairs = sortedKeys.map((key) => {
const value = this.toStableJson(obj[key])
return `"${key}":${value}`
})
return `{${pairs.join(',')}}`
}
/**
* Save entity version to content-addressable storage
*
* @param version Version metadata
* @param entity Entity data
*/
async saveVersion(
version: EntityVersion,
entity: NounMetadata
): Promise<void> {
await this.initialize()
// Content-addressable path: .brainy/versions/entities/{entityId}/{contentHash}.json
const versionPath = this.getVersionPath(
version.entityId,
version.contentHash
)
// Check if content already exists (deduplication)
const exists = await this.contentExists(versionPath)
if (exists) {
// Content already stored - no need to write again
return
}
// Store entity data
await this.writeVersionData(versionPath, entity)
}
/**
* Load entity version from storage
*
* @param version Version metadata
* @returns Entity data or null if not found
*/
async loadVersion(version: EntityVersion): Promise<NounMetadata | null> {
await this.initialize()
const versionPath = this.getVersionPath(
version.entityId,
version.contentHash
)
try {
return await this.readVersionData(versionPath)
} catch (error) {
console.error(`Failed to load version ${version.version}:`, error)
return null
}
}
/**
* Delete entity version from storage
*
* @param version Version to delete
*/
async deleteVersion(version: EntityVersion): Promise<void> {
await this.initialize()
const versionPath = this.getVersionPath(
version.entityId,
version.contentHash
)
await this.deleteVersionData(versionPath)
}
/**
* Get version storage key
*
* Uses __system_ prefix so BaseStorage routes to system storage (_system/ directory)
* This keeps version data separate from entity data.
*
* @param entityId Entity ID
* @param contentHash Content hash
* @returns Storage key for version content
*/
private getVersionPath(entityId: string, contentHash: string): string {
// Use system-prefixed key for BaseStorage.saveMetadata/getMetadata
// BaseStorage recognizes __system_ prefix and routes to _system/ directory
return `__system_version_${entityId}_${contentHash}`
}
/**
* Check if content exists in storage
*
* @param key Storage key
* @returns True if exists
*/
private async contentExists(key: string): Promise<boolean> {
try {
// Use getMetadata to check existence
const adapter = this.brain.storageAdapter
if (!adapter) return false
const data = await adapter.getMetadata(key)
return data !== null
} catch {
return false
}
}
/**
* Write version data to storage
*
* @param key Storage key
* @param entity Entity data
*/
private async writeVersionData(
key: string,
entity: NounMetadata
): Promise<void> {
const adapter = this.brain.storageAdapter
if (!adapter) {
throw new Error('Storage adapter not available')
}
// Use saveMetadata for storing version content
// The key is system-prefixed so it routes to _system/ directory
await adapter.saveMetadata(key, entity)
}
/**
* Read version data from storage
*
* @param key Storage key
* @returns Entity data
*/
private async readVersionData(key: string): Promise<NounMetadata> {
const adapter = this.brain.storageAdapter
if (!adapter) {
throw new Error('Storage adapter not available')
}
// Use getMetadata for reading version content
const entity = await adapter.getMetadata(key)
if (!entity) {
throw new Error(`Version data not found: ${key}`)
}
return entity
}
/**
* Delete version data from storage
*
* Note: Version content is content-addressed and immutable.
* Deleting the version index entry (via VersionIndex.removeVersion) is sufficient.
* The content may be shared with other versions (same contentHash).
*
* We don't actually delete version content to avoid breaking
* other versions that may reference the same content hash.
* A separate garbage collection process could clean up unreferenced content.
*
* @param key Storage key (unused - kept for API compatibility)
*/
private async deleteVersionData(_key: string): Promise<void> {
// Version content is content-addressed and may be shared.
// We don't delete it here to prevent breaking other versions.
// The version INDEX is deleted via VersionIndex.removeVersion().
// A GC process could clean up unreferenced content in the future.
}
}

520
src/versioning/VersioningAPI.ts
View file

@ -1,520 +0,0 @@
/**
* VersioningAPI - Public API for Entity Versioning
*
* User-friendly wrapper around VersionManager with:
* - Clean, simple API
* - Smart defaults
* - Error handling
* - Type safety
*
* Usage:
* const version = await brain.versions.save('entity-123', { tag: 'v1.0' })
* const versions = await brain.versions.list('entity-123')
* await brain.versions.restore('entity-123', 5)
* const diff = await brain.versions.compare('entity-123', 2, 5)
*
* NO MOCKS - Production implementation
*/
import { VersionManager } from './VersionManager.js'
import type {
EntityVersion,
SaveVersionOptions,
RestoreOptions,
PruneOptions,
VersionQuery
} from './VersionManager.js'
import type { VersionDiff, DiffOptions } from './VersionDiff.js'
import type { BaseStorage } from '../storage/baseStorage.js'
/**
* VersioningAPI - User-friendly versioning interface
*/
export class VersioningAPI {
private manager: VersionManager
private brain: any // Brainy instance
constructor(brain: any) {
this.brain = brain
this.manager = new VersionManager(brain as BaseStorage)
}
/**
* Save current state of entity as a new version
*
* Creates a version snapshot of the current entity state.
* Automatically handles deduplication - if content hasn't changed,
* returns the last version instead of creating a duplicate.
*
* @param entityId Entity ID to version
* @param options Save options
* @returns Created (or existing) version metadata
*
* @example
* ```typescript
* // Simple save
* const version = await brain.versions.save('user-123')
*
* // Save with tag and description
* const version = await brain.versions.save('user-123', {
* tag: 'v1.0',
* description: 'Initial release',
* author: 'alice'
* })
*
* // Save and create commit
* const version = await brain.versions.save('user-123', {
* tag: 'milestone-1',
* createCommit: true,
* commitMessage: 'Milestone 1 complete'
* })
* ```
*/
async save(
entityId: string,
options: SaveVersionOptions = {}
): Promise<EntityVersion> {
return this.manager.save(entityId, options)
}
/**
* List all versions of an entity
*
* Returns versions sorted by version number (newest first).
* Supports filtering by tag, date range, and pagination.
*
* @param entityId Entity ID
* @param options Query options
* @returns List of versions (newest first)
*
* @example
* ```typescript
* // Get all versions
* const versions = await brain.versions.list('user-123')
*
* // Get last 10 versions
* const recent = await brain.versions.list('user-123', { limit: 10 })
*
* // Get tagged versions
* const tagged = await brain.versions.list('user-123', { tag: 'v*' })
*
* // Get versions from last 30 days
* const recent = await brain.versions.list('user-123', {
* startDate: Date.now() - 30 * 24 * 60 * 60 * 1000
* })
* ```
*/
async list(
entityId: string,
options: Partial<VersionQuery> = {}
): Promise<EntityVersion[]> {
return this.manager.list(entityId, options)
}
/**
* Get specific version of an entity
*
* @param entityId Entity ID
* @param version Version number (1-indexed)
* @returns Version metadata or null if not found
*
* @example
* ```typescript
* const version = await brain.versions.getVersion('user-123', 5)
* if (version) {
* console.log(`Version ${version.version} created at ${new Date(version.timestamp)}`)
* }
* ```
*/
async getVersion(
entityId: string,
version: number
): Promise<EntityVersion | null> {
return this.manager.getVersion(entityId, version)
}
/**
* Get version by tag
*
* @param entityId Entity ID
* @param tag Version tag
* @returns Version metadata or null if not found
*
* @example
* ```typescript
* const version = await brain.versions.getVersionByTag('user-123', 'v1.0')
* ```
*/
async getVersionByTag(
entityId: string,
tag: string
): Promise<EntityVersion | null> {
return this.manager.getVersionByTag(entityId, tag)
}
/**
* Get latest version of an entity
*
* @param entityId Entity ID
* @returns Latest version or null if no versions exist
*
* @example
* ```typescript
* const latest = await brain.versions.getLatest('user-123')
* ```
*/
async getLatest(entityId: string): Promise<EntityVersion | null> {
return this.manager.getLatest(entityId)
}
/**
* Get version content without restoring
*
* Allows you to preview version data without modifying the current entity.
*
* @param entityId Entity ID
* @param version Version number or tag
* @returns Version content
*
* @example
* ```typescript
* // Preview version 5 without restoring
* const oldData = await brain.versions.getContent('user-123', 5)
* console.log('Version 5 had name:', oldData.name)
*
* // Compare with current
* const current = await brain.getNounMetadata('user-123')
* console.log('Current name:', current.name)
* ```
*/
async getContent(
entityId: string,
version: number | string
): Promise<any> {
// Get version metadata
let versionMeta: EntityVersion | null
if (typeof version === 'number') {
versionMeta = await this.manager.getVersion(entityId, version)
} else {
versionMeta = await this.manager.getVersionByTag(entityId, version)
}
if (!versionMeta) {
throw new Error(
`Version ${version} not found for entity ${entityId}`
)
}
// Load version content
const versionStorage = (this.manager as any).versionStorage
const content = await versionStorage.loadVersion(versionMeta)
if (!content) {
throw new Error(
`Version content not found for entity ${entityId} version ${version}`
)
}
return content
}
/**
* Restore entity to a specific version
*
* Overwrites current entity state with the specified version.
* Optionally creates a snapshot before restoring for undo capability.
*
* @param entityId Entity ID
* @param version Version number or tag to restore to
* @param options Restore options
* @returns Restored version metadata
*
* @example
* ```typescript
* // Simple restore
* await brain.versions.restore('user-123', 5)
*
* // Restore with safety snapshot
* await brain.versions.restore('user-123', 5, {
* createSnapshot: true,
* snapshotTag: 'before-restore'
* })
*
* // Restore by tag
* await brain.versions.restore('user-123', 'v1.0')
* ```
*/
async restore(
entityId: string,
version: number | string,
options: RestoreOptions = {}
): Promise<EntityVersion> {
return this.manager.restore(entityId, version, options)
}
/**
* Compare two versions of an entity
*
* Generates a deep diff showing added, removed, modified, and type-changed fields.
*
* @param entityId Entity ID
* @param fromVersion Version number or tag (older)
* @param toVersion Version number or tag (newer)
* @returns Diff between versions
*
* @example
* ```typescript
* // Compare version 2 to version 5
* const diff = await brain.versions.compare('user-123', 2, 5)
*
* console.log(`Added fields: ${diff.added.length}`)
* console.log(`Removed fields: ${diff.removed.length}`)
* console.log(`Modified fields: ${diff.modified.length}`)
*
* // Print human-readable diff
* import { formatDiff } from './VersionDiff.js'
* console.log(formatDiff(diff))
* ```
*/
async compare(
entityId: string,
fromVersion: number | string,
toVersion: number | string
): Promise<VersionDiff> {
return this.manager.compare(entityId, fromVersion, toVersion)
}
/**
* Prune old versions based on retention policy
*
* Removes old versions while preserving recent and tagged versions.
* Use dryRun to preview what would be deleted without actually deleting.
*
* @param entityId Entity ID (or '*' for all entities - NOT IMPLEMENTED YET)
* @param options Prune options
* @returns Count of deleted and kept versions
*
* @example
* ```typescript
* // Keep last 10 versions, delete rest
* const result = await brain.versions.prune('user-123', {
* keepRecent: 10
* })
* console.log(`Deleted ${result.deleted} versions`)
*
* // Keep last 30 days, preserve tagged versions
* const result = await brain.versions.prune('user-123', {
* keepAfter: Date.now() - 30 * 24 * 60 * 60 * 1000,
* keepTagged: true
* })
*
* // Dry run - see what would be deleted
* const result = await brain.versions.prune('user-123', {
* keepRecent: 5,
* dryRun: true
* })
* console.log(`Would delete ${result.deleted} versions`)
* ```
*/
async prune(
entityId: string,
options: PruneOptions
): Promise<{ deleted: number; kept: number }> {
return this.manager.prune(entityId, options)
}
/**
* Get version count for an entity
*
* @param entityId Entity ID
* @returns Number of versions
*
* @example
* ```typescript
* const count = await brain.versions.count('user-123')
* console.log(`Entity has ${count} versions`)
* ```
*/
async count(entityId: string): Promise<number> {
return this.manager.getVersionCount(entityId)
}
/**
* Check if entity has any versions
*
* @param entityId Entity ID
* @returns True if entity has versions
*
* @example
* ```typescript
* if (await brain.versions.hasVersions('user-123')) {
* console.log('Entity is versioned')
* }
* ```
*/
async hasVersions(entityId: string): Promise<boolean> {
return this.manager.hasVersions(entityId)
}
/**
* Clear all versions for an entity
*
* WARNING: This permanently deletes all version history!
*
* @param entityId Entity ID
* @returns Number of versions deleted
*
* @example
* ```typescript
* const deleted = await brain.versions.clear('user-123')
* console.log(`Deleted ${deleted} versions`)
* ```
*/
async clear(entityId: string): Promise<number> {
return this.manager.clear(entityId)
}
// ===== CONVENIENCE METHODS =====
/**
* Quick save with auto-generated tag
*
* Generates tag in format: 'auto-{timestamp}'
*
* @param entityId Entity ID
* @param description Optional description
* @returns Created version
*/
async quickSave(
entityId: string,
description?: string
): Promise<EntityVersion> {
return this.save(entityId, {
tag: `auto-${Date.now()}`,
description
})
}
/**
* Undo last change (restore to previous version)
*
* Safely restores to previous version with automatic snapshot.
*
* @param entityId Entity ID
* @returns Restored version metadata
*/
async undo(entityId: string): Promise<EntityVersion | null> {
const versions = await this.list(entityId, { limit: 2 })
if (versions.length < 2) {
// No previous version to restore to
return null
}
const previousVersion = versions[1] // Second most recent
return this.restore(entityId, previousVersion.version, {
createSnapshot: true,
snapshotTag: 'before-undo'
})
}
/**
* Get version history summary
*
* @param entityId Entity ID
* @returns Summary statistics
*/
async history(entityId: string): Promise<{
total: number
oldest?: EntityVersion
newest?: EntityVersion
tagged: number
branches: string[]
}> {
const versions = await this.list(entityId)
const tagged = versions.filter((v) => v.tag !== undefined).length
const branches = [...new Set(versions.map((v) => v.branch))]
return {
total: versions.length,
oldest: versions[versions.length - 1],
newest: versions[0],
tagged,
branches
}
}
/**
* Revert to previous version (alias for undo with better semantics)
*
* Restores entity to the previous version with automatic safety snapshot.
*
* @param entityId Entity ID
* @returns Restored version or null if not possible
*
* @example
* ```typescript
* // Made a mistake? Revert!
* await brain.update('user-123', { name: 'Wrong Name' })
* await brain.versions.revert('user-123') // Back to previous state
* ```
*/
async revert(entityId: string): Promise<EntityVersion | null> {
return this.undo(entityId)
}
/**
* Tag an existing version
*
* Updates the tag of an existing version.
* Note: This creates a new version with the tag, not modifying the existing one.
*
* @param entityId Entity ID
* @param version Version number
* @param tag New tag
* @returns Updated version
*/
async tag(
entityId: string,
version: number,
tag: string
): Promise<EntityVersion> {
// Get the version
const existingVersion = await this.getVersion(entityId, version)
if (!existingVersion) {
throw new Error(`Version ${version} not found for entity ${entityId}`)
}
// Create new version with tag
// Note: This is a limitation - we can't modify existing versions
// because they're immutable. Instead, we create a new version.
return this.save(entityId, {
tag,
description: `Tagged version ${version} as ${tag}`,
metadata: { originalVersion: version }
})
}
/**
* Get diff between entity's current state and a version
*
* @param entityId Entity ID
* @param version Version to compare against
* @returns Diff showing changes
*/
async diffWithCurrent(
entityId: string,
version: number | string
): Promise<VersionDiff> {
// Save current state (will be deduplicated if unchanged)
const currentVersion = await this.save(entityId, {
tag: 'temp-compare',
description: 'Temporary version for comparison'
})
// Compare
return this.compare(entityId, version, currentVersion.version)
}
}

25
src/vfs/VirtualFileSystem.ts
View file

@ -104,8 +104,11 @@ export class VirtualFileSystem implements IVirtualFileSystem {
private get blobStorage() { private get blobStorage() {
// TypeScript doesn't know about blobStorage on storage, use type assertion // TypeScript doesn't know about blobStorage on storage, use type assertion
const storage = this.brain['storage'] as any const storage = this.brain['storage'] as any
if (!storage || !('blobStorage' in storage)) { if (!storage?.blobStorage) {
throw new Error('BlobStorage not available. Requires COW-enabled storage adapter.') throw new Error(
'BlobStorage not available. The storage adapter must run ' +
'initializeBlobStorage() before the VFS is used (brain.init() does this).'
)
} }
return storage.blobStorage return storage.blobStorage
} }
@ -437,9 +440,13 @@ export class VirtualFileSystem implements IVirtualFileSystem {
existingId = null existingId = null
} }
// Detect MIME type BEFORE the blob write so the store's auto-compression
// policy can skip zstd for already-compressed media (JPEG, MP4, ZIP, …).
const mimeType = mimeDetector.detectMimeType(name, buffer)
// Unified blob storage for ALL files (no size-based branching) // Unified blob storage for ALL files (no size-based branching)
// Store in BlobStorage (content-addressable, auto-deduplication, streaming) // Store in BlobStorage (content-addressable, auto-deduplication)
const blobHash = await this.blobStorage.write(buffer) const blobHash = await this.blobStorage.write(buffer, { mimeType })
// Get blob metadata (size, compression info) // Get blob metadata (size, compression info)
const blobMetadata = await this.blobStorage.getMetadata(blobHash) const blobMetadata = await this.blobStorage.getMetadata(blobHash)
@ -448,12 +455,9 @@ export class VirtualFileSystem implements IVirtualFileSystem {
type: 'blob', type: 'blob',
hash: blobHash, hash: blobHash,
size: buffer.length, size: buffer.length,
compressed: blobMetadata?.compressed compressed: blobMetadata ? blobMetadata.compression !== 'none' : undefined
} }
// Detect MIME type (using comprehensive MimeTypeDetector)
const mimeType = mimeDetector.detectMimeType(name, buffer)
// Create metadata // Create metadata
const metadata: VFSMetadata = { const metadata: VFSMetadata = {
path, path,
@ -2716,11 +2720,6 @@ export class VirtualFileSystem implements IVirtualFileSystem {
return results return results
} }
/**
* Get all versions of a file (semantic versioning)
*/
createReadStream(path: string, options?: ReadStreamOptions): NodeJS.ReadableStream { createReadStream(path: string, options?: ReadStreamOptions): NodeJS.ReadableStream {
// Lazy import to avoid circular dependencies // Lazy import to avoid circular dependencies
const { VFSReadStream } = require('./streams/VFSReadStream.js') const { VFSReadStream } = require('./streams/VFSReadStream.js')

10
tests/helpers/TestWrappingAdapter.ts
View file

@ -1,10 +1,10 @@
/** /**
* TestWrappingAdapter: Real COW storage adapter for testing * TestWrappingAdapter: Real blob store adapter for testing
* *
* This adapter ACTUALLY wraps binary data in JSON (like production storage) * This adapter ACTUALLY wraps binary data in JSON (like production storage)
* to properly test the unwrap logic in BlobStorage. * to properly test the unwrap logic in BlobStorage.
* *
* Unlike InMemoryCOWAdapter which stores Buffers directly, * Unlike InMemoryBlobAdapter which stores Buffers directly,
* this adapter simulates real storage behavior: * this adapter simulates real storage behavior:
* 1. Compresses with gzip * 1. Compresses with gzip
* 2. Wraps binary as {_binary: true, data: "base64..."} * 2. Wraps binary as {_binary: true, data: "base64..."}
@ -13,14 +13,14 @@
* This ensures tests exercise the ACTUAL code path that caused v5.10.0 regression. * This ensures tests exercise the ACTUAL code path that caused v5.10.0 regression.
*/ */
import { COWStorageAdapter } from '../../src/storage/cow/BlobStorage.js' import { BlobStoreAdapter } from '../../src/storage/blobStorage.js'
import { gzip, gunzip } from 'zlib' import { gzip, gunzip } from 'zlib'
import { promisify } from 'util' import { promisify } from 'util'
const gzipAsync = promisify(gzip) const gzipAsync = promisify(gzip)
const gunzipAsync = promisify(gunzip) const gunzipAsync = promisify(gunzip)
export class TestWrappingAdapter implements COWStorageAdapter { export class TestWrappingAdapter implements BlobStoreAdapter {
private storage = new Map<string, Buffer>() private storage = new Map<string, Buffer>()
async get(key: string): Promise<any | undefined> { async get(key: string): Promise<any | undefined> {
@ -39,7 +39,7 @@ export class TestWrappingAdapter implements COWStorageAdapter {
} }
async put(key: string, data: Buffer): Promise<void> { async put(key: string, data: Buffer): Promise<void> {
// v6.2.0: Use key-based dispatch (matches baseStorage COW adapter) // Use key-based dispatch (matches the baseStorage blob adapter)
// NO GUESSING - key format explicitly declares data type // NO GUESSING - key format explicitly declares data type
const obj = key.includes('-meta:') || key.startsWith('ref:') const obj = key.includes('-meta:') || key.startsWith('ref:')
? JSON.parse(data.toString()) // Metadata/refs: ALWAYS JSON ? JSON.parse(data.toString()) // Metadata/refs: ALWAYS JSON

104
tests/integration/commit-ref-bug.test.ts
View file

@ -1,104 +0,0 @@
/**
* Integration test to reproduce commit() ref update bug
* Bug: commit() creates blobs but doesn't update branch refs
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
import * as path from 'path'
import * as zlib from 'zlib'
const TEST_DATA_PATH = './test-commit-ref-bug-data'
describe('Commit Ref Update Bug (v5.2.0)', () => {
let brain: Brainy
beforeEach(async () => {
// Clean up test data
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
path: TEST_DATA_PATH,
branch: 'main',
enableCompression: true
},
silent: false // Enable logging
})
await brain.init()
})
afterEach(() => {
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
})
it('should update branch ref after commit', async () => {
// Add an entity
await brain.add({
data: 'Test entity',
type: 'concept',
metadata: { test: true }
})
// Create first commit
const commit1Hash = await brain.commit({
message: 'First commit',
author: 'test@example.com'
})
console.log(`\n=== Commit 1 created: ${commit1Hash} ===`)
// Check the ref file directly
const refPath = path.join(TEST_DATA_PATH, '_cow', 'ref:refs', 'heads', 'main.gz')
expect(fs.existsSync(refPath), 'Ref file should exist').toBe(true)
const refContent1 = JSON.parse(
zlib.gunzipSync(fs.readFileSync(refPath)).toString()
)
console.log('Ref after commit 1:', {
commitHash: refContent1.commitHash,
updatedAt: new Date(refContent1.updatedAt).toISOString()
})
// THIS IS THE BUG: commitHash should equal commit1Hash
expect(refContent1.commitHash).toBe(commit1Hash)
expect(refContent1.commitHash).not.toBe('0'.repeat(64))
// Add another entity
await brain.add({
data: 'Second entity',
type: 'concept'
})
// Create second commit
const commit2Hash = await brain.commit({
message: 'Second commit',
author: 'test@example.com'
})
console.log(`\n=== Commit 2 created: ${commit2Hash} ===`)
// Check ref again
const refContent2 = JSON.parse(
zlib.gunzipSync(fs.readFileSync(refPath)).toString()
)
console.log('Ref after commit 2:', {
commitHash: refContent2.commitHash,
updatedAt: new Date(refContent2.updatedAt).toISOString()
})
// THIS IS THE BUG: commitHash should equal commit2Hash
expect(refContent2.commitHash).toBe(commit2Hash)
expect(refContent2.updatedAt).toBeGreaterThan(refContent1.updatedAt)
})
})

573
tests/integration/commit-state-capture.test.ts
View file

@ -1,573 +0,0 @@
/**
* Commit State Capture Integration Tests (v5.4.0 Phase 1)
*
* Tests the new captureState parameter on commit() that enables
* historical time-travel by capturing entity snapshots in trees.
*
* Tests:
* 1. Lightweight commit (captureState: false uses NULL_HASH)
* 2. Default behavior (captureState omitted → captures state)
* 3. Empty workspace handling
* 4. Performance benchmarks (100, 1K, 10K entities)
* 5. BlobStorage deduplication
* 6. Tree structure validity
* 7. Storage adapter compatibility
* 8. VFS entity capture
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
import * as path from 'path'
const TEST_DATA_PATH = './test-commit-state-capture'
describe('Commit State Capture (v5.4.0)', () => {
let brain: Brainy
beforeEach(async () => {
// Clean up test data
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
fs.mkdirSync(TEST_DATA_PATH, { recursive: true })
brain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'filesystem',
path: TEST_DATA_PATH
},
disableAutoRebuild: true,
silent: true
})
await brain.init()
})
afterEach(() => {
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
})
it('should use NULL_HASH when captureState is false (backward compat)', async () => {
console.log('\n📋 Test 1: Backward compatibility')
// Add some entities
await brain.add({ type: 'concept', data: { title: 'Test' } })
// Commit with explicit captureState: false (lightweight metadata-only commit)
const commitId = await brain.commit({
message: 'Test commit',
captureState: false // Explicit false — skips state capture
})
console.log(` Commit ID: ${commitId.slice(0, 8)}`)
// Read commit object
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { NULL_HASH } = await import('../../src/storage/cow/constants.js')
const commit = await CommitObject.read(blobStorage, commitId)
// Verify tree is NULL_HASH (empty)
expect(commit.tree).toBe(NULL_HASH)
console.log(` ✅ Tree is NULL_HASH (backward compatible)`)
})
it('should capture state when captureState is omitted (default: true)', async () => {
console.log('\n📋 Test 2: Default behavior (captureState omitted → captures state)')
await brain.add({ type: 'concept', data: { title: 'Test' } })
// Commit WITHOUT captureState parameter (omitted = default true)
const commitId = await brain.commit({
message: 'Test commit'
// captureState not specified → defaults to true
})
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { NULL_HASH } = await import('../../src/storage/cow/constants.js')
const commit = await CommitObject.read(blobStorage, commitId)
// Default behavior now captures state (tree hash is NOT null)
expect(commit.tree).not.toBe(NULL_HASH)
console.log(` ✅ Tree is ${commit.tree.slice(0, 8)}... (state captured by default)`)
})
it('should capture entity state when captureState is true', async () => {
console.log('\n📋 Test 3: State capture enabled')
// Add entities (add() returns entity ID, not entity object)
const entity1Id = await brain.add({
type: 'person',
data: { name: 'Alice', age: 30 }
})
const entity2Id = await brain.add({
type: 'concept',
data: { title: 'AI', description: 'Artificial Intelligence' }
})
console.log(` Added 2 entities: ${entity1Id.slice(0, 8)}, ${entity2Id.slice(0, 8)}`)
// Commit WITH captureState
const commitId = await brain.commit({
message: 'Snapshot with entities',
captureState: true // CAPTURE STATE!
})
console.log(` Commit ID: ${commitId.slice(0, 8)}`)
// Read commit object
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { NULL_HASH, isNullHash } = await import('../../src/storage/cow/constants.js')
const commit = await CommitObject.read(blobStorage, commitId)
// Verify tree is NOT NULL_HASH
expect(commit.tree).not.toBe(NULL_HASH)
expect(isNullHash(commit.tree)).toBe(false)
console.log(` ✅ Tree is NOT NULL_HASH: ${commit.tree.slice(0, 8)}`)
// Verify tree exists and can be read
const { TreeObject } = await import('../../src/storage/cow/TreeObject.js')
const tree = await TreeObject.read(blobStorage, commit.tree)
expect(tree).toBeDefined()
expect(tree.entries.length).toBeGreaterThan(0)
console.log(` ✅ Tree has ${tree.entries.length} entries`)
// Verify entities are in tree
let foundEntities = 0
const entityNames: string[] = []
for (const entry of tree.entries) {
if (entry.name.startsWith('entities/')) {
foundEntities++
entityNames.push(entry.name)
}
}
console.log(` Found ${foundEntities} entities: ${entityNames.join(', ')}`)
// At least 2 entities should be captured (the ones we added)
// There may be system entities (VFS root, metadata, etc.)
expect(foundEntities).toBeGreaterThanOrEqual(2)
console.log(` ✅ Found ${foundEntities} entities in tree (at least 2 as expected)`)
})
it('should handle empty workspace correctly', async () => {
console.log('\n📋 Test 4: Empty workspace (no user entities)')
// Check how many entities exist before commit
const entitiesBefore = await brain.find({ excludeVFS: false })
console.log(` Entities before commit: ${entitiesBefore.length}`)
// Commit with NO user-added entities
const commitId = await brain.commit({
message: 'Empty snapshot',
captureState: true
})
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { NULL_HASH, isNullHash } = await import('../../src/storage/cow/constants.js')
const commit = await CommitObject.read(blobStorage, commitId)
// If there were NO entities before commit, tree should be NULL_HASH
// If there WERE entities (system entities), tree should contain them
if (entitiesBefore.length === 0) {
expect(commit.tree).toBe(NULL_HASH)
console.log(` ✅ No entities: tree is NULL_HASH (correct)`)
} else {
expect(isNullHash(commit.tree)).toBe(false)
console.log(` ✅ ${entitiesBefore.length} system entities captured in tree`)
}
})
it('should perform well with small workspace (100 entities)', async () => {
console.log('\n📋 Test 5: Performance - 100 entities')
// Add 100 entities
for (let i = 0; i < 100; i++) {
await brain.add({
type:'concept',
data: { id: i, title: `Concept ${i}` }
})
}
console.log(` Added 100 entities`)
// Measure commit time
const start = Date.now()
const commitId = await brain.commit({
message: '100 entity snapshot',
captureState: true
})
const duration = Date.now() - start
console.log(` Commit completed in ${duration}ms`)
console.log(` Commit ID: ${commitId.slice(0, 8)}`)
// Should complete in < 1s
expect(duration).toBeLessThan(1000)
console.log(` ✅ Performance: ${duration}ms < 1000ms`)
})
it('should deduplicate unchanged entities across commits', async () => {
console.log('\n📋 Test 6: BlobStorage deduplication')
// Add initial entities (add() returns entity IDs)
const entity1Id = await brain.add({
type:'person',
data: { name: 'Alice', age: 30 }
})
const entity2Id = await brain.add({
type:'person',
data: { name: 'Bob', age: 25 }
})
// First commit with captureState
const commit1Id = await brain.commit({
message: 'First snapshot',
captureState: true
})
console.log(` Commit 1: ${commit1Id.slice(0, 8)}`)
// Modify ONLY entity2
await brain.update({
id: entity2Id,
data: { name: 'Bob', age: 26 } // Changed age
})
// Second commit with captureState
const commit2Id = await brain.commit({
message: 'Second snapshot',
captureState: true
})
console.log(` Commit 2: ${commit2Id.slice(0, 8)}`)
// Read both trees
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { TreeObject } = await import('../../src/storage/cow/TreeObject.js')
const commit1 = await CommitObject.read(blobStorage, commit1Id)
const commit2 = await CommitObject.read(blobStorage, commit2Id)
const tree1 = await TreeObject.read(blobStorage, commit1.tree)
const tree2 = await TreeObject.read(blobStorage, commit2.tree)
// Find entity1 blob hash in both trees
const entity1Entry1 = tree1.entries.find(e => e.name === `entities/${entity1Id}`)
const entity1Entry2 = tree2.entries.find(e => e.name === `entities/${entity1Id}`)
expect(entity1Entry1).toBeDefined()
expect(entity1Entry2).toBeDefined()
// entity1 unchanged, so blob hash should be SAME (deduplication!)
expect(entity1Entry1!.hash).toBe(entity1Entry2!.hash)
console.log(` ✅ Unchanged entity1 has same blob hash (deduplicated)`)
// Find entity2 blob hash in both trees
const entity2Entry1 = tree1.entries.find(e => e.name === `entities/${entity2Id}`)
const entity2Entry2 = tree2.entries.find(e => e.name === `entities/${entity2Id}`)
expect(entity2Entry1).toBeDefined()
expect(entity2Entry2).toBeDefined()
// entity2 changed, so blob hash should be DIFFERENT
expect(entity2Entry1!.hash).not.toBe(entity2Entry2!.hash)
console.log(` ✅ Changed entity2 has different blob hash (new version)`)
})
it('should create valid tree structure that can be read back', async () => {
console.log('\n📋 Test 7: Tree structure validity')
// Add entities (add() returns entity IDs)
const entity1Id = await brain.add({
type:'person',
data: { name: 'Alice' }
})
const entity2Id = await brain.add({
type:'concept',
data: { title: 'AI' }
})
// Commit with captureState
const commitId = await brain.commit({
message: 'Test snapshot',
captureState: true
})
// Read tree
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { TreeObject } = await import('../../src/storage/cow/TreeObject.js')
const commit = await CommitObject.read(blobStorage, commitId)
const tree = await TreeObject.read(blobStorage, commit.tree)
// Verify tree structure
expect(tree.entries).toBeDefined()
expect(Array.isArray(tree.entries)).toBe(true)
// Walk tree and deserialize entities
let foundEntities = 0
for await (const entry of TreeObject.walk(blobStorage, tree)) {
if (entry.type !== 'blob' || !entry.name.startsWith('entities/')) {
continue
}
// Read blob
const blob = await blobStorage.read(entry.hash)
const entity = JSON.parse(blob.toString())
// Verify entity structure
expect(entity).toHaveProperty('id')
expect(entity).toHaveProperty('type')
expect(entity).toHaveProperty('data')
expect(entity).toHaveProperty('metadata')
console.log(` Found entity: ${entity.id.slice(0, 8)} (${entity.type})`)
foundEntities++
}
// At least 2 entities should be found (the ones we added)
expect(foundEntities).toBeGreaterThanOrEqual(2)
console.log(` ✅ Successfully walked tree and deserialized ${foundEntities} entities`)
})
it('should capture VFS entities along with regular entities', async () => {
console.log('\n📋 Test 8: VFS entity capture')
// Add regular entity (add() returns entity ID)
const conceptId = await brain.add({
type:'concept',
data: { title: 'Test Concept' }
})
// Add VFS file
await brain.vfs.init()
await brain.vfs.writeFile('/test.md', 'Test content')
console.log(` Added concept: ${conceptId.slice(0, 8)}`)
console.log(` Added VFS file: /test.md`)
// Commit with captureState
const commitId = await brain.commit({
message: 'Snapshot with VFS',
captureState: true
})
// Read tree and verify both types of entities are captured
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { TreeObject } = await import('../../src/storage/cow/TreeObject.js')
const commit = await CommitObject.read(blobStorage, commitId)
const tree = await TreeObject.read(blobStorage, commit.tree)
let regularEntities = 0
let vfsEntities = 0
for await (const entry of TreeObject.walk(blobStorage, tree)) {
if (entry.type !== 'blob' || !entry.name.startsWith('entities/')) {
continue
}
const blob = await blobStorage.read(entry.hash)
const entity = JSON.parse(blob.toString())
if (entity.metadata?.isVFS || entity.metadata?.vfsType) {
vfsEntities++
console.log(` Found VFS entity: ${entity.metadata.path}`)
} else {
regularEntities++
console.log(` Found regular entity: ${entity.id.slice(0, 8)} (${entity.type})`)
}
}
expect(regularEntities).toBeGreaterThanOrEqual(1)
expect(vfsEntities).toBeGreaterThanOrEqual(2) // File + root directory
console.log(` ✅ Captured ${regularEntities} regular + ${vfsEntities} VFS entities`)
})
it('should work with Memory storage adapter', async () => {
console.log('\n📋 Test 9: Memory storage adapter')
const memBrain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' },
silent: true
})
await memBrain.init()
// Add entity
await memBrain.add({ type:'concept', data: { title: 'Test' } })
// Commit with captureState
const commitId = await memBrain.commit({
message: 'Memory storage test',
captureState: true
})
const blobStorage = (memBrain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { isNullHash } = await import('../../src/storage/cow/constants.js')
const commit = await CommitObject.read(blobStorage, commitId)
expect(isNullHash(commit.tree)).toBe(false)
console.log(` ✅ Memory storage: tree captured successfully`)
})
it('should work with TypeAware storage adapter', async () => {
console.log('\n📋 Test 10: TypeAware storage adapter')
const testPath = './test-typeaware-commit-capture'
if (fs.existsSync(testPath)) {
fs.rmSync(testPath, { recursive: true, force: true })
}
const typeAwareBrain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'typeaware',
path: testPath
},
silent: true
})
await typeAwareBrain.init()
// Add entity
await typeAwareBrain.add({ type:'person', data: { name: 'Alice' } })
// Commit with captureState
const commitId = await typeAwareBrain.commit({
message: 'TypeAware storage test',
captureState: true
})
const blobStorage = (typeAwareBrain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { isNullHash } = await import('../../src/storage/cow/constants.js')
const commit = await CommitObject.read(blobStorage, commitId)
expect(isNullHash(commit.tree)).toBe(false)
console.log(` ✅ TypeAware storage: tree captured successfully`)
// Cleanup
if (fs.existsSync(testPath)) {
fs.rmSync(testPath, { recursive: true, force: true })
}
})
it('should capture relationships along with entities', async () => {
console.log('\n📋 Test 11: Relationship capture (CRITICAL FIX)')
// Add entities
const aliceId = await brain.add({
type: 'person',
data: { name: 'Alice', role: 'Developer' }
})
const bobId = await brain.add({
type: 'person',
data: { name: 'Bob', role: 'Designer' }
})
const projectId = await brain.add({
type: 'concept',
data: { title: 'Project X', description: 'AI Platform' }
})
console.log(` Added 3 entities: Alice, Bob, Project X`)
// Create relationships (using valid VerbTypes)
await brain.relate({
from: aliceId,
to: projectId,
type: 'relatedTo' // Alice works on Project X
})
await brain.relate({
from: bobId,
to: projectId,
type: 'relatedTo' // Bob works on Project X
})
await brain.relate({
from: aliceId,
to: bobId,
type: 'relatedTo' // Alice collaborates with Bob
})
console.log(` Created 3 relationships`)
// Commit with captureState
const commitId = await brain.commit({
message: 'Snapshot with entities + relationships',
captureState: true
})
console.log(` Commit ID: ${commitId.slice(0, 8)}`)
// Read tree and verify relationships are captured
const blobStorage = (brain.storage as any).blobStorage
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const { TreeObject } = await import('../../src/storage/cow/TreeObject.js')
const commit = await CommitObject.read(blobStorage, commitId)
const tree = await TreeObject.read(blobStorage, commit.tree)
// Count entities and relationships in tree
let entityCount = 0
let relationCount = 0
for (const entry of tree.entries) {
if (entry.name.startsWith('entities/')) {
entityCount++
} else if (entry.name.startsWith('relations/')) {
relationCount++
}
}
console.log(` Tree contains: ${entityCount} entities, ${relationCount} relationships`)
// Verify we have at least the entities and relationships we created
expect(entityCount).toBeGreaterThanOrEqual(3)
expect(relationCount).toBe(3)
console.log(` ✅ Relationships captured in tree!`)
// Verify we can deserialize relationships from tree
let deserializedRelations = 0
for (const entry of tree.entries) {
if (!entry.name.startsWith('relations/')) continue
const blob = await blobStorage.read(entry.hash)
const relation = JSON.parse(blob.toString())
// Verify relationship structure
expect(relation).toHaveProperty('sourceId')
expect(relation).toHaveProperty('targetId')
expect(relation).toHaveProperty('verb')
console.log(` Found relation: ${relation.verb} (${relation.sourceId.slice(0, 8)} → ${relation.targetId.slice(0, 8)})`)
deserializedRelations++
}
expect(deserializedRelations).toBe(3)
console.log(` ✅ Successfully deserialized ${deserializedRelations} relationships from tree`)
})
})

191
tests/integration/cow-commit-storage.test.ts
View file

@ -1,191 +0,0 @@
/**
* Regression test for v5.3.3 bug fix:
* Commits were being stored as blob:${hash} instead of commit:${hash}
*
* This caused getHistory() to return empty arrays because it couldn't find
* commit objects on disk.
*
* Related bugs:
* - v5.3.0 snapshot regression filed via consumer bug report (BRAINY_V5.3.0_SNAPSHOT_BUG_REPORT.md, internal)
* - Root cause: BlobStorage.ts hardcoded 'blob:' prefix in 9 locations
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs/promises'
import * as path from 'path'
describe('COW Commit Storage Type-Aware Prefixes', () => {
let brain: Brainy
let testDir: string
beforeEach(async () => {
testDir = path.join('/tmp', `brainy-cow-test-${Date.now()}`)
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
path: testDir,
branch: 'main',
enableCompression: true
},
disableAutoRebuild: true,
silent: true
})
await brain.init()
})
afterEach(async () => {
// Clean up test directory
try {
await fs.rm(testDir, { recursive: true, force: true })
} catch (err) {
// Ignore cleanup errors
}
})
it('should store commits with commit: prefix (not blob: prefix)', async () => {
// Create a commit
await brain.commit('Test commit for type-aware storage')
// Check filesystem directly
const cowDir = path.join(testDir, '_cow')
const files = await fs.readdir(cowDir)
// Should have commit: files
const commitFiles = files.filter(f => f.startsWith('commit:'))
expect(commitFiles.length).toBeGreaterThan(0)
// Should NOT have blob: files that are actually commits
// (blob: files should only be for actual blob data)
const blobFiles = files.filter(f => f.startsWith('blob:') && !f.includes('-meta'))
// Read metadata of blob files to ensure none are commits
for (const blobFile of blobFiles) {
const metaFile = blobFile.replace('blob:', 'blob:-meta:')
if (files.includes(metaFile)) {
const metaPath = path.join(cowDir, metaFile)
const metaContent = await fs.readFile(metaPath, 'utf8')
const metadata = JSON.parse(metaContent)
expect(metadata.type).not.toBe('commit')
}
}
})
it('should allow getHistory() to retrieve commits', async () => {
// Create multiple commits
await brain.commit('First commit')
await brain.add({
data: 'Testing commit storage',
type: 'concept'
})
await brain.commit('Second commit with entity')
// Get history - should NOT be empty
const history = await brain.getHistory({ limit: 10 })
expect(history).toBeDefined()
expect(Array.isArray(history)).toBe(true)
expect(history.length).toBeGreaterThanOrEqual(2)
// Verify commit structure
expect(history[0]).toHaveProperty('hash')
expect(history[0]).toHaveProperty('message')
expect(history[0]).toHaveProperty('timestamp')
expect(history[0]).toHaveProperty('author')
})
it('should store trees with tree: prefix (if trees are created)', async () => {
// Add an entity and commit
await brain.add({
data: 'Testing tree storage',
type: 'concept'
})
await brain.commit('Commit with tree')
// Check filesystem - trees might or might not be created depending on implementation
// The important thing is that IF trees are created, they use tree: prefix
const cowDir = path.join(testDir, '_cow')
const files = await fs.readdir(cowDir)
// Verify commit files exist (this is the critical part)
const commitFiles = files.filter(f => f.startsWith('commit:'))
expect(commitFiles.length).toBeGreaterThan(0)
// If tree files exist, they should use tree: prefix (not blob:)
const treeFiles = files.filter(f => f.startsWith('tree:'))
const blobTrees = files.filter(f => f.startsWith('blob:') && !f.includes('-meta'))
// Trees should be in tree: files, not blob: files
// (Or no tree files at all if implementation doesn't create them)
expect(blobTrees.filter(f => f.includes('tree')).length).toBe(0)
})
it('should maintain backward compatibility with old blob: prefix', async () => {
// This test verifies that read() auto-detects type by trying multiple prefixes
// We'll verify this by checking that the auto-detection code works
// Create a commit normally (will use commit: prefix)
await brain.commit('Test commit for backward compat check')
// The commit should be readable even though read() tries multiple prefixes
const history = await brain.getHistory({ limit: 1 })
expect(history.length).toBe(1)
// Verify the commit hash is readable via BlobStorage
const blobStorage = (brain as any).storage.blobStorage
const commitHash = history[0].hash
// This should work via auto-detection
const readData = await blobStorage.read(commitHash)
expect(readData).toBeDefined()
expect(readData.length).toBeGreaterThan(0)
})
it('should handle fork() and snapshot creation with proper commit storage', async () => {
// Add some data
await brain.add({
data: 'Testing snapshots',
type: 'concept'
})
await brain.commit('Commit before fork')
// Create a fork (snapshot branch)
const snapshotBranch = `snapshot-${Date.now()}`
await brain.fork(snapshotBranch)
// Get history - should work (this is the critical test)
const history = await brain.getHistory({ limit: 10 })
expect(history.length).toBeGreaterThanOrEqual(1)
// History working proves commits are stored correctly
})
it('should properly delete commits with type-aware prefix', async () => {
// Create a commit
await brain.commit('Commit to delete')
const history = await brain.getHistory({ limit: 1 })
const commitHash = history[0].hash
// Delete via BlobStorage
const blobStorage = (brain as any).storage.blobStorage
await blobStorage.delete(commitHash)
// Verify deleted
const exists = await blobStorage.has(commitHash)
expect(exists).toBe(false)
// Verify files removed from disk
const cowDir = path.join(testDir, '_cow')
const files = await fs.readdir(cowDir)
const commitFile = files.find(f => f.includes(commitHash) && f.startsWith('commit:'))
expect(commitFile).toBeUndefined()
})
})

574
tests/integration/cow-full-integration.test.ts
View file

@ -1,574 +0,0 @@
import { NounType } from '../../src/types/graphTypes.js'
/**
* Comprehensive COW Integration Tests
*
* Verifies COW works with:
* - All 8 storage adapters (Memory, OPFS, FileSystem, S3, R2, GCS, Azure, TypeAware)
* - Billion-scale performance
* - find() graph-aware queries
* - Triple Intelligence natural language
* - VFS (Virtual File System)
* - All 4 indexes (HNSW, Metadata, GraphAdjacency, DeletedItems)
* - Distributed mode (read-only, write-only instances)
* - 256 UUID-based sharding
*
* This is the CRITICAL test that proves v5.0.0 is production-ready.
*/
import { describe, it, expect, beforeEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import type { StorageAdapter } from '../../src/storage/adapters/baseStorageAdapter.js'
describe('COW Full Integration', () => {
describe('Storage Adapter Compatibility', () => {
it('should work with Memory adapter', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Add entity
const entity = await brain.add({
type: NounType.Person,
data: { name: 'Alice' }
})
// Fork (uses COW)
const fork = await brain.fork('test-branch')
// Verify entity exists in fork
const retrieved = await fork.get(entity.id)
expect(retrieved.data.name).toBe('Alice')
await brain.destroy()
await fork.destroy()
})
it('should work with FileSystem adapter', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'filesystem',
path: '/tmp/brainy-cow-test-fs'
}
})
await brain.init()
const entity = await brain.add({
type: NounType.Document,
data: { title: 'Test' }
})
const fork = await brain.fork('fs-branch')
const retrieved = await fork.get(entity.id)
expect(retrieved.data.title).toBe('Test')
await brain.destroy()
await fork.destroy()
})
it('should work with TypeAware adapter (most advanced)', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'typeaware',
path: '/tmp/brainy-cow-test-typeaware'
}
})
await brain.init()
const entity = await brain.add({
type: NounType.Product,
data: { name: 'Widget', price: 99.99 }
})
const fork = await brain.fork('typeaware-branch')
const retrieved = await fork.get(entity.id)
expect(retrieved.data.price).toBe(99.99)
await brain.destroy()
await fork.destroy()
})
})
describe('Billion-Scale Performance', () => {
it('should fork 1M entities in < 2 seconds', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Add 1M entities (representative of billion-scale)
console.log('Adding 1M entities...')
const start = Date.now()
for (let i = 0; i < 1_000_000; i++) {
await brain.add({
type: NounType.Thing,
data: { index: i },
skipCommit: true // Batch commit
})
if (i % 100_000 === 0) {
console.log(` ${i / 1000}K entities added...`)
}
}
await brain.commit({ message: 'Add 1M entities' })
const addTime = Date.now() - start
console.log(`Added 1M entities in ${addTime}ms`)
// Fork (should be instant via COW)
console.log('Forking...')
const forkStart = Date.now()
const fork = await brain.fork('million-test')
const forkTime = Date.now() - forkStart
console.log(`Forked 1M entities in ${forkTime}ms`)
// CRITICAL: Fork must be < 2 seconds
expect(forkTime).toBeLessThan(2000)
// Verify data integrity
const entity = await fork.get((await brain.find({ limit: 1 }))[0].id)
expect(entity.noun).toBe('entity')
await brain.destroy()
await fork.destroy()
}, 120000) // 2-minute timeout for 1M entity test
it('should deduplicate billions of identical vectors', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Same vector used for all entities (simulates common embeddings)
const commonVector = Array(384).fill(0.1)
// Add 10K entities with same vector
for (let i = 0; i < 10_000; i++) {
await brain.add({
type: NounType.Document,
data: { id: i },
vector: commonVector,
skipCommit: true
})
}
await brain.commit({ message: 'Add 10K with duplicate vectors' })
// Check storage stats
const stats = brain.storage.blobStorage.getStats()
// Should have massive deduplication savings
// (10K vectors but only 1 unique blob)
expect(stats.dedupSavings).toBeGreaterThan(0)
console.log(`Deduplication savings: ${(stats.dedupSavings / 1024 / 1024).toFixed(2)}MB`)
await brain.destroy()
}, 60000)
})
describe('find() Integration', () => {
it('should work with graph-aware find() queries', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Create entities with relationships
const alice = await brain.add({
type: NounType.Person,
data: { name: 'Alice' }
})
const bob = await brain.add({
type: NounType.Person,
data: { name: 'Bob' }
})
await brain.addRelationship(alice.id, 'knows', bob.id)
// Fork
const fork = await brain.fork('find-test')
// find() should work on fork
const results = await fork.find({
type: NounType.Person,
where: { name: 'Alice' }
})
expect(results).toHaveLength(1)
expect(results[0].data.name).toBe('Alice')
// Graph traversal should work
const connected = await fork.getVerbsBySource(alice.id)
expect(connected).toHaveLength(1)
expect(connected[0].verb).toBe('knows')
await brain.destroy()
await fork.destroy()
})
it('should preserve metadata index across fork', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Add entities with indexed metadata
await brain.add({
type: NounType.Product,
data: { price: 100, category: 'electronics' }
})
await brain.add({
type: NounType.Product,
data: { price: 200, category: 'electronics' }
})
const fork = await brain.fork('metadata-test')
// Metadata queries should work
const cheap = await fork.find({
type: NounType.Product,
where: { price: { $lt: 150 } }
})
expect(cheap).toHaveLength(1)
expect(cheap[0].data.price).toBe(100)
await brain.destroy()
await fork.destroy()
})
})
describe('Triple Intelligence Integration', () => {
it('should preserve Triple Intelligence across fork', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' },
intelligence: {
enabled: true,
modelDelivery: 'local'
}
})
await brain.init()
// Add entities
await brain.add({
type: NounType.Person,
data: { name: 'Alice', age: 30 }
})
await brain.add({
type: NounType.Person,
data: { name: 'Bob', age: 25 }
})
const fork = await brain.fork('intelligence-test')
// Natural language query should work on fork
const results = await fork.query('people older than 28')
// Should find Alice (age 30)
expect(results.length).toBeGreaterThan(0)
await brain.destroy()
await fork.destroy()
})
})
describe('VFS Integration', () => {
it('should preserve VFS structure across fork', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' },
vfs: { enabled: true }
})
await brain.init()
// Create VFS structure
const folder = await brain.vfs.mkdir('/documents')
const file = await brain.vfs.writeFile('/documents/readme.txt', 'Hello World')
const fork = await brain.fork('vfs-test')
// VFS should work on fork
const content = await fork.vfs.readFile('/documents/readme.txt')
expect(content).toBe('Hello World')
const files = await fork.vfs.readdir('/documents')
expect(files).toContain('readme.txt')
await brain.destroy()
await fork.destroy()
})
it('should support VFS paths in billions of entities', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' },
vfs: { enabled: true }
})
await brain.init()
// Create deep directory structure
await brain.vfs.mkdir('/project/src/components/ui')
// Add 1000 files
for (let i = 0; i < 1000; i++) {
await brain.vfs.writeFile(
`/project/src/components/ui/component${i}.tsx`,
`export default function Component${i}() { return null }`
)
}
const fork = await brain.fork('vfs-scale-test')
// VFS operations should be fast on fork
const files = await fork.vfs.readdir('/project/src/components/ui')
expect(files).toHaveLength(1000)
await brain.destroy()
await fork.destroy()
}, 60000)
})
describe('All 4 Indexes Integration', () => {
it('should preserve HNSW index across fork', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Add entities with vectors
const e1 = await brain.add({
type: NounType.Document,
data: { text: 'machine learning' },
vector: [1, 0, 0]
})
const e2 = await brain.add({
type: NounType.Document,
data: { text: 'artificial intelligence' },
vector: [0.9, 0.1, 0]
})
const fork = await brain.fork('hnsw-test')
// Vector search should work on fork
const similar = await fork.search([1, 0, 0], { limit: 1 })
expect(similar[0].id).toBe(e1.id)
await brain.destroy()
await fork.destroy()
})
it('should preserve GraphAdjacency index across fork', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
const a = await brain.add({ type: NounType.Thing, data: { name: 'A' } })
const b = await brain.add({ type: NounType.Thing, data: { name: 'B' } })
const c = await brain.add({ type: NounType.Thing, data: { name: 'C' } })
await brain.addRelationship(a.id, 'connects', b.id)
await brain.addRelationship(b.id, 'connects', c.id)
const fork = await brain.fork('graph-test')
// Graph queries should work
const outgoing = await fork.getVerbsBySource(a.id)
const incoming = await fork.getVerbsByTarget(b.id)
expect(outgoing).toHaveLength(1)
expect(incoming).toHaveLength(1)
await brain.destroy()
await fork.destroy()
})
it('should preserve DeletedItems index across fork', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
const entity = await brain.add({
type: NounType.Thing,
data: { value: 'test' }
})
await brain.delete(entity.id)
const fork = await brain.fork('deleted-test')
// Deleted items should be tracked in fork
const exists = await fork.get(entity.id).catch(() => null)
expect(exists).toBeNull()
await brain.destroy()
await fork.destroy()
})
})
describe('Distributed Mode', () => {
it('should work with read-only instances', async () => {
// Main brain (read-write)
const main = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await main.init()
const entity = await main.add({
type: NounType.Thing,
data: { value: 'test' }
})
await main.commit({ message: 'Add entity' })
// Read-only instance (shares storage)
const readonly = new Brainy({ requireSubtype: false,
storage: main.config.storage,
readOnly: true
})
await readonly.init()
// Can read from main
const retrieved = await readonly.get(entity.id)
expect(retrieved.data.value).toBe('test')
// Can fork read-only instance
const fork = await readonly.fork('readonly-fork')
const forked = await fork.get(entity.id)
expect(forked.data.value).toBe('test')
await main.destroy()
await readonly.destroy()
await fork.destroy()
})
it('should work with write-only instances', async () => {
// Write-only instance
const writeOnly = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' },
writeOnly: true
})
await writeOnly.init()
await writeOnly.add({
type: NounType.Message,
data: { message: 'test' }
})
await writeOnly.commit({ message: 'Add log' })
// Fork should work even on write-only
const fork = await writeOnly.fork('writeonly-fork')
expect(fork).toBeTruthy()
await writeOnly.destroy()
await fork.destroy()
})
})
describe('256 UUID Sharding', () => {
it('should work with sharded storage', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'memory',
sharding: {
enabled: true,
shardCount: 256
}
}
})
await brain.init()
// Add entities across shards
for (let i = 0; i < 1000; i++) {
await brain.add({
noun: 'sharded',
data: { index: i },
skipCommit: true
})
}
await brain.commit({ message: 'Add sharded entities' })
// Fork should work with sharding
const fork = await brain.fork('sharded-test')
const results = await fork.find({ noun: 'sharded' })
expect(results).toHaveLength(1000)
await brain.destroy()
await fork.destroy()
})
})
describe('Time-Travel Queries (Enterprise Preview)', () => {
it('should support asOf queries', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' }
})
await brain.init()
// Time 1: Add entity
await brain.add({
type: NounType.Document,
data: { version: 1 }
})
await brain.commit({ message: 'Version 1' })
const time1 = Date.now()
// Wait a bit
await new Promise(resolve => setTimeout(resolve, 100))
// Time 2: Update entity
const docs = await brain.find({ type: NounType.Document })
await brain.update(docs[0].id, { version: 2 })
await brain.commit({ message: 'Version 2' })
// asOf(time1) should return version 1
const snapshot = await brain.asOf(time1)
const retrieved = await snapshot.find({ type: NounType.Document })
expect(retrieved[0].data.version).toBe(1)
await brain.destroy()
await snapshot.destroy()
})
})
})

38
tests/integration/db-mvcc.test.ts
View file

@ -952,4 +952,42 @@ describe('8.0 Db API — generational MVCC', () => {
await spec1.release() await spec1.release()
await db.release() await db.release()
}) })
it('transactionLog() returns committed entries newest first, with meta and limit', async () => {
const brain = await openMemoryBrain()
// Nothing committed yet — the log is empty (single-op writes do not log).
await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' })
expect(await brain.transactionLog()).toEqual([])
const first = await brain.transact(
[{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }],
{ meta: { author: 'job-1' } }
)
const second = await brain.transact(
[{ op: 'update', id: uid('txlog-a'), metadata: { v: 2 } }],
{ meta: { author: 'job-2' } }
)
const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }])
const entries = await brain.transactionLog()
expect(entries.map((entry) => entry.generation)).toEqual([
third.generation,
second.generation,
first.generation
])
expect(entries[1].meta).toEqual({ author: 'job-2' })
expect(entries[2].meta).toEqual({ author: 'job-1' })
expect(entries[0].meta).toBeUndefined()
for (const entry of entries) {
expect(entry.timestamp).toBeGreaterThan(0)
}
const limited = await brain.transactionLog({ limit: 2 })
expect(limited.map((entry) => entry.generation)).toEqual([third.generation, second.generation])
await first.release()
await second.release()
await third.release()
})
}) })

144
tests/integration/empty-tree-bug.test.ts
View file

@ -1,144 +0,0 @@
/**
* Reproduction test for empty tree bug in v5.3.1
*
* BUG: brain.getHistory() throws "Blob not found: 0000...0" when commits
* have empty trees (entityCount: 0).
*
* The zero hash represents an empty tree and should be handled gracefully.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
const TEST_DATA_PATH = './test-empty-tree-bug-data'
describe('Empty Tree Bug (v5.3.1)', () => {
let brain: Brainy
beforeEach(async () => {
// Clean up test data
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
path: TEST_DATA_PATH,
branch: 'main',
enableCompression: true
},
disableAutoRebuild: true,
silent: true
})
await brain.init()
})
afterEach(() => {
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
})
it('should handle getHistory() with empty tree commit', async () => {
// Create a commit with NO entities (empty tree)
const commitId = await brain.commit({
message: 'Empty workspace snapshot',
author: 'test-user',
metadata: {
timestamp: Date.now(),
isSnapshot: true
}
})
expect(commitId).toBeTruthy()
console.log('✅ Commit created:', commitId)
// Read the commit blob to inspect it
const blobStorage = (brain.storage as any).blobStorage
const refManager = (brain.storage as any).refManager
const { CommitObject } = await import('../../src/storage/cow/CommitObject.js')
const commit = await CommitObject.read(blobStorage, commitId)
console.log('📄 Commit structure:', JSON.stringify(commit, null, 2))
// Check what the ref points to
const refHash = await refManager.resolveRef('main')
// If ref is zero hash, that's the bug
const zeroHash = '0000000000000000000000000000000000000000000000000000000000000000'
if (refHash === zeroHash) {
throw new Error(`BUG CONFIRMED: Ref "main" points to zero hash! commitId=${commitId}, refHash=${refHash}, tree=${commit.tree}`)
}
// Verify commitId is not zero hash
expect(commitId).not.toBe(zeroHash)
expect(refHash).toBe(commitId)
// This should NOT throw "Blob not found: 0000...0"
// Empty trees should be handled gracefully
try {
const history = await brain.getHistory({ limit: 50 })
expect(history).toBeDefined()
expect(Array.isArray(history)).toBe(true)
console.log('✅ getHistory() works with empty tree')
console.log(' History length:', history.length)
} catch (error: any) {
console.log('❌ Error during getHistory():', error.message)
console.log(' Commit tree hash:', commit.tree)
console.log(' Commit parent hash:', commit.parent)
throw error
}
})
it('should handle multiple commits with empty trees', async () => {
// Create first empty commit
await brain.commit({
message: 'First empty snapshot',
author: 'user1'
})
// Create second empty commit
await brain.commit({
message: 'Second empty snapshot',
author: 'user2'
})
// Get history - should NOT throw
const history = await brain.getHistory({ limit: 10 })
expect(history).toBeDefined()
expect(history.length).toBeGreaterThanOrEqual(2)
console.log('✅ Multiple empty commits work')
})
it('should handle mixed commits (empty + with entities)', async () => {
// Create empty commit
await brain.commit({
message: 'Empty snapshot',
author: 'user1'
})
// Add entity
await brain.add({
data: 'Test entity',
type: 'concept'
})
// Create commit with entities
await brain.commit({
message: 'Snapshot with entities',
author: 'user2'
})
// Get history - should handle both empty and non-empty trees
const history = await brain.getHistory({ limit: 10 })
expect(history).toBeDefined()
expect(history.length).toBeGreaterThanOrEqual(2)
console.log('✅ Mixed commits (empty + entities) work')
})
})

267
tests/integration/fork-persistence.test.ts
View file

@ -1,267 +0,0 @@
/**
* Fork Persistence Integration Test
*
* Tests the complete fork() → listBranches() → checkout() workflow
* to prevent regression of the v5.3.6 bug where fork() silently failed
* to persist branches to storage (internal bug report).
*
* @see https://github.com/soulcraftlabs/brainy/issues/XXX
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy'
import * as fs from 'fs'
import * as path from 'path'
describe('Fork Persistence (v5.3.6 Bug Fix)', () => {
let brain: Brainy
let testDir: string
beforeEach(async () => {
// Use filesystem storage to mirror cloud storage behavior
testDir = `/tmp/brainy-fork-test-${Date.now()}`
brain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'filesystem',
path: testDir
},
silent: true
})
await brain.init()
})
afterEach(async () => {
// Cleanup test directory
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
})
it('should persist forked branches to storage', async () => {
// Add data and commit
await brain.add({ data: { value: 1 }, type: 'concept' })
const commitId = await brain.commit({
message: 'Initial commit',
author: 'test@example.com'
})
expect(commitId).toBeTruthy()
// Fork a new branch
const branchName = `test-branch-${Date.now()}`
await brain.fork(branchName, {
author: 'test@example.com',
message: 'Test fork for persistence'
})
// CRITICAL: Verify branch exists in storage (this would fail in v5.3.5)
const branches = await brain.listBranches()
expect(branches).toContain(branchName)
expect(branches).toContain('main')
})
it('should allow checkout of forked branch', async () => {
// Setup: Add data, commit, fork
await brain.add({ data: { value: 1 }, type: 'concept' })
await brain.commit({
message: 'Initial commit',
author: 'test@example.com'
})
const branchName = `checkout-test-${Date.now()}`
await brain.fork(branchName, {
author: 'test@example.com',
message: 'Test fork for checkout'
})
// CRITICAL: Checkout should succeed (this would fail in v5.3.5)
await expect(brain.checkout(branchName)).resolves.not.toThrow()
// Verify we're on the new branch
const currentBranch = await brain.getCurrentBranch()
expect(currentBranch).toBe(branchName)
})
it('should persist fork across multiple storage operations', async () => {
// Create initial state
await brain.add({ data: { step: 1 }, type: 'concept' })
await brain.commit({
message: 'Step 1',
author: 'test@example.com'
})
// Fork branch 1
const branch1 = `branch-1-${Date.now()}`
await brain.fork(branch1, {
author: 'test@example.com',
message: 'Fork 1'
})
// Add more data on main
await brain.add({ data: { step: 2 }, type: 'concept' })
await brain.commit({
message: 'Step 2 on main',
author: 'test@example.com'
})
// Fork branch 2
const branch2 = `branch-2-${Date.now()}`
await brain.fork(branch2, {
author: 'test@example.com',
message: 'Fork 2'
})
// Verify both branches exist
const branches = await brain.listBranches()
expect(branches).toContain('main')
expect(branches).toContain(branch1)
expect(branches).toContain(branch2)
// Verify all branches are accessible
await expect(brain.checkout(branch1)).resolves.not.toThrow()
await expect(brain.checkout(branch2)).resolves.not.toThrow()
await expect(brain.checkout('main')).resolves.not.toThrow()
})
it('should return new Brainy instance pointing to fork', async () => {
// Verify fork() returns a working Brainy instance on the new branch
await brain.add({ data: { value: 1 }, type: 'concept' })
await brain.commit({
message: 'Initial commit',
author: 'test@example.com'
})
// Fork and get new instance
const validBranch = `return-test-${Date.now()}`
const forkedBrain = await brain.fork(validBranch, {
author: 'test@example.com',
message: 'Test fork return value'
})
// Verify the returned instance is on the new branch
const forkCurrentBranch = await forkedBrain.getCurrentBranch()
expect(forkCurrentBranch).toBe(validBranch)
// Verify original instance is still on main
const mainCurrentBranch = await brain.getCurrentBranch()
expect(mainCurrentBranch).toBe('main')
// Verify both instances can see the branch
const mainBranches = await brain.listBranches()
const forkBranches = await forkedBrain.listBranches()
expect(mainBranches).toContain(validBranch)
expect(forkBranches).toContain(validBranch)
})
it('should handle snapshot naming convention (consumer use case)', async () => {
// Reproduce exact Consumer snapshot workflow
await brain.add({ data: { test: true }, type: 'concept' })
const commitId = await brain.commit({
message: 'Consumer snapshot test',
author: 'workshop@example.com'
})
// Use the consumer's exact naming convention
const timestamp = Date.now()
const snapshotBranch = `snapshot-${timestamp}`
// Create snapshot branch (this is what failed in the consumer report)
await brain.fork(snapshotBranch, {
author: 'workshop@example.com',
message: `Snapshot: Consumer test`,
metadata: {
timestamp,
userId: 'test-user',
isSnapshot: true,
snapshotCommit: commitId
}
})
// Verify snapshot can be listed
const branches = await brain.listBranches()
expect(branches).toContain(snapshotBranch)
// Verify snapshot can be restored (checked out)
await expect(brain.checkout(snapshotBranch)).resolves.not.toThrow()
// Verify we're on the snapshot branch
const currentBranch = await brain.getCurrentBranch()
expect(currentBranch).toBe(snapshotBranch)
})
it('should verify _cow paths are not branch-scoped', async () => {
// This test verifies the resolveBranchPath fix works correctly
// by checking that COW metadata is accessible globally
await brain.add({ data: { value: 1 }, type: 'concept' })
await brain.commit({
message: 'Test commit',
author: 'test@example.com'
})
const branch1 = `branch-a-${Date.now()}`
const branch2 = `branch-b-${Date.now()}`
// Create two branches
await brain.fork(branch1, {
author: 'test@example.com',
message: 'Branch A'
})
await brain.fork(branch2, {
author: 'test@example.com',
message: 'Branch B'
})
// From any branch, we should be able to list ALL branches
// This proves COW metadata is global, not branch-scoped
await brain.checkout(branch1)
let branches = await brain.listBranches()
expect(branches).toContain('main')
expect(branches).toContain(branch1)
expect(branches).toContain(branch2)
await brain.checkout(branch2)
branches = await brain.listBranches()
expect(branches).toContain('main')
expect(branches).toContain(branch1)
expect(branches).toContain(branch2)
await brain.checkout('main')
branches = await brain.listBranches()
expect(branches).toContain('main')
expect(branches).toContain(branch1)
expect(branches).toContain(branch2)
})
it('should persist fork with metadata', async () => {
// Test that metadata is preserved in fork
await brain.add({ data: { value: 1 }, type: 'concept' })
await brain.commit({
message: 'Test commit',
author: 'test@example.com'
})
const branchName = `metadata-test-${Date.now()}`
const metadata = {
timestamp: Date.now(),
userId: 'test-user-123',
customField: 'test-value',
nested: { data: true }
}
await brain.fork(branchName, {
author: 'test@example.com',
message: 'Fork with metadata',
metadata
})
// Verify branch exists
const branches = await brain.listBranches()
expect(branches).toContain(branchName)
// Verify branch is accessible
await expect(brain.checkout(branchName)).resolves.not.toThrow()
})
})

142
tests/integration/history-ref-resolution-bug.test.ts
View file

@ -1,142 +0,0 @@
/**
* Regression test for v5.3.0 history ref resolution bug
*
* BUG: brain.getHistory() was passing `heads/${branch}` to commitLog.getHistory(),
* but RefManager.normalizeRefName() expects either:
* - Full ref: 'refs/heads/main'
* - Short name: 'main' (normalized to 'refs/heads/main')
*
* Passing 'heads/main' caused normalization to 'refs/heads/heads/main' → ref not found!
*
* Fixed in: src/brainy.ts lines 2354, 2856, 2895
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
const TEST_DATA_PATH = './test-history-ref-bug-data'
describe('History Ref Resolution Bug (v5.3.0)', () => {
let brain: Brainy
beforeEach(async () => {
// Clean up test data
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
path: TEST_DATA_PATH,
branch: 'main',
enableCompression: true
},
disableAutoRebuild: true,
silent: true
})
await brain.init()
})
afterEach(() => {
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
})
it('should retrieve history after commit (bug: ref not found)', async () => {
// Add an entity
await brain.add({
data: 'Test entity for history bug',
type: 'concept',
metadata: { test: true }
})
// Create a commit
const commitId = await brain.commit({
message: 'Test snapshot',
author: 'dev-user',
metadata: {
timestamp: Date.now(),
isSnapshot: true
}
})
expect(commitId).toBeTruthy()
expect(commitId.length).toBe(64) // SHA-256 hash
// THIS WAS FAILING with "Ref not found: heads/main"
// Because getHistory() passed 'heads/main' instead of 'main'
// After fix: should not throw "Ref not found" error
try {
const history = await brain.getHistory({ limit: 50 })
expect(history).toBeDefined()
expect(Array.isArray(history)).toBe(true)
// Note: history might be empty due to empty tree blob issue,
// but the important part is NO "Ref not found: heads/main" error
} catch (error: any) {
// Should NOT get "Ref not found: heads/main" error
expect(error.message).not.toContain('Ref not found: heads/main')
// Other errors (like tree blob) are separate issues
if (error.message.includes('Blob not found')) {
// This is a known issue with empty tree - not our bug
console.log('✅ Ref resolution working (tree blob issue is separate)')
return
}
throw error
}
})
it('should not throw ref resolution error for multiple commits', async () => {
// Create first commit
await brain.add({ data: 'Entity 1', type: 'concept' })
await brain.commit({
message: 'First commit',
author: 'user1'
})
// Create second commit
await brain.add({ data: 'Entity 2', type: 'concept' })
await brain.commit({
message: 'Second commit',
author: 'user2'
})
// Get history - should NOT throw "Ref not found: heads/main"
try {
const history = await brain.getHistory({ limit: 10 })
expect(history).toBeDefined()
console.log('✅ getHistory() works without ref resolution error')
} catch (error: any) {
// Should NOT get "Ref not found: heads/main" error
expect(error.message).not.toContain('Ref not found: heads/main')
expect(error.message).not.toContain('Ref not found: main')
// Other errors are acceptable
console.log('✅ No ref resolution error (other error is acceptable)')
}
})
it('should use fork() to test COW (correct API)', async () => {
// fork() is the public API that uses COW internally
await brain.add({
data: 'Original entity',
type: 'concept'
})
// Fork creates a new branch and switches to it
const fork = await brain.fork('feature-branch')
// Add entity in fork
await fork.add({
data: 'Fork entity',
type: 'concept'
})
// Both should work without ref resolution errors
expect(fork).toBeDefined()
console.log('✅ fork() works (uses COW refs internally)')
})
})

150
tests/integration/hybrid-search-cow.test.ts
View file

@ -1,150 +0,0 @@
/**
* Hybrid Search COW Integration Tests (v7.7.0)
*
* Verifies that hybrid search works correctly with:
* - fork() - COW branching
* - asOf() - Historical queries
* - VFS entities
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy'
import { NounType } from '../../src/types/graphTypes'
import * as fs from 'fs'
import * as path from 'path'
import * as os from 'os'
describe('Hybrid Search with COW (v7.7.0)', () => {
let brain: Brainy<any>
let testDir: string
beforeEach(async () => {
// Use filesystem storage for COW support
testDir = path.join(os.tmpdir(), `brainy-cow-test-${Date.now()}`)
fs.mkdirSync(testDir, { recursive: true })
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { basePath: testDir }
}
})
await brain.init()
})
afterEach(async () => {
await brain.close()
// Cleanup test directory
try {
fs.rmSync(testDir, { recursive: true, force: true })
} catch (e) {
// Ignore cleanup errors
}
})
describe('fork() compatibility', () => {
it('should perform hybrid search in forked brain', async () => {
// Add entity to main branch
const mainId = await brain.add({
data: 'Python programming language tutorial',
type: NounType.Document,
metadata: { branch: 'main' }
})
// Commit main branch
await brain.commit({ message: 'Add Python doc' })
// Fork
const fork = await brain.fork('feature-branch')
// Add entity to fork
const forkId = await fork.add({
data: 'JavaScript programming guide',
type: NounType.Document,
metadata: { branch: 'feature' }
})
// Hybrid search in fork should find both
const forkResults = await fork.find({
query: 'programming',
limit: 10
})
expect(forkResults.length).toBeGreaterThanOrEqual(2)
expect(forkResults.some(r => r.id === mainId)).toBe(true)
expect(forkResults.some(r => r.id === forkId)).toBe(true)
// Hybrid search in main should only find main entity
const mainResults = await brain.find({
query: 'programming',
limit: 10
})
expect(mainResults.some(r => r.id === mainId)).toBe(true)
// Fork entity should NOT be visible in main
expect(mainResults.some(r => r.id === forkId)).toBe(false)
await fork.close()
})
it('should support text-only search in forked brain', async () => {
await brain.add({
data: 'exact keyword match test',
type: NounType.Document,
metadata: { test: true }
})
await brain.commit({ message: 'Add test doc' })
const fork = await brain.fork('text-search-test')
const results = await fork.find({
query: 'exact keyword',
searchMode: 'text',
limit: 5
})
expect(results.length).toBeGreaterThan(0)
await fork.close()
})
})
describe('VFS compatibility', () => {
it('should include VFS file content in hybrid search', async () => {
// Write a file via VFS
const vfs = brain.vfs
await vfs.writeFile('/docs/readme.txt', 'This is a readme file with important information')
// Search should find VFS content
const results = await brain.find({
query: 'readme important',
limit: 10
})
// VFS entities should appear in results
expect(results.length).toBeGreaterThan(0)
})
it('should exclude VFS with excludeVFS flag in hybrid search', async () => {
// Add regular entity
const entityId = await brain.add({
data: 'regular document about readme files',
type: NounType.Document,
metadata: { source: 'api' }
})
// Write VFS file
await brain.vfs.writeFile('/readme.md', 'VFS readme content')
// Search with excludeVFS should only find regular entity
const results = await brain.find({
query: 'readme',
excludeVFS: true,
limit: 10
})
expect(results.some(r => r.id === entityId)).toBe(true)
// VFS entities should be excluded
expect(results.every(r => !r.metadata?.vfsType)).toBe(true)
})
})
})

80
tests/integration/hybrid-search-vfs.test.ts Normal file
View file

@ -0,0 +1,80 @@
/**
* Hybrid Search VFS Integration Tests
*
* Verifies that hybrid search works correctly with VFS entities:
* file content is searchable by default, and `excludeVFS` filters
* VFS entities out of results.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy'
import { NounType } from '../../src/types/graphTypes'
import * as fs from 'fs'
import * as path from 'path'
import * as os from 'os'
describe('Hybrid Search with VFS', () => {
let brain: Brainy<any>
let testDir: string
beforeEach(async () => {
testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`)
fs.mkdirSync(testDir, { recursive: true })
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { basePath: testDir }
}
})
await brain.init()
})
afterEach(async () => {
await brain.close()
// Cleanup test directory
try {
fs.rmSync(testDir, { recursive: true, force: true })
} catch (e) {
// Ignore cleanup errors
}
})
it('should include VFS file content in hybrid search', async () => {
// Write a file via VFS
const vfs = brain.vfs
await vfs.writeFile('/docs/readme.txt', 'This is a readme file with important information')
// Search should find VFS content
const results = await brain.find({
query: 'readme important',
limit: 10
})
// VFS entities should appear in results
expect(results.length).toBeGreaterThan(0)
})
it('should exclude VFS with excludeVFS flag in hybrid search', async () => {
// Add regular entity
const entityId = await brain.add({
data: 'regular document about readme files',
type: NounType.Document,
metadata: { source: 'api' }
})
// Write VFS file
await brain.vfs.writeFile('/readme.md', 'VFS readme content')
// Search with excludeVFS should only find regular entity
const results = await brain.find({
query: 'readme',
excludeVFS: true,
limit: 10
})
expect(results.some(r => r.id === entityId)).toBe(true)
// VFS entities should be excluded
expect(results.every(r => !r.metadata?.vfsType)).toBe(true)
})
})

143
tests/integration/initial-commit-null-parent.test.ts
View file

@ -1,143 +0,0 @@
/**
* Regression test for v5.3.3 NULL parent bug
*
* BUG: CommitObject.walk() didn't guard against NULL parent hash, causing
* "Blob metadata not found: 0000...0000" error when calling getHistory()
* on a fresh database with only the initial commit.
*
* ROOT CAUSE: Initial commit has parent = null or NULL_HASH ('0000...0000'),
* but while(currentHash) treated the string as truthy and tried to read it.
*
* FIXED IN: v5.3.4
* - Added isNullHash() guard in CommitObject.walk()
* - Added defensive check in BlobStorage.read()
* - Created NULL_HASH constant to prevent hardcoding
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NULL_HASH } from '../../src/storage/cow/constants.js'
import * as fs from 'fs'
const TEST_DATA_PATH = './test-null-parent-data'
describe('Initial Commit NULL Parent Bug (v5.3.3 regression)', () => {
let brain: Brainy
beforeEach(async () => {
// Clean up test data
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
path: TEST_DATA_PATH,
branch: 'main',
enableCompression: true
},
disableAutoRebuild: true,
silent: true
})
await brain.init()
})
afterEach(() => {
if (fs.existsSync(TEST_DATA_PATH)) {
fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true })
}
})
it('should handle getHistory() on fresh database with only initial commit', async () => {
// Fresh brain has only the initial commit (created during init)
// This was throwing "Blob metadata not found: 0000...0000" in v5.3.3
// Get history - should NOT throw
const history = await brain.getHistory({ limit: 10 })
// Should return the initial commit
expect(history).toBeDefined()
expect(Array.isArray(history)).toBe(true)
expect(history.length).toBe(1)
// Initial commit should have specific properties
const initialCommit = history[0]
expect(initialCommit.message).toBe('Initial commit')
expect(initialCommit.author).toBe('system')
// Parent can be null or undefined for initial commit
expect(initialCommit.parent == null).toBe(true)
})
it('should stop walk at initial commit (parent = null)', async () => {
// Note: brain.commit() creates "Snapshot commit" messages by default
// Create 2 commits
await brain.add({ data: 'First entity', type: 'concept' })
await brain.commit('First user commit')
await brain.add({ data: 'Second entity', type: 'concept' })
await brain.commit('Second user commit')
// Get full history
const history = await brain.getHistory({ limit: 100 })
// Should have at least 3 commits (2 user + 1 initial)
expect(history.length).toBeGreaterThanOrEqual(3)
// Last commit should be initial commit with no parent
const initialCommit = history[history.length - 1]
expect(initialCommit.message).toBe('Initial commit')
// Parent can be null or undefined for initial commit
expect(initialCommit.parent == null).toBe(true)
})
it('should not attempt to read NULL hash from BlobStorage', async () => {
// Direct test: attempting to read NULL_HASH should throw clear error
const blobStorage = (brain as any).storage.blobStorage
// Should throw clear error message
await expect(
blobStorage.read(NULL_HASH)
).rejects.toThrow(/sentinel value/)
// Error message should mention what NULL_HASH is for
await expect(
blobStorage.read(NULL_HASH)
).rejects.toThrow(/no parent/)
})
it('should handle multiple calls to getHistory() consistently', async () => {
// Ensure caching doesn't break NULL hash handling
// First call
const history1 = await brain.getHistory({ limit: 10 })
expect(history1.length).toBe(1)
// Add a commit
await brain.add({ data: 'Test entity', type: 'concept' })
await brain.commit('Test commit')
// Second call (should now have 2 commits)
const history2 = await brain.getHistory({ limit: 10 })
expect(history2.length).toBe(2)
// Third call (should still have 2 commits)
const history3 = await brain.getHistory({ limit: 10 })
expect(history3.length).toBe(2)
// All should end with initial commit
expect(history1[history1.length - 1].message).toBe('Initial commit')
expect(history2[history2.length - 1].message).toBe('Initial commit')
expect(history3[history3.length - 1].message).toBe('Initial commit')
})
it('should handle walking with maxDepth on initial commit', async () => {
// Edge case: maxDepth = 1 on fresh database
const history = await brain.getHistory({ limit: 1 })
expect(history.length).toBe(1)
expect(history[0].message).toBe('Initial commit')
})
})

153
tests/integration/memory-enhancements-v5.11.0.test.ts
View file

@ -2,7 +2,6 @@
* Integration tests for Memory Enhancements (v5.11.0) * Integration tests for Memory Enhancements (v5.11.0)
* *
* End-to-end tests verifying: * End-to-end tests verifying:
* - streamHistory() works in production scenarios
* - Container detection works correctly * - Container detection works correctly
* - Memory limits are applied correctly * - Memory limits are applied correctly
* - getMemoryStats() provides accurate information * - getMemoryStats() provides accurate information
@ -40,86 +39,6 @@ describe('Memory Enhancements Integration (v5.11.0)', () => {
}) })
}) })
describe('Production Workflow: Consumer Snapshot Timeline', () => {
it('should stream 1000 snapshots efficiently', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { path: testDir }
},
silent: true
})
await brain.init()
// Create a realistic workflow: user creates entities and saves snapshots
for (let i = 0; i < 100; i++) {
// Add 10 entities
for (let j = 0; j < 10; j++) {
await brain.add({
type: 'document',
data: `Chapter ${i}, Section ${j}`
})
}
// Save snapshot
await brain.commit({ message: `Version ${i}`, captureState: true })
}
// Stream history efficiently
const snapshots: string[] = []
const startTime = Date.now()
for await (const commit of brain.streamHistory({ limit: 100 })) {
snapshots.push(commit.message)
}
const duration = Date.now() - startTime
expect(snapshots.length).toBe(100)
expect(duration).toBeLessThan(5000) // Should complete in < 5s
await brain.close()
})
it('should handle large snapshot with filtering', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { path: testDir }
},
silent: true
})
await brain.init()
// Create snapshots by different authors
for (let i = 0; i < 50; i++) {
await brain.add({ type: 'document', data: `Content ${i}` })
await brain.commit({
message: `Snapshot ${i}`,
author: i % 3 === 0 ? 'alice' : i % 3 === 1 ? 'bob' : 'charlie',
captureState: true
})
}
// Stream only alice's snapshots
const aliceSnapshots: any[] = []
for await (const commit of brain.streamHistory({ author: 'alice', limit: 100 })) {
aliceSnapshots.push(commit)
}
expect(aliceSnapshots.length).toBeGreaterThan(0)
aliceSnapshots.forEach(commit => {
expect(commit.author).toBe('alice')
})
await brain.close()
})
})
describe('Production Workflow: Cloud Run Deployment', () => { describe('Production Workflow: Cloud Run Deployment', () => {
it('should detect 4GB Cloud Run container and set optimal limits', async () => { it('should detect 4GB Cloud Run container and set optimal limits', async () => {
process.env.CLOUD_RUN_MEMORY = '4Gi' process.env.CLOUD_RUN_MEMORY = '4Gi'
@ -190,51 +109,6 @@ describe('Memory Enhancements Integration (v5.11.0)', () => {
}) })
}) })
describe('Combined Features: Stream + Memory Management', () => {
it('should stream large histories within memory limits', async () => {
process.env.CLOUD_RUN_MEMORY = '2Gi'
const brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { path: testDir }
},
silent: true
})
await brain.init()
// Create many snapshots
for (let i = 0; i < 200; i++) {
await brain.add({ type: 'document', data: `Data ${i}` })
if (i % 5 === 0) {
await brain.commit({ message: `Checkpoint ${i / 5}`, captureState: true })
}
}
// Verify memory limits
const stats = brain.getMemoryStats()
expect(stats.limits.maxQueryLimit).toBe(5000) // 2GB * 0.25
// Stream all snapshots (should be ~40)
const heapBefore = process.memoryUsage().heapUsed
let count = 0
for await (const commit of brain.streamHistory({ limit: 100 })) {
count++
}
const heapAfter = process.memoryUsage().heapUsed
const heapGrowth = heapAfter - heapBefore
expect(count).toBeGreaterThan(0)
expect(heapGrowth).toBeLessThan(20 * 1024 * 1024) // < 20MB growth
await brain.close()
})
})
describe('Memory Stats API', () => { describe('Memory Stats API', () => {
it('should provide actionable recommendations', async () => { it('should provide actionable recommendations', async () => {
process.env.CLOUD_RUN_MEMORY = '4Gi' process.env.CLOUD_RUN_MEMORY = '4Gi'
@ -352,32 +226,5 @@ describe('Memory Enhancements Integration (v5.11.0)', () => {
await brain.close() await brain.close()
}) })
it('should keep getHistory() working as before', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { path: testDir }
},
silent: true
})
await brain.init()
// Create some snapshots
for (let i = 0; i < 10; i++) {
await brain.add({ type: 'note', data: `Test ${i}` })
await brain.commit({ message: `Snapshot ${i}`, captureState: true })
}
// Old API still works
const history = await brain.getHistory({ limit: 10 })
expect(history.length).toBe(10)
expect(history[0]).toHaveProperty('hash')
expect(history[0]).toHaveProperty('message')
await brain.close()
})
}) })
}) })

34
tests/integration/metadata-only-comprehensive.test.ts
View file

@ -291,40 +291,6 @@ describe('Metadata-Only Comprehensive Integration (v5.11.1)', () => {
}) })
}) })
describe('COW and Fork', () => {
it('should work with metadata-only in forks', async () => {
const brain = new Brainy({ requireSubtype: false,
storage: { adapter: 'memory' },
silent: true
})
await brain.init()
const id = await brain.add({
data: 'Original',
type: NounType.Document,
metadata: { version: 1 }
})
await brain.commit({ message: 'Initial commit' })
const fork = await brain.fork('test-branch')
// Metadata-only in fork
const entity = await fork.get(id)
expect(entity).toBeTruthy()
expect(entity!.metadata.version).toBe(1)
expect(entity!.vector).toEqual([])
// Full entity in fork
const full = await fork.get(id, { includeVectors: true })
expect(full!.vector.length).toBe(384)
await brain.close()
await fork.close()
})
})
describe('Performance Verification', () => { describe('Performance Verification', () => {
it('metadata-only should be significantly faster', async () => { it('metadata-only should be significantly faster', async () => {
const brain = new Brainy({ requireSubtype: false, const brain = new Brainy({ requireSubtype: false,

224
tests/integration/migration.test.ts
View file

@ -4,7 +4,7 @@
* Verifies: * Verifies:
* - MigrationRunner with in-memory storage * - MigrationRunner with in-memory storage
* - brain.migrate() public API (dry-run and apply) * - brain.migrate() public API (dry-run and apply)
* - Backup branch creation with metadata tagging * - Pre-migration snapshots via `backupTo` (persist-before-migrate + restore)
* - No-op when MIGRATIONS array is empty * - No-op when MIGRATIONS array is empty
* - Warning log when autoMigrate is false * - Warning log when autoMigrate is false
* - Multi-tenant independent migration state * - Multi-tenant independent migration state
@ -12,6 +12,9 @@
*/ */
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/brainy.js' import { Brainy } from '../../src/brainy.js'
import { MigrationRunner, MIGRATIONS } from '../../src/migration/index.js' import { MigrationRunner, MIGRATIONS } from '../../src/migration/index.js'
import type { Migration } from '../../src/migration/index.js' import type { Migration } from '../../src/migration/index.js'
@ -242,13 +245,21 @@ describe('Migration System', () => {
}) })
}) })
// ─── Backup branch tests ────────────────────────────────────── // ─── Pre-migration snapshot tests ─────────────────────────────
describe('Backup branches', () => { describe('Pre-migration snapshots (backupTo)', () => {
it('should create a backup branch before migrating', async () => { let backupDir: string
beforeEach(() => {
backupDir = path.join(os.tmpdir(), `brainy-migration-backup-${Date.now()}-${Math.random().toString(36).slice(2)}`)
})
afterEach(() => {
fs.rmSync(backupDir, { recursive: true, force: true })
})
it('should persist a snapshot before migrating and report its path', async () => {
await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { x: 1 } }) await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { x: 1 } })
// Need a commit for fork to work
await brain.commit({ message: 'test', author: 'test' })
const migration: Migration = { const migration: Migration = {
id: 'test-backup', id: 'test-backup',
@ -259,45 +270,19 @@ describe('Migration System', () => {
} }
await withMigrations([migration], async () => { await withMigrations([migration], async () => {
const result = await brain.migrate() const result = await brain.migrate({ backupTo: backupDir })
const r = result as any const r = result as any
expect(r.backupBranch).toBe('pre-migration-2.0.0') expect(r.backupPath).toBe(backupDir)
expect(r.migrationsApplied).toContain('test-backup')
const branches = await brain.listBranches() // The snapshot is a self-contained store directory
expect(branches).toContain('pre-migration-2.0.0') expect(fs.existsSync(backupDir)).toBe(true)
expect(fs.readdirSync(backupDir).length).toBeGreaterThan(0)
}) })
}) })
it('should tag backup branch with system:backup metadata', async () => { it('should restore the pre-migration state wholesale from the snapshot', async () => {
await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { z: 1 } })
await brain.commit({ message: 'test', author: 'test' })
const migration: Migration = {
id: 'test-tag',
version: '4.0.0',
description: 'Tag test',
applies: 'nouns',
transform: (m) => 'z' in m ? { ...m, tagged: true } : null
}
await withMigrations([migration], async () => {
await brain.migrate()
// Verify metadata tag on the backup branch ref
const refManager = (brain as any).storage.refManager
if (refManager) {
const ref = await refManager.getRef('pre-migration-4.0.0')
expect(ref).toBeDefined()
expect(ref?.metadata?.type).toBe('system:backup')
expect(ref?.metadata?.migrationVersion).toBe('4.0.0')
expect(ref?.metadata?.author).toBe('brainy-migration')
}
})
})
it('should keep backup branch as a named restore point', async () => {
const id = await brain.add({ type: NounType.Concept, data: { name: 'Restore Test' }, metadata: { original: true } }) const id = await brain.add({ type: NounType.Concept, data: { name: 'Restore Test' }, metadata: { original: true } })
await brain.commit({ message: 'before migration', author: 'test' })
const migration: Migration = { const migration: Migration = {
id: 'test-restore', id: 'test-restore',
@ -313,29 +298,39 @@ describe('Migration System', () => {
} }
await withMigrations([migration], async () => { await withMigrations([migration], async () => {
const result = await brain.migrate() const result = await brain.migrate({ backupTo: backupDir })
const r = result as any const r = result as any
expect(r.backupPath).toBe(backupDir)
// Verify migration was applied // Verify migration was applied
const migrated = await brain.get(id) const migrated = await brain.get(id)
expect(migrated?.metadata?.migrated).toBe(true) expect(migrated?.metadata?.migrated).toBe(true)
expect(migrated?.metadata?.original).toBe(false)
// Verify backup branch exists as a named restore point // Restore the snapshot — the store returns to its pre-migration state
expect(r.backupBranch).toBe('pre-migration-3.0.0') await brain.restore(backupDir, { confirm: true })
const branches = await brain.listBranches() const restored = await brain.get(id)
expect(branches).toContain('pre-migration-3.0.0') expect(restored?.metadata?.original).toBe(true)
expect(restored?.metadata?.migrated).toBeUndefined()
})
})
// Verify the backup ref points to the pre-migration commit it('should report backupPath null when no backupTo is supplied', async () => {
const refManager = (brain as any).storage.refManager await brain.add({ type: NounType.Concept, data: { name: 'NoBackup' }, metadata: { q: 1 } })
if (refManager) {
const mainRef = await refManager.getRef('main') const migration: Migration = {
const backupRef = await refManager.getRef('pre-migration-3.0.0') id: 'test-no-backup',
expect(backupRef).toBeDefined() version: '4.0.0',
// Backup was forked before migration, so both share the same commit description: 'Add field',
// (the pre-migration commit). After migration, main's overlay has new data, applies: 'nouns',
// but the commit pointer is unchanged. transform: (m) => 'q' in m && !('r' in m) ? { ...m, r: 2 } : null
expect(backupRef.commitHash).toBe(mainRef.commitHash)
} }
await withMigrations([migration], async () => {
const result = await brain.migrate()
const r = result as any
expect(r.backupPath).toBeNull()
expect(r.migrationsApplied).toContain('test-no-backup')
}) })
}) })
}) })
@ -520,86 +515,6 @@ describe('Migration System', () => {
}) })
}) })
// ─── Multi-branch migration ─────────────────────────────────
describe('Multi-branch migration', () => {
it('should migrate entities on all branches including feature branches', async () => {
// Setup: create entities on main, fork a branch, add entities on the branch
const mainId = await brain.add({
type: NounType.Concept,
data: { name: 'Main Entity' },
metadata: { legacy: true, source: 'main' }
})
await brain.commit({ message: 'initial', author: 'test' })
// Fork a feature branch and add branch-local entity
const fork = await brain.fork('feature-x')
const branchId = await fork.add({
type: NounType.Concept,
data: { name: 'Branch Entity' },
metadata: { legacy: true, source: 'branch' }
})
// Define migration that transforms the 'legacy' field
const migration: Migration = {
id: 'test-multi-branch',
version: '1.0.0',
description: 'Mark legacy entities as upgraded',
applies: 'nouns',
transform: (m) => {
if (m.legacy === true) {
return { ...m, legacy: false, upgraded: true }
}
return null // Already migrated or not applicable
}
}
await withMigrations([migration], async () => {
// Migrate from main — should reach all branches
const result = await brain.migrate()
const r = result as any
expect(r.migrationsApplied).toContain('test-multi-branch')
// Main entity + branch-local entity should both be modified
expect(r.entitiesModified).toBeGreaterThanOrEqual(2)
// Verify main entity was transformed
const mainEntity = await brain.get(mainId)
expect(mainEntity?.metadata?.upgraded).toBe(true)
expect(mainEntity?.metadata?.legacy).toBe(false)
})
await fork.close()
})
it('should skip system:backup branches during multi-branch migration', async () => {
await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { v: 1 } })
await brain.commit({ message: 'test', author: 'test' })
// Create two migrations — first creates a backup branch, second should skip it
const migration1: Migration = {
id: 'test-skip-backup-1',
version: '5.0.0',
description: 'First migration',
applies: 'nouns',
transform: (m) => typeof m.v === 'number' && !('m1' in m) ? { ...m, m1: true } : null
}
await withMigrations([migration1], async () => {
const result = await brain.migrate()
const r = result as any
expect(r.backupBranch).toBe('pre-migration-5.0.0')
// Verify backup branch exists
const branches = await brain.listBranches()
expect(branches).toContain('pre-migration-5.0.0')
// Migration should not have errored from trying to migrate the backup
expect(r.migrationsApplied).toContain('test-skip-backup-1')
})
})
})
// ─── MigrationRunner unit-level tests ───────────────────────── // ─── MigrationRunner unit-level tests ─────────────────────────
describe('MigrationRunner', () => { describe('MigrationRunner', () => {
@ -684,47 +599,6 @@ describe('Migration System', () => {
}) })
}) })
it('should propagate errors from branch migrations into the result', async () => {
// Create entity on main, commit, fork a branch, add a branch-local entity that will fail
await brain.add({ type: NounType.Concept, data: { name: 'MainOk' }, metadata: { branchTest: 1 } })
await brain.commit({ message: 'initial', author: 'test' })
const fork = await brain.fork('branch-with-error')
await fork.add({ type: NounType.Concept, data: { name: 'BranchBad' }, metadata: { branchTest: 'crash' } })
const migration: Migration = {
id: 'test-branch-error-prop',
version: '1.0.0',
description: 'Throws on non-number branchTest',
applies: 'nouns',
transform: (m) => {
if ('branchTest' in m) {
if (typeof m.branchTest !== 'number') {
throw new Error('branchTest must be a number')
}
return { ...m, branchTest: (m.branchTest as number) * 10 }
}
return null
}
}
await withMigrations([migration], async () => {
const result = await brain.migrate()
const r = result as any
// Main entity should have migrated successfully
expect(r.migrationsApplied).toContain('test-branch-error-prop')
expect(r.entitiesModified).toBeGreaterThanOrEqual(1)
// Branch error should appear in the combined result
expect(r.errors.length).toBeGreaterThanOrEqual(1)
const branchError = r.errors.find((e: any) => e.error.includes('branchTest must be a number'))
expect(branchError).toBeDefined()
})
await fork.close()
})
it('should stop early when maxErrors is exceeded', async () => { it('should stop early when maxErrors is exceeded', async () => {
// Add many entities that will all fail // Add many entities that will all fail
for (let i = 0; i < 5; i++) { for (let i = 0; i < 5; i++) {

229
tests/integration/phase3TypeFirstQuery.integration.test.ts
View file

@ -1,229 +0,0 @@
/**
* Phase 3 Integration Tests - Type-First Query Optimization
*
* End-to-end tests verifying Phase 3 works with the complete Brainy system
* Target: 8 tests covering real-world scenarios
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import { TypeAwareHNSWIndex } from '../../src/hnsw/typeAwareHNSWIndex.js'
describe('Phase 3: Type-First Query Optimization - Integration', () => {
let brainy: Brainy<any>
beforeEach(async () => {
// Initialize with memory storage and TypeAwareHNSWIndex
brainy = new Brainy({ requireSubtype: false,
name: 'phase3-test',
dimension: 384,
storage: {
type: 'memory' // Use memory for fast tests
},
index: {
M: 16,
efConstruction: 200,
efSearch: 50
},
debug: false // Disable debug logging for tests
})
await brainy.initialize()
})
afterEach(async () => {
// Clean up
await brainy.close()
})
// ========== Basic Type Inference Tests (3 tests) ==========
describe('Basic Type Inference', () => {
it('should automatically infer Person type from "engineer" query', async () => {
// Add test data
await brainy.add({
type: NounType.Person,
data: { name: 'Alice', role: 'engineer' }
})
await brainy.add({
type: NounType.Document,
data: { title: 'Engineering Guide' }
})
// Query with natural language
const results = await brainy.find('Find engineers')
// Should find Person, not Document
expect(results.length).toBeGreaterThan(0)
// Verify TypeAwareHNSWIndex is being used
expect((brainy as any).index).toBeInstanceOf(TypeAwareHNSWIndex)
})
it('should handle queries with explicit type override', async () => {
await brainy.add({
type: NounType.Person,
data: { name: 'Bob', role: 'developer' }
})
await brainy.add({
type: NounType.Document,
data: { title: 'Development Process' }
})
// Query with explicit type should override inference
const results = await brainy.find({
query: 'development',
type: NounType.Document // Explicit override
})
// Should only find documents
expect(results.length).toBeGreaterThan(0)
expect(results.every(r => r.entity.noun === NounType.Document)).toBe(true)
})
it('should handle multi-type queries efficiently', async () => {
// Add diverse data
await brainy.add({
type: NounType.Person,
data: { name: 'Charlie', company: 'TechCorp' }
})
await brainy.add({
type: NounType.Organization,
data: { name: 'TechCorp', industry: 'Software' }
})
await brainy.add({
type: NounType.Document,
data: { title: 'TechCorp Overview' }
})
// Query that should infer multiple types
const results = await brainy.find('people at TechCorp')
expect(results.length).toBeGreaterThan(0)
// Should find both Person and Organization
const types = results.map(r => r.entity.noun)
expect(types).toContain(NounType.Person)
})
})
// ========== Performance Tests (2 tests) ==========
describe('Performance Impact', () => {
it('should reduce query latency for type-specific queries', async () => {
// Add 100 diverse entities
for (let i = 0; i < 50; i++) {
await brainy.add({
type: NounType.Person,
data: { name: `Person ${i}`, role: 'engineer' }
})
}
for (let i = 0; i < 50; i++) {
await brainy.add({
type: NounType.Document,
data: { title: `Document ${i}` }
})
}
// Measure query with type inference
const start = Date.now()
const results = await brainy.find('Find engineers')
const elapsed = Date.now() - start
expect(results.length).toBeGreaterThan(0)
expect(elapsed).toBeLessThan(1000) // Should be fast even with 100 entities
// Verify results are correct type
const personResults = results.filter(r => r.entity.noun === NounType.Person)
expect(personResults.length).toBeGreaterThan(0)
}, 10000)
it('should handle high-volume queries without degradation', async () => {
// Add test data
for (let i = 0; i < 20; i++) {
await brainy.add({
type: NounType.Person,
data: { name: `Person ${i}` }
})
}
// Run 10 queries in sequence
const latencies: number[] = []
for (let i = 0; i < 10; i++) {
const start = Date.now()
await brainy.find('Find people')
const elapsed = Date.now() - start
latencies.push(elapsed)
}
// Verify consistent performance (no degradation)
const avgLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length
const maxLatency = Math.max(...latencies)
expect(maxLatency).toBeLessThan(avgLatency * 2) // Max should not be > 2x avg
}, 15000)
})
// ========== Edge Cases (2 tests) ==========
describe('Edge Cases', () => {
it('should handle queries with no matching types gracefully', async () => {
await brainy.add({
type: NounType.Person,
data: { name: 'Dave' }
})
// Query that won't infer any specific type
const results = await brainy.find('random stuff xyz')
// Should still work (fallback to all-types)
expect(Array.isArray(results)).toBe(true)
})
it('should handle empty query gracefully', async () => {
await brainy.add({
type: NounType.Person,
data: { name: 'Eve' }
})
// Empty query should return results
const results = await brainy.find({ limit: 5 })
expect(Array.isArray(results)).toBe(true)
expect(results.length).toBeGreaterThan(0)
})
})
// ========== Backward Compatibility (1 test) ==========
describe('Backward Compatibility', () => {
it('should work with all existing query patterns', async () => {
await brainy.add({
type: NounType.Person,
data: { name: 'Frank', age: 30 }
})
// Test various query patterns
const results1 = await brainy.find({ query: 'Frank' })
expect(results1.length).toBeGreaterThan(0)
const results2 = await brainy.find({ type: NounType.Person })
expect(results2.length).toBeGreaterThan(0)
const results3 = await brainy.find({
where: { age: 30 }
})
expect(results3.length).toBeGreaterThan(0)
const results4 = await brainy.find('Find Frank')
expect(results4.length).toBeGreaterThan(0)
})
})
})

83
tests/integration/storage-batch-operations.test.ts
View file

@ -1,9 +1,8 @@
/** /**
* Storage-Level Batch Operations Test Suite v5.12.0 * Storage-Level Batch Operations Test Suite v5.12.0
* *
* Comprehensive testing of new storage-level batch APIs: * Comprehensive testing of storage-level batch APIs:
* - storage.getNounMetadataBatch() - Batch metadata reads * - storage.getNounMetadataBatch() - Batch metadata reads
* - storage.readBatchWithInheritance() - COW-aware batch reads
* - storage.getVerbsBySourceBatch() - Batch relationship queries * - storage.getVerbsBySourceBatch() - Batch relationship queries
* - brain.batchGet() - High-level batch entity retrieval * - brain.batchGet() - High-level batch entity retrieval
* - PathResolver.getChildren() - VFS batch operations * - PathResolver.getChildren() - VFS batch operations
@ -11,10 +10,8 @@
* Coverage: * Coverage:
* ✅ Type-aware storage compatibility * ✅ Type-aware storage compatibility
* ✅ Sharding preservation * ✅ Sharding preservation
* ✅ COW (Copy-on-Write) integration * ✅ Write-cache coherence
* ✅ fork() and branch isolation
* ✅ Performance improvements (N+1 → batched) * ✅ Performance improvements (N+1 → batched)
* ✅ Cloud adapter native batch APIs
*/ */
import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { describe, it, expect, beforeEach, afterEach } from 'vitest'
@ -129,13 +126,13 @@ describe('Storage-Level Batch Operations v5.12.0', () => {
}) })
describe('storage.getNounMetadataBatch() - Storage Layer', () => { describe('storage.getNounMetadataBatch() - Storage Layer', () => {
it('should batch fetch noun metadata with type caching', async () => { it('should batch fetch noun metadata across types via ID-first paths', async () => {
// Add entities of different types // Add entities of different types
const id1 = await brain.add({ type: 'document', data: 'Doc' }) const id1 = await brain.add({ type: 'document', data: 'Doc' })
const id2 = await brain.add({ type: 'thing', data: 'Thing' }) const id2 = await brain.add({ type: 'thing', data: 'Thing' })
const id3 = await brain.add({ type: 'person', data: 'Person' }) const id3 = await brain.add({ type: 'person', data: 'Person' })
// Access storage directly // Access storage directly (8.0 layout: ID-first paths, no type lookup)
const storage = brain.storage as any const storage = brain.storage as any
const results = await storage.getNounMetadataBatch([id1, id2, id3]) const results = await storage.getNounMetadataBatch([id1, id2, id3])
@ -143,32 +140,18 @@ describe('Storage-Level Batch Operations v5.12.0', () => {
expect(results.get(id1)?.noun).toBe('document') expect(results.get(id1)?.noun).toBe('document')
expect(results.get(id2)?.noun).toBe('thing') expect(results.get(id2)?.noun).toBe('thing')
expect(results.get(id3)?.noun).toBe('person') expect(results.get(id3)?.noun).toBe('person')
// Type cache should be populated
expect(storage.nounTypeCache.has(id1)).toBe(true)
expect(storage.nounTypeCache.get(id1)).toBe('document')
}) })
it('should handle uncached IDs by trying multiple types', async () => { it('should omit missing ids from the result map', async () => {
// Add entity
const id = await brain.add({ type: 'document', data: 'Test' }) const id = await brain.add({ type: 'document', data: 'Test' })
const missing = '00000000-0000-4000-8000-000000000000'
// Clear type cache to simulate uncached scenario
const storage = brain.storage as any const storage = brain.storage as any
storage.nounTypeCache.delete(id) const results = await storage.getNounMetadataBatch([id, missing])
// Batch fetch should still work (tries all types)
const results = await storage.getNounMetadataBatch([id])
expect(results.size).toBe(1) expect(results.size).toBe(1)
expect(results.get(id)?.noun).toBe('document') expect(results.get(id)?.noun).toBe('document')
expect(results.has(missing)).toBe(false)
// Cache should be repopulated (or may still be empty if metadata doesn't populate it)
// This is acceptable as long as the data is retrieved correctly
const cachedType = storage.nounTypeCache.get(id)
if (cachedType !== undefined) {
expect(cachedType).toBe('document')
}
}) })
it('should preserve sharding in all paths', async () => { it('should preserve sharding in all paths', async () => {
@ -212,32 +195,7 @@ describe('Storage-Level Batch Operations v5.12.0', () => {
}) })
}) })
describe('COW Integration - readBatchWithInheritance()', () => { describe('Write-cache coherence', () => {
it('should resolve branch paths before reading', async () => {
// Add entity on main
const id = await brain.add({ type: 'document', data: 'Main branch' })
// Create fork
const fork = await brain.fork('test-branch')
// Add entity on fork
const forkId = await fork.add({ type: 'document', data: 'Fork branch' })
// Batch get on fork should see fork entity
const forkResults = await fork.batchGet([forkId, id])
expect(forkResults.size).toBe(2)
expect(forkResults.get(forkId)?.data).toBe('Fork branch')
expect(forkResults.get(id)?.data).toBe('Main branch') // Inherited
// Batch get on main should NOT see fork entity
const mainResults = await brain.batchGet([forkId, id])
expect(mainResults.size).toBe(1)
expect(mainResults.has(forkId)).toBe(false) // Not on main
expect(mainResults.get(id)?.data).toBe('Main branch')
})
it('should respect write cache for dirty entities', async () => { it('should respect write cache for dirty entities', async () => {
// Add entity // Add entity
const id = await brain.add({ type: 'document', data: 'Original' }) const id = await brain.add({ type: 'document', data: 'Original' })
@ -250,29 +208,6 @@ describe('Storage-Level Batch Operations v5.12.0', () => {
expect(results.get(id)?.data).toBe('Updated') expect(results.get(id)?.data).toBe('Updated')
}) })
it('should inherit from parent commits for missing entities', async () => {
// Add entities on main
const id1 = await brain.add({ type: 'document', data: 'Main 1' })
const id2 = await brain.add({ type: 'document', data: 'Main 2' })
// Commit
await brain.commit('Initial entities')
// Create fork
const fork = await brain.fork('child-branch')
// Add new entity only on fork
const forkId = await fork.add({ type: 'document', data: 'Fork only' })
// Batch get on fork should inherit main entities
const results = await fork.batchGet([id1, id2, forkId])
expect(results.size).toBe(3)
expect(results.get(id1)?.data).toBe('Main 1') // Inherited
expect(results.get(id2)?.data).toBe('Main 2') // Inherited
expect(results.get(forkId)?.data).toBe('Fork only') // Fork's own
})
}) })
describe('getVerbsBySourceBatch() - Batch Relationship Queries', () => { describe('getVerbsBySourceBatch() - Batch Relationship Queries', () => {

527
tests/integration/typeAwareHNSW.integration.test.ts
View file

@ -1,527 +0,0 @@
/**
* TypeAwareHNSW Integration Tests
*
* End-to-end tests for Phase 2 Type-Aware HNSW implementation.
* Tests cover:
* - Brainy integration (add, find)
* - Storage integration (FileSystem, Memory)
* - Rebuild functionality
* - Cache behavior
* - Large datasets
* - Performance characteristics
*
* Total: 15 integration tests
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { TypeAwareHNSWIndex } from '../../src/hnsw/typeAwareHNSWIndex.js'
import type { NounType } from '../../src/types/graphTypes.js'
import { euclideanDistance } from '../../src/utils/index.js'
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
import { v4 as uuidv4 } from 'uuid'
import fs from 'fs'
import path from 'path'
describe('TypeAwareHNSW Integration Tests', () => {
const TEST_DATA_DIR = path.join(process.cwd(), '.test-data-type-aware-hnsw')
afterEach(() => {
// Clean up test data directory
if (fs.existsSync(TEST_DATA_DIR)) {
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true })
}
})
// ===================================================================
// 1. STORAGE INTEGRATION
// ===================================================================
describe('Storage Integration', () => {
it('should work with MemoryStorage', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Add entities with valid UUIDs
const personId = uuidv4()
const docId = uuidv4()
await index.addItem(
{ id: personId, vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: docId, vector: [0, 1, 0] },
'document' as NounType
)
// Search
const results = await index.search([1, 0, 0], 2)
expect(results).toHaveLength(2)
expect(results[0][0]).toBe(personId) // Closest to [1,0,0]
})
it('should work with FileSystemStorage', async () => {
const storage = new FileSystemStorage(TEST_DATA_DIR)
await storage.init()
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Add entities with valid UUIDs
const personId = uuidv4()
const docId = uuidv4()
await index.addItem(
{ id: personId, vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: docId, vector: [0, 1, 0] },
'document' as NounType
)
// Search should work
const results = await index.search([1, 0, 0], 2)
expect(results).toHaveLength(2)
})
})
// ===================================================================
// 2. REBUILD FUNCTIONALITY
// ===================================================================
describe('Rebuild Functionality', () => {
it('should handle rebuild gracefully when no data exists', async () => {
const storage = new MemoryStorage()
await storage.init()
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Rebuild with no data - should not crash
await index.rebuild()
expect(index.size()).toBe(0)
})
it('should skip rebuild when no storage adapter', async () => {
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage: null }
)
// Should not crash when storage is null
await index.rebuild()
expect(index.size()).toBe(0)
})
it('should allow specifying types to rebuild', async () => {
const storage = new MemoryStorage()
await storage.init()
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Rebuild specific types (no data, but should not crash)
await index.rebuild({ types: ['person', 'document'] as NounType[] })
expect(index.size()).toBe(0)
})
})
// ===================================================================
// 3. LARGE DATASET TESTS
// ===================================================================
describe('Large Dataset Tests', () => {
it('should handle 1000 entities across 5 types', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 8, efConstruction: 100, efSearch: 50 },
euclideanDistance,
{ storage }
)
const types: NounType[] = [
'person',
'document',
'event',
'organization',
'location'
]
// Add 200 entities per type = 1000 total
for (const type of types) {
for (let i = 0; i < 200; i++) {
await index.addItem(
{
id: `${type}-${i}`,
vector: [
Math.random(),
Math.random(),
Math.random(),
Math.random()
]
},
type
)
}
}
expect(index.size()).toBe(1000)
expect(index.getActiveTypes()).toHaveLength(5)
// Verify each type has correct count
for (const type of types) {
expect(index.sizeForType(type)).toBe(200)
}
// Verify search works
const results = await index.search([0.5, 0.5, 0.5, 0.5], 10)
expect(results).toHaveLength(10)
}, 30000) // 30s timeout for large dataset
it('should handle unbalanced distribution (1 dominant type)', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 8, efConstruction: 100, efSearch: 50 },
euclideanDistance,
{ storage }
)
// Add 900 person entities (dominant type)
for (let i = 0; i < 900; i++) {
await index.addItem(
{
id: `person-${i}`,
vector: [Math.random(), Math.random(), Math.random()]
},
'person' as NounType
)
}
// Add 100 document entities
for (let i = 0; i < 100; i++) {
await index.addItem(
{
id: `doc-${i}`,
vector: [Math.random(), Math.random(), Math.random()]
},
'document' as NounType
)
}
expect(index.size()).toBe(1000)
expect(index.sizeForType('person' as NounType)).toBe(900)
expect(index.sizeForType('document' as NounType)).toBe(100)
// Verify search works correctly for both types
const personResults = await index.search(
[0.5, 0.5, 0.5],
10,
'person' as NounType
)
const docResults = await index.search(
[0.5, 0.5, 0.5],
10,
'document' as NounType
)
expect(personResults.length).toBeGreaterThanOrEqual(10)
expect(docResults.length).toBeGreaterThanOrEqual(10)
// All person results should be from person type
personResults.forEach((result) => {
expect(result[0]).toMatch(/^person-/)
})
}, 30000)
})
// ===================================================================
// 4. TYPE-SPECIFIC QUERIES
// ===================================================================
describe('Type-Specific Queries', () => {
let index: TypeAwareHNSWIndex
beforeEach(async () => {
const storage = new MemoryStorage()
index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Add entities of different types
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'person-2', vector: [0.9, 0.1, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
await index.addItem(
{ id: 'doc-2', vector: [0, 0.9, 0.1] },
'document' as NounType
)
await index.addItem(
{ id: 'event-1', vector: [0, 0, 1] },
'event' as NounType
)
})
it('should search single type only (fast path)', async () => {
const results = await index.search([1, 0, 0], 2, 'person' as NounType)
expect(results).toHaveLength(2)
expect(results[0][0]).toBe('person-1')
expect(results[1][0]).toBe('person-2')
// Verify no document or event results
results.forEach((result) => {
expect(result[0]).toMatch(/^person-/)
})
})
it('should search multiple types', async () => {
const results = await index.search(
[0.5, 0.5, 0],
5,
['person', 'document'] as NounType[]
)
expect(results).toHaveLength(4) // 2 person + 2 document
const ids = results.map((r) => r[0])
expect(ids).toContain('person-1')
expect(ids).toContain('person-2')
expect(ids).toContain('doc-1')
expect(ids).toContain('doc-2')
expect(ids).not.toContain('event-1') // Not searched
})
it('should fall back to all-types search when type unknown', async () => {
const results = await index.search([0, 0, 1], 5)
expect(results).toHaveLength(5)
const ids = results.map((r) => r[0])
expect(ids).toContain('event-1') // Found in all-types search
})
})
// ===================================================================
// 5. MEMORY ISOLATION
// ===================================================================
describe('Memory Isolation', () => {
it('should maintain separate memory for each type', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Add entities of different types
for (let i = 0; i < 100; i++) {
await index.addItem(
{ id: `person-${i}`, vector: [Math.random(), Math.random(), 0] },
'person' as NounType
)
}
for (let i = 0; i < 100; i++) {
await index.addItem(
{ id: `doc-${i}`, vector: [0, Math.random(), Math.random()] },
'document' as NounType
)
}
// Get stats to verify memory isolation
const stats = index.getStats()
expect(stats.typeCount).toBe(2)
expect(stats.typeStats.has('person' as NounType)).toBe(true)
expect(stats.typeStats.has('document' as NounType)).toBe(true)
const personStats = stats.typeStats.get('person' as NounType)!
const docStats = stats.typeStats.get('document' as NounType)!
expect(personStats.nodeCount).toBe(100)
expect(docStats.nodeCount).toBe(100)
// Verify memory is tracked separately (may be 0 in MemoryStorage)
expect(personStats.memoryMB).toBeGreaterThanOrEqual(0)
expect(docStats.memoryMB).toBeGreaterThanOrEqual(0)
// Clear one type and verify other is unaffected
index.clearType('person' as NounType)
expect(index.sizeForType('person' as NounType)).toBe(0)
expect(index.sizeForType('document' as NounType)).toBe(100)
})
})
// ===================================================================
// 6. CACHE BEHAVIOR
// ===================================================================
describe('Cache Behavior', () => {
it('should use UnifiedCache across all type indexes', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
// Add entities to different types
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
// Get cache stats for each type
const personStats = index.getStatsForType('person' as NounType)
const docStats = index.getStatsForType('document' as NounType)
expect(personStats).not.toBeNull()
expect(docStats).not.toBeNull()
// Verify cache stats are available
expect(personStats!.cacheStats).toBeDefined()
expect(docStats!.cacheStats).toBeDefined()
// UnifiedCache is shared, so both should have cache stats
expect(personStats!.cacheStats.hnswCache).toBeDefined()
expect(docStats!.cacheStats.hnswCache).toBeDefined()
})
})
// ===================================================================
// 7. PERFORMANCE CHARACTERISTICS
// ===================================================================
describe('Performance Characteristics', () => {
it('should have faster single-type search than all-types search', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 8, efConstruction: 100, efSearch: 50 },
euclideanDistance,
{ storage }
)
// Add 500 entities across 5 types
const types: NounType[] = [
'person',
'document',
'event',
'organization',
'location'
]
for (const type of types) {
for (let i = 0; i < 100; i++) {
await index.addItem(
{
id: `${type}-${i}`,
vector: [Math.random(), Math.random(), Math.random()]
},
type
)
}
}
// Measure single-type search time
const singleTypeStart = Date.now()
await index.search([0.5, 0.5, 0.5], 10, 'person' as NounType)
const singleTypeTime = Date.now() - singleTypeStart
// Measure all-types search time
const allTypesStart = Date.now()
await index.search([0.5, 0.5, 0.5], 10)
const allTypesTime = Date.now() - allTypesStart
// Single-type should be faster (but this is a loose check for small dataset)
// At billion scale, this difference would be 10x
expect(singleTypeTime).toBeLessThanOrEqual(allTypesTime * 2)
}, 15000)
it('should demonstrate memory reduction', async () => {
const storage = new MemoryStorage()
const index = new TypeAwareHNSWIndex(
{ M: 8, efConstruction: 100, efSearch: 50 },
euclideanDistance,
{ storage }
)
// Add 1000 entities across 10 types
const types: NounType[] = [
'person',
'document',
'event',
'organization',
'location',
'product',
'concept',
'project',
'task',
'message'
]
for (const type of types) {
for (let i = 0; i < 100; i++) {
await index.addItem(
{
id: `${type}-${i}`,
vector: [Math.random(), Math.random(), Math.random(), Math.random()]
},
type
)
}
}
const stats = index.getStats()
expect(stats.totalNodes).toBe(1000)
expect(stats.typeCount).toBe(10)
// Verify memory reduction is calculated
expect(stats.estimatedMonolithicMemoryMB).toBeGreaterThan(0)
expect(stats.totalMemoryMB).toBeGreaterThanOrEqual(0)
expect(stats.memoryReductionPercent).toBeGreaterThanOrEqual(0)
// At this scale, reduction should be significant
expect(stats.totalMemoryMB).toBeLessThan(
stats.estimatedMonolithicMemoryMB
)
}, 30000)
})
})

511
tests/integration/versioning.test.ts
View file

@ -1,511 +0,0 @@
/**
* Entity Versioning Integration Tests (v5.3.0, v6.3.0)
*
* Tests the complete versioning workflow:
* - Save versions
* - List versions
* - Restore versions
* - Compare versions
* - Prune versions
* - Branch isolation
* - Index pollution prevention
*
* v6.3.0: Pure key-value storage, no index pollution, restore() updates all indexes
*/
import { describe, it, expect, beforeEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import type { EntityVersion } from '../../src/versioning/VersionManager.js'
import { randomUUID } from 'crypto'
// Helper to generate valid UUIDs for tests
const uuid = () => randomUUID().replace(/-/g, '')
describe('Entity Versioning (v5.3.0)', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy({ requireSubtype: false,
storage: { type: 'memory' },
silent: true
})
await brain.init()
})
describe('Core Versioning API', () => {
it('should save and retrieve versions', async () => {
const entityId = uuid()
// Add initial entity
await brain.add({
data: 'Alice',
id: entityId,
type: 'person', // v6.3.0: Use valid NounType
metadata: {
name: 'Alice',
email: 'alice@example.com'
}
})
// Save version 1
const v1 = await brain.versions.save(entityId, {
tag: 'v1.0',
description: 'Initial version'
})
expect(v1.version).toBe(1)
expect(v1.entityId).toBe(entityId)
expect(v1.tag).toBe('v1.0')
expect(v1.contentHash).toBeDefined()
// Update entity
await brain.update({ id: entityId, metadata: { name: 'Alice Smith' } })
// Save version 2
const v2 = await brain.versions.save(entityId, {
tag: 'v2.0',
description: 'Updated name'
})
expect(v2.version).toBe(2)
expect(v2.entityId).toBe(entityId)
// List versions
const versions = await brain.versions.list(entityId)
expect(versions).toHaveLength(2)
expect(versions[0].version).toBe(2) // Newest first
expect(versions[1].version).toBe(1)
})
it('should deduplicate identical content', async () => {
const entityId = uuid()
await brain.add({
data: 'Doc',
id: entityId,
type: 'document',
metadata: {
name: 'Doc',
content: 'Hello'
}
})
// Save version 1
const v1 = await brain.versions.save(entityId, { tag: 'v1' })
// Save again without changes
const v2 = await brain.versions.save(entityId, { tag: 'v2' })
// Should return existing version (same content hash)
expect(v2.version).toBe(v1.version)
expect(v2.contentHash).toBe(v1.contentHash)
// Only one version should exist
const versions = await brain.versions.list(entityId)
expect(versions).toHaveLength(1)
})
it('should restore to previous version', async () => {
const entityId = uuid()
await brain.add({
data: 'Config',
id: entityId,
type: 'thing',
metadata: {
name: 'Config',
settings: { theme: 'light' }
}
})
// Save v1
await brain.versions.save(entityId, { tag: 'v1' })
// Update
await brain.update({ id: entityId, metadata: { settings: { theme: 'dark' } } })
// Save v2
await brain.versions.save(entityId, { tag: 'v2' })
// Verify current state (getNounMetadata returns flat NounMetadata)
let current = await brain.getNounMetadata(entityId)
expect(current?.settings?.theme).toBe('dark')
// Restore to v1
await brain.versions.restore(entityId, 1)
// Verify restored state
current = await brain.getNounMetadata(entityId)
expect(current?.settings?.theme).toBe('light')
})
it('should compare versions', async () => {
const entityId = uuid()
await brain.add({
data: 'Bob',
id: entityId,
type: 'person', // v6.3.0: Use valid NounType
metadata: {
name: 'Bob',
email: 'bob@example.com',
age: 30
}
})
await brain.versions.save(entityId, { tag: 'v1' })
// Update
await brain.update({
id: entityId,
metadata: {
name: 'Robert',
email: 'robert@example.com',
city: 'NYC'
}
})
await brain.versions.save(entityId, { tag: 'v2' })
// Compare versions
const diff = await brain.versions.compare(entityId, 1, 2)
expect(diff.totalChanges).toBeGreaterThan(0)
expect(diff.modified.length).toBeGreaterThan(0)
expect(diff.added.length).toBeGreaterThan(0)
// Check specific changes
const nameChange = diff.modified.find(c => c.path.includes('name'))
expect(nameChange).toBeDefined()
expect(nameChange?.oldValue).toBe('Bob')
expect(nameChange?.newValue).toBe('Robert')
const cityAdd = diff.added.find(c => c.path.includes('city'))
expect(cityAdd).toBeDefined()
expect(cityAdd?.newValue).toBe('NYC')
})
it('should get version content without restoring', async () => {
const entityId = uuid()
await brain.add({
data: 'Note',
id: entityId,
type: 'document',
metadata: {
name: 'Note',
content: 'Version 1'
}
})
await brain.versions.save(entityId, { tag: 'v1' })
await brain.update({ id: entityId, metadata: { content: 'Version 2' } })
await brain.versions.save(entityId, { tag: 'v2' })
// Get v1 content without restoring (returns flat NounMetadata)
const v1Content = await brain.versions.getContent(entityId, 1)
expect(v1Content.content).toBe('Version 1')
// Current should still be v2 (getNounMetadata returns flat NounMetadata)
const current = await brain.getNounMetadata(entityId)
expect(current?.content).toBe('Version 2')
})
it('should prune old versions', async () => {
const entityId = uuid()
await brain.add({
data: 'Log',
id: entityId,
type: 'document',
metadata: {
name: 'Log'
}
})
// Create 10 versions
for (let i = 1; i <= 10; i++) {
await brain.update({ id: entityId, metadata: { content: `Entry ${i}` } })
await brain.versions.save(entityId, { tag: `v${i}` })
}
// Verify all 10 exist
let versions = await brain.versions.list(entityId)
expect(versions.length).toBeGreaterThanOrEqual(10)
// Prune to keep only 5 most recent
const result = await brain.versions.prune(entityId, {
keepRecent: 5,
keepTagged: false
})
expect(result.deleted).toBeGreaterThan(0)
expect(result.kept).toBe(5)
// Verify only 5 remain
versions = await brain.versions.list(entityId)
expect(versions).toHaveLength(5)
})
it('should support version tags', async () => {
const entityId = uuid()
await brain.add({
data: 'App',
id: entityId,
type: 'thing',
metadata: {
name: 'App'
}
})
await brain.versions.save(entityId, { tag: 'alpha' })
await brain.update({ id: entityId, metadata: { version: '0.2' } })
await brain.versions.save(entityId, { tag: 'beta' })
await brain.update({ id: entityId, metadata: { version: '1.0' } })
await brain.versions.save(entityId, { tag: 'release' })
// Get by tag
const beta = await brain.versions.getVersionByTag(entityId, 'beta')
expect(beta).toBeDefined()
expect(beta?.tag).toBe('beta')
// Restore by tag
await brain.versions.restore(entityId, 'beta')
const current = await brain.getNounMetadata(entityId)
expect(current?.version).toBe('0.2')
})
it('should support undo/revert', async () => {
const entityId = uuid()
await brain.add({
data: 'Data',
id: entityId,
type: 'thing',
metadata: {
name: 'Data',
value: 100
}
})
// Save v1 (value=100)
await brain.versions.save(entityId, { tag: 'v1' })
// Update and save v2 (value=200)
await brain.update({ id: entityId, metadata: { value: 200 } })
await brain.versions.save(entityId, { tag: 'v2' })
// undo() restores to SECOND-MOST-RECENT version and creates a snapshot
// Note: undo() internally calls restore() with createSnapshot: true
const undone = await brain.versions.undo(entityId)
expect(undone).not.toBeNull()
// After undo, entity should have v1's value
const currentVal = await brain.getNounMetadata(entityId)
expect(currentVal?.value).toBe(100) // v1's value
// Now we have: [before-undo, v2, v1] - undo created a snapshot
const versionsAfterUndo = await brain.versions.list(entityId)
expect(versionsAfterUndo.length).toBeGreaterThanOrEqual(2)
// Update and save v3 (value=300)
await brain.update({ id: entityId, metadata: { value: 300 } })
await brain.versions.save(entityId, { tag: 'v3' })
// revert() is alias for undo()
const reverted = await brain.versions.revert(entityId)
expect(reverted).not.toBeNull()
// The entity should be restored to second-most-recent version
const revertedVal = await brain.getNounMetadata(entityId)
// After revert from v3, we go to the previous version
expect(revertedVal?.value).toBeDefined()
})
})
describe('Branch Isolation', () => {
it('should isolate versions by branch', async () => {
const entityId = uuid()
await brain.add({
data: 'Test',
id: entityId,
type: 'thing',
metadata: {
name: 'Test'
}
})
// Save versions on main
await brain.versions.save(entityId, { tag: 'main-v1' })
await brain.update({ id: entityId, metadata: { name: 'Main Update' } })
await brain.versions.save(entityId, { tag: 'main-v2' })
// Main should have versions
const mainVersionsBefore = await brain.versions.list(entityId)
expect(mainVersionsBefore.length).toBeGreaterThanOrEqual(2)
// Main versions should have branch='main'
expect(mainVersionsBefore.every(v => v.branch === 'main')).toBe(true)
// Switch to feature branch
// Note: fork() creates the branch and returns a NEW instance
// We need to checkout() to switch the current instance to the new branch
await brain.fork('feature')
await brain.checkout('feature')
// Save version on feature with unique tag
await brain.update({ id: entityId, metadata: { name: 'Feature Update' } })
await brain.versions.save(entityId, { tag: 'feature-only-v1' })
// Feature versions list - may include inherited versions from COW
const featureVersions = await brain.versions.list(entityId)
expect(featureVersions.length).toBeGreaterThan(0)
// Feature should have our unique tag (this is the NEW version on feature)
const featureOnlyVersion = featureVersions.find(v => v.tag === 'feature-only-v1')
expect(featureOnlyVersion).toBeDefined()
// The version we just saved on feature should have branch='feature'
expect(featureOnlyVersion!.branch).toBe('feature')
// Switch back to main
await brain.checkout('main')
// Main versions should not have feature's unique tag
const mainVersionsAfter = await brain.versions.list(entityId)
const featureTagOnMain = mainVersionsAfter.find(v => v.tag === 'feature-only-v1')
expect(featureTagOnMain).toBeUndefined() // Feature-only version not on main
// Main should still have its original versions
expect(mainVersionsAfter.some(v => v.tag === 'main-v1')).toBe(true)
expect(mainVersionsAfter.some(v => v.tag === 'main-v2')).toBe(true)
})
})
describe('Edge Cases', () => {
it('should handle non-existent entities gracefully', async () => {
const nonExistentId = uuid()
await expect(
brain.versions.save(nonExistentId, { tag: 'v1' })
).rejects.toThrow(`Entity ${nonExistentId} not found`)
})
it('should handle empty version history', async () => {
const entityId = uuid()
await brain.add({ data: 'New', id: entityId, type: 'thing', metadata: { name: 'New' } })
expect(await brain.versions.count(entityId)).toBe(0)
expect(await brain.versions.hasVersions(entityId)).toBe(false)
expect(await brain.versions.getLatest(entityId)).toBeNull()
})
it('should handle version not found', async () => {
const entityId = uuid()
await brain.add({ data: 'Test', id: entityId, type: 'thing', metadata: { name: 'Test' } })
await expect(
brain.versions.restore(entityId, 999)
).rejects.toThrow('Version 999 not found')
})
})
describe('Index Pollution Prevention (v6.3.0)', () => {
it('should NOT pollute find() results with versions', async () => {
const entityId = uuid()
// Add entity
await brain.add({
data: 'Test entity',
id: entityId,
type: 'document',
metadata: { name: 'Test', category: 'pollution-test' }
})
// Save multiple versions
await brain.versions.save(entityId, { tag: 'v1' })
await brain.update({ id: entityId, metadata: { name: 'Updated' } })
await brain.versions.save(entityId, { tag: 'v2' })
await brain.update({ id: entityId, metadata: { name: 'Updated Again' } })
await brain.versions.save(entityId, { tag: 'v3' })
// Verify 3 versions exist
const versions = await brain.versions.list(entityId)
expect(versions).toHaveLength(3)
// find() should return ONLY the real entity, not version entries
const results = await brain.find({ where: { category: 'pollution-test' } })
expect(results).toHaveLength(1)
expect(results[0].id).toBe(entityId)
// Verify no version entities pollute the index
// (This was the bug - _isVersion entities appeared in find())
const allDocs = await brain.find({ where: { type: 'document' }, limit: 100 })
const versionEntities = allDocs.filter((e: any) => e.metadata?._isVersion)
expect(versionEntities).toHaveLength(0)
})
it('should keep current entity fully indexed after versioning', async () => {
const entityId = uuid()
// Add entity
await brain.add({
data: 'Searchable content about machine learning',
id: entityId,
type: 'document',
metadata: { name: 'ML Doc', topic: 'ai' }
})
// Save versions
await brain.versions.save(entityId, { tag: 'v1' })
await brain.update({ id: entityId, metadata: { name: 'Updated ML Doc' } })
await brain.versions.save(entityId, { tag: 'v2' })
// Entity should still be searchable by metadata
const byMetadata = await brain.find({ where: { topic: 'ai' } })
expect(byMetadata).toHaveLength(1)
expect(byMetadata[0].id).toBe(entityId)
// Entity should still be searchable by vector similarity
const bySimilarity = await brain.find({ query: 'machine learning', limit: 5 })
const found = bySimilarity.find((e: any) => e.id === entityId)
expect(found).toBeDefined()
})
it('should update indexes when restoring a version', async () => {
const entityId = uuid()
// Add entity with initial state
await brain.add({
data: 'Initial content',
id: entityId,
type: 'document',
metadata: { name: 'Doc', status: 'draft' }
})
await brain.versions.save(entityId, { tag: 'draft' })
// Update to published state
await brain.update({ id: entityId, metadata: { status: 'published' } })
await brain.versions.save(entityId, { tag: 'published' })
// Verify current state
let published = await brain.find({ where: { status: 'published' } })
expect(published).toHaveLength(1)
let drafts = await brain.find({ where: { status: 'draft' } })
expect(drafts).toHaveLength(0)
// Restore to draft version
await brain.versions.restore(entityId, 'draft')
// Indexes should update - now entity is draft again
published = await brain.find({ where: { status: 'published' } })
expect(published).toHaveLength(0)
drafts = await brain.find({ where: { status: 'draft' } })
expect(drafts).toHaveLength(1)
expect(drafts[0].id).toBe(entityId)
})
})
})

431
tests/integration/vfs-historical-reads.test.ts
View file

@ -1,431 +0,0 @@
/**
* VFS Historical Reads Integration Test (v5.3.7)
*
* Tests commitId support for time-travel reads:
* - readFile(path, { commitId })
* - readdir(path, { commitId })
* - stat(path, { commitId })
* - exists(path, { commitId })
*
* This is the CRITICAL test for a consumer's time-travel feature.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
import * as path from 'path'
describe('VFS Historical Reads (v5.3.7)', () => {
const testDir = path.join(process.cwd(), 'test-vfs-historical')
let brain: Brainy
beforeAll(async () => {
// Clean up test directory
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
fs.mkdirSync(testDir, { recursive: true })
// Initialize Brainy with filesystem storage (required for COW)
brain = new Brainy({ requireSubtype: false,
storage: {
adapter: 'filesystem',
path: testDir
}
})
await brain.init()
// Initialize VFS
await brain.vfs.init()
})
afterAll(async () => {
// Clean up test directory
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
})
it('should read files from historical commits', async () => {
console.log('\n📁 Test 1: Historical file reading')
// Commit 1: Empty workspace
await brain.commit({
message: 'Initial empty state',
author: 'test@example.com'
})
const history1 = await brain.getHistory({ limit: 1 })
const emptyCommitId = history1[0].hash
console.log(` Empty commit: ${emptyCommitId.slice(0, 8)}`)
// Create first file
await brain.vfs.writeFile('/test.md', 'Version 1')
// Commit 2: First file added
await brain.commit({
message: 'Added test.md',
author: 'test@example.com'
})
const history2 = await brain.getHistory({ limit: 1 })
const v1CommitId = history2[0].hash
console.log(` V1 commit: ${v1CommitId.slice(0, 8)}`)
// Update file
await brain.vfs.writeFile('/test.md', 'Version 2')
// Commit 3: File updated
await brain.commit({
message: 'Updated test.md',
author: 'test@example.com'
})
const history3 = await brain.getHistory({ limit: 1 })
const v2CommitId = history3[0].hash
console.log(` V2 commit: ${v2CommitId.slice(0, 8)}`)
// Test 1a: Read current file (should be V2)
const currentContent = await brain.vfs.readFile('/test.md')
expect(currentContent.toString()).toBe('Version 2')
console.log(` ✅ Current content: "${currentContent.toString()}"`)
// Test 1b: Read file at V1 commit (should be V1)
const v1Content = await brain.vfs.readFile('/test.md', { commitId: v1CommitId })
expect(v1Content.toString()).toBe('Version 1')
console.log(` ✅ V1 content: "${v1Content.toString()}"`)
// Test 1c: Read file at empty commit (should fail - file didn't exist)
await expect(
brain.vfs.readFile('/test.md', { commitId: emptyCommitId })
).rejects.toThrow('File not found at commit')
console.log(` ✅ Empty commit correctly throws ENOENT`)
})
it('should list directory contents from historical commits', async () => {
console.log('\n📂 Test 2: Historical directory listing')
// Get current commit (from previous test, has /test.md)
const history1 = await brain.getHistory({ limit: 1 })
const beforeProjectCommit = history1[0].hash
// Create project directory with files
await brain.vfs.mkdir('/project')
await brain.vfs.writeFile('/project/app.ts', 'console.log("v1")')
await brain.vfs.writeFile('/project/utils.ts', 'export const util = 1')
// Commit: Project created
await brain.commit({
message: 'Added project directory',
author: 'test@example.com'
})
const history2 = await brain.getHistory({ limit: 1 })
const projectCommit = history2[0].hash
console.log(` Project commit: ${projectCommit.slice(0, 8)}`)
// Add more files
await brain.vfs.writeFile('/project/config.json', '{"version": 1}')
// Commit: Config added
await brain.commit({
message: 'Added config',
author: 'test@example.com'
})
const history3 = await brain.getHistory({ limit: 1 })
const configCommit = history3[0].hash
console.log(` Config commit: ${configCommit.slice(0, 8)}`)
// Test 2a: Read current directory (should have 3 files)
const currentEntries = await brain.vfs.readdir('/project')
expect(currentEntries.length).toBe(3)
expect(currentEntries).toContain('app.ts')
expect(currentEntries).toContain('utils.ts')
expect(currentEntries).toContain('config.json')
console.log(` ✅ Current directory: ${currentEntries.length} files`)
// Test 2b: Read directory at project commit (should have 2 files, no config)
const projectEntries = await brain.vfs.readdir('/project', { commitId: projectCommit })
expect(projectEntries.length).toBe(2)
expect(projectEntries).toContain('app.ts')
expect(projectEntries).toContain('utils.ts')
expect(projectEntries).not.toContain('config.json')
console.log(` ✅ Project commit directory: ${projectEntries.length} files (no config)`)
// Test 2c: Read directory at before-project commit (should fail - dir didn't exist)
await expect(
brain.vfs.readdir('/project', { commitId: beforeProjectCommit })
).rejects.toThrow('Directory not found at commit')
console.log(` ✅ Before project commit correctly throws ENOENT`)
// Test 2d: Read root directory at different commits
const currentRoot = await brain.vfs.readdir('/')
const projectRoot = await brain.vfs.readdir('/', { commitId: projectCommit })
// Current root should have both /test.md and /project
expect(currentRoot.length).toBeGreaterThanOrEqual(2)
// Project commit root should also have both
expect(projectRoot.length).toBeGreaterThanOrEqual(2)
console.log(` ✅ Root directory listings work at different commits`)
})
it('should stat files from historical commits', async () => {
console.log('\n📊 Test 3: Historical file stats')
// Create a file and get its initial stats
await brain.vfs.writeFile('/stats-test.txt', 'Initial content')
// Commit: Initial file
await brain.commit({
message: 'Added stats-test.txt',
author: 'test@example.com'
})
const history1 = await brain.getHistory({ limit: 1 })
const initialCommit = history1[0].hash
console.log(` Initial commit: ${initialCommit.slice(0, 8)}`)
// Get initial stats
const initialStats = await brain.vfs.stat('/stats-test.txt', { commitId: initialCommit })
expect(initialStats.size).toBe('Initial content'.length)
expect(initialStats.isFile()).toBe(true)
console.log(` ✅ Initial stats: ${initialStats.size} bytes`)
// Wait a moment to ensure different timestamp
await new Promise(resolve => setTimeout(resolve, 100))
// Update file with different content
await brain.vfs.writeFile('/stats-test.txt', 'Much longer updated content here')
// Commit: Updated file
await brain.commit({
message: 'Updated stats-test.txt',
author: 'test@example.com'
})
const history2 = await brain.getHistory({ limit: 1 })
const updatedCommit = history2[0].hash
console.log(` Updated commit: ${updatedCommit.slice(0, 8)}`)
// Test 3a: Current stats (should reflect new size)
const currentStats = await brain.vfs.stat('/stats-test.txt')
expect(currentStats.size).toBe('Much longer updated content here'.length)
console.log(` ✅ Current stats: ${currentStats.size} bytes`)
// Test 3b: Historical stats (should reflect old size)
const historicalStats = await brain.vfs.stat('/stats-test.txt', { commitId: initialCommit })
expect(historicalStats.size).toBe('Initial content'.length)
expect(historicalStats.size).not.toBe(currentStats.size)
console.log(` ✅ Historical stats: ${historicalStats.size} bytes (different from current)`)
// Test 3c: Stat directory at historical commit
const dirStats = await brain.vfs.stat('/', { commitId: initialCommit })
expect(dirStats.isDirectory()).toBe(true)
expect(dirStats.isFile()).toBe(false)
console.log(` ✅ Directory stats work with commitId`)
})
it('should check existence from historical commits', async () => {
console.log('\n✅ Test 4: Historical existence checks')
// Get current commit
const history1 = await brain.getHistory({ limit: 1 })
const beforeNewFileCommit = history1[0].hash
// Create a new file
await brain.vfs.writeFile('/new-file.txt', 'New file content')
// Commit: New file added
await brain.commit({
message: 'Added new-file.txt',
author: 'test@example.com'
})
const history2 = await brain.getHistory({ limit: 1 })
const afterNewFileCommit = history2[0].hash
console.log(` Before new file: ${beforeNewFileCommit.slice(0, 8)}`)
console.log(` After new file: ${afterNewFileCommit.slice(0, 8)}`)
// Test 4a: File exists in current state
const existsNow = await brain.vfs.exists('/new-file.txt')
expect(existsNow).toBe(true)
console.log(` ✅ File exists in current state`)
// Test 4b: File exists at after-commit
const existsAfter = await brain.vfs.exists('/new-file.txt', { commitId: afterNewFileCommit })
expect(existsAfter).toBe(true)
console.log(` ✅ File exists at after-commit`)
// Test 4c: File does NOT exist at before-commit
const existsBefore = await brain.vfs.exists('/new-file.txt', { commitId: beforeNewFileCommit })
expect(existsBefore).toBe(false)
console.log(` ✅ File correctly doesn't exist at before-commit`)
// Test 4d: Non-existent file returns false at any commit
const neverExists = await brain.vfs.exists('/never-created.txt', { commitId: afterNewFileCommit })
expect(neverExists).toBe(false)
console.log(` ✅ Non-existent file returns false at any commit`)
// Test 4e: Directory existence checks
const rootExists = await brain.vfs.exists('/', { commitId: beforeNewFileCommit })
expect(rootExists).toBe(true)
console.log(` ✅ Root directory exists at all commits`)
})
it('should handle readdir with withFileTypes at historical commits', async () => {
console.log('\n📋 Test 5: Historical readdir with withFileTypes')
// Create mixed content
await brain.vfs.mkdir('/mixed')
await brain.vfs.writeFile('/mixed/file1.txt', 'File 1')
await brain.vfs.mkdir('/mixed/subdir')
await brain.vfs.writeFile('/mixed/file2.txt', 'File 2')
// Commit: Mixed content
await brain.commit({
message: 'Added mixed directory',
author: 'test@example.com'
})
const history = await brain.getHistory({ limit: 1 })
const mixedCommit = history[0].hash
console.log(` Mixed commit: ${mixedCommit.slice(0, 8)}`)
// Test 5a: readdir with withFileTypes at historical commit
const entries = await brain.vfs.readdir('/mixed', {
withFileTypes: true,
commitId: mixedCommit
})
expect(entries.length).toBe(3)
// Check that we got VFSDirent objects
const fileEntries = entries.filter((e: any) => e.type === 'file')
const dirEntries = entries.filter((e: any) => e.type === 'directory')
expect(fileEntries.length).toBe(2)
expect(dirEntries.length).toBe(1)
console.log(` ✅ withFileTypes: ${fileEntries.length} files, ${dirEntries.length} dirs`)
// Verify entries have correct properties
const firstEntry = entries[0] as any
expect(firstEntry).toHaveProperty('name')
expect(firstEntry).toHaveProperty('path')
expect(firstEntry).toHaveProperty('type')
expect(firstEntry).toHaveProperty('entityId')
console.log(` ✅ VFSDirent objects have all required properties`)
})
it('should handle errors gracefully for invalid commits', async () => {
console.log('\n⚠️ Test 6: Error handling for invalid commits')
const fakeCommitId = 'f'.repeat(64) // Invalid commit hash
// Test 6a: readFile with invalid commit
await expect(
brain.vfs.readFile('/test.md', { commitId: fakeCommitId })
).rejects.toThrow('Invalid commit ID')
console.log(` ✅ readFile throws on invalid commit`)
// Test 6b: readdir with invalid commit
await expect(
brain.vfs.readdir('/', { commitId: fakeCommitId })
).rejects.toThrow('Invalid commit ID')
console.log(` ✅ readdir throws on invalid commit`)
// Test 6c: stat with invalid commit
await expect(
brain.vfs.stat('/test.md', { commitId: fakeCommitId })
).rejects.toThrow('Invalid commit ID')
console.log(` ✅ stat throws on invalid commit`)
// Test 6d: exists with invalid commit (returns false, doesn't throw)
const exists = await brain.vfs.exists('/test.md', { commitId: fakeCommitId })
expect(exists).toBe(false)
console.log(` ✅ exists returns false on invalid commit (doesn't throw)`)
})
it('should maintain performance at scale', async () => {
console.log('\n⚡ Test 7: Performance with multiple files')
// Create multiple files
const fileCount = 50 // Reasonable for integration test
for (let i = 0; i < fileCount; i++) {
await brain.vfs.writeFile(`/perf-test-${i}.txt`, `File ${i} content`)
}
// Commit: Many files
await brain.commit({
message: 'Added 50 files for performance test',
author: 'test@example.com'
})
const history = await brain.getHistory({ limit: 1 })
const perfCommit = history[0].hash
console.log(` Performance commit: ${perfCommit.slice(0, 8)}`)
// Test 7a: Historical readdir performance
const startReaddir = Date.now()
const entries = await brain.vfs.readdir('/', { commitId: perfCommit })
const readdirTime = Date.now() - startReaddir
expect(entries.length).toBeGreaterThanOrEqual(fileCount)
console.log(` ✅ readdir (${entries.length} entries): ${readdirTime}ms`)
expect(readdirTime).toBeLessThan(5000) // Should complete in < 5s
// Test 7b: Historical readFile performance
const startReadFile = Date.now()
const content = await brain.vfs.readFile('/perf-test-25.txt', { commitId: perfCommit })
const readFileTime = Date.now() - startReadFile
expect(content.toString()).toBe('File 25 content')
console.log(` ✅ readFile: ${readFileTime}ms`)
expect(readFileTime).toBeLessThan(1000) // Should complete in < 1s
// Test 7c: Historical exists performance
const startExists = Date.now()
const exists = await brain.vfs.exists('/perf-test-25.txt', { commitId: perfCommit })
const existsTime = Date.now() - startExists
expect(exists).toBe(true)
console.log(` ✅ exists: ${existsTime}ms`)
expect(existsTime).toBeLessThan(1000) // Should complete in < 1s
})
it('should work with nested directory structures', async () => {
console.log('\n🌲 Test 8: Nested directories at historical commits')
// Create nested structure
await brain.vfs.mkdir('/deep')
await brain.vfs.mkdir('/deep/level1')
await brain.vfs.mkdir('/deep/level1/level2')
await brain.vfs.writeFile('/deep/level1/level2/deep-file.txt', 'Deep content')
// Commit: Deep structure
await brain.commit({
message: 'Added deep directory structure',
author: 'test@example.com'
})
const history1 = await brain.getHistory({ limit: 1 })
const deepCommit = history1[0].hash
console.log(` Deep structure commit: ${deepCommit.slice(0, 8)}`)
// Test 8a: Read deep file
const content = await brain.vfs.readFile('/deep/level1/level2/deep-file.txt', {
commitId: deepCommit
})
expect(content.toString()).toBe('Deep content')
console.log(` ✅ Read file from nested directory`)
// Test 8b: List intermediate directory
const level1Entries = await brain.vfs.readdir('/deep/level1', { commitId: deepCommit })
expect(level1Entries).toContain('level2')
console.log(` ✅ List intermediate directory`)
// Test 8c: Stat deep directory
const dirStats = await brain.vfs.stat('/deep/level1/level2', { commitId: deepCommit })
expect(dirStats.isDirectory()).toBe(true)
console.log(` ✅ Stat nested directory`)
// Test 8d: Check existence of deep path
const exists = await brain.vfs.exists('/deep/level1/level2/deep-file.txt', {
commitId: deepCommit
})
expect(exists).toBe(true)
console.log(` ✅ Check existence of deep path`)
})
})

307
tests/integration/vfs-versioning.test.ts
View file

@ -1,307 +0,0 @@
/**
* VFS File Versioning Integration Tests (v6.3.2)
*
* Tests the fix for the VFS versioning bug where file content was stale.
*
* Bug: VFS files store content in BlobStorage, but versioning captured stale
* embedding text from entity.data instead of actual file content.
*
* Fix: VersionManager.save() now reads fresh content from BlobStorage for VFS files,
* and restore() writes content back to BlobStorage.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
import * as path from 'path'
describe('VFS File Versioning (v6.3.2 Fix)', () => {
const testDir = path.join(process.cwd(), 'test-vfs-versioning-fix')
let brain: Brainy
beforeAll(async () => {
// Clean up from any previous run
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
fs.mkdirSync(testDir, { recursive: true })
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
options: { path: testDir }
},
silent: true
})
await brain.init()
})
afterAll(() => {
// Clean up test directory
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
})
describe('Core VFS Versioning', () => {
it('should version actual file content, not stale embedding text', async () => {
// Create a text file
await brain.vfs.writeFile('/test/file.md', 'Initial content - version 1')
// Get the file entity
const stat = await brain.vfs.stat('/test/file.md')
const entityId = stat.entityId
// Save version 1
await brain.versions.save(entityId, { description: 'Version 1' })
// Modify the file with different content
await brain.vfs.writeFile('/test/file.md', 'Modified content - version 2 with more text')
// Save version 2
await brain.versions.save(entityId, { description: 'Version 2' })
// Retrieve both versions
const v1Content = await brain.versions.getContent(entityId, 1)
const v2Content = await brain.versions.getContent(entityId, 2)
// CRITICAL TEST: The data should be DIFFERENT between versions
// This was the bug - both had the same stale content
expect(v1Content.data).not.toBe(v2Content.data)
// Verify actual content
expect(v1Content.data).toBe('Initial content - version 1')
expect(v2Content.data).toBe('Modified content - version 2 with more text')
})
it('should restore VFS file content correctly', async () => {
// Create initial file
await brain.vfs.writeFile('/docs/readme.md', 'README version 1')
const stat = await brain.vfs.stat('/docs/readme.md')
const entityId = stat.entityId
// Save version 1
await brain.versions.save(entityId, { tag: 'v1' })
// Modify file
await brain.vfs.writeFile('/docs/readme.md', 'README version 2 - updated')
// Save version 2
await brain.versions.save(entityId, { tag: 'v2' })
// Verify current content is v2
let currentContent = await brain.vfs.readFile('/docs/readme.md')
expect(currentContent.toString()).toBe('README version 2 - updated')
// Restore to v1
await brain.versions.restore(entityId, 1)
// Verify content is now v1
currentContent = await brain.vfs.readFile('/docs/readme.md')
expect(currentContent.toString()).toBe('README version 1')
})
it('should handle multiple file edits and versions', async () => {
await brain.vfs.writeFile('/project/config.json', '{"version": 1}')
const stat = await brain.vfs.stat('/project/config.json')
const entityId = stat.entityId
// Create 5 versions
for (let i = 1; i <= 5; i++) {
await brain.vfs.writeFile('/project/config.json', `{"version": ${i}}`)
await brain.versions.save(entityId, { tag: `v${i}` })
}
// Verify all 5 versions exist
const versions = await brain.versions.list(entityId)
expect(versions).toHaveLength(5)
// Verify each version has correct content
for (let i = 1; i <= 5; i++) {
const content = await brain.versions.getContent(entityId, i)
expect(content.data).toBe(`{"version": ${i}}`)
}
})
it('should restore to any version in history', async () => {
await brain.vfs.writeFile('/notes/todo.txt', 'Task 1')
const stat = await brain.vfs.stat('/notes/todo.txt')
const entityId = stat.entityId
await brain.versions.save(entityId, { tag: 'initial' })
await brain.vfs.writeFile('/notes/todo.txt', 'Task 1\nTask 2')
await brain.versions.save(entityId, { tag: 'two-tasks' })
await brain.vfs.writeFile('/notes/todo.txt', 'Task 1\nTask 2\nTask 3')
await brain.versions.save(entityId, { tag: 'three-tasks' })
// Restore to middle version
await brain.versions.restore(entityId, 'two-tasks')
let content = await brain.vfs.readFile('/notes/todo.txt')
expect(content.toString()).toBe('Task 1\nTask 2')
// Restore to initial version
await brain.versions.restore(entityId, 'initial')
content = await brain.vfs.readFile('/notes/todo.txt')
expect(content.toString()).toBe('Task 1')
// Restore to latest version
await brain.versions.restore(entityId, 3)
content = await brain.vfs.readFile('/notes/todo.txt')
expect(content.toString()).toBe('Task 1\nTask 2\nTask 3')
})
})
describe('Binary File Versioning', () => {
it('should version binary files with base64 encoding', async () => {
// Create a simple binary file (simulated image header)
const binaryContent1 = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x01])
const binaryContent2 = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x02])
await brain.vfs.writeFile('/images/test.png', binaryContent1)
const stat = await brain.vfs.stat('/images/test.png')
const entityId = stat.entityId
// Save version 1
await brain.versions.save(entityId, { tag: 'v1' })
// Modify binary content
await brain.vfs.writeFile('/images/test.png', binaryContent2)
// Save version 2
await brain.versions.save(entityId, { tag: 'v2' })
// Retrieve and verify both versions are different
const v1Content = await brain.versions.getContent(entityId, 1)
const v2Content = await brain.versions.getContent(entityId, 2)
expect(v1Content.data).not.toBe(v2Content.data)
// Verify binary content can be restored
await brain.versions.restore(entityId, 1)
const restored = await brain.vfs.readFile('/images/test.png')
expect(Buffer.compare(restored, binaryContent1)).toBe(0)
})
})
describe('Version Deduplication', () => {
it('should deduplicate identical VFS file content', async () => {
await brain.vfs.writeFile('/cache/data.txt', 'Cached data')
const stat = await brain.vfs.stat('/cache/data.txt')
const entityId = stat.entityId
// Save version 1
const v1 = await brain.versions.save(entityId, { tag: 'v1' })
// Save again without changes - should dedupe
const v2 = await brain.versions.save(entityId, { tag: 'v2' })
// Should return same version (content hash match)
expect(v2.version).toBe(v1.version)
expect(v2.contentHash).toBe(v1.contentHash)
// Only 1 version should exist
const versions = await brain.versions.list(entityId)
expect(versions).toHaveLength(1)
})
})
describe('Mixed Entity Versioning', () => {
it('should handle VFS and non-VFS entities in same brain', async () => {
// Create a VFS file
await brain.vfs.writeFile('/files/doc.txt', 'File content v1')
const fileStat = await brain.vfs.stat('/files/doc.txt')
const fileId = fileStat.entityId
// Create a regular entity
const regularId = await brain.add({
data: 'Regular entity',
type: 'document',
metadata: { title: 'Regular Doc' }
})
// Version both
await brain.versions.save(fileId, { tag: 'file-v1' })
await brain.versions.save(regularId, { tag: 'regular-v1' })
// Update both
await brain.vfs.writeFile('/files/doc.txt', 'File content v2')
await brain.update({ id: regularId, metadata: { title: 'Updated Regular Doc' } })
// Version both again
await brain.versions.save(fileId, { tag: 'file-v2' })
await brain.versions.save(regularId, { tag: 'regular-v2' })
// Verify VFS file versions have different content
const fileV1 = await brain.versions.getContent(fileId, 1)
const fileV2 = await brain.versions.getContent(fileId, 2)
expect(fileV1.data).toBe('File content v1')
expect(fileV2.data).toBe('File content v2')
// Verify regular entity versions work too
const regV1 = await brain.versions.getContent(regularId, 1)
const regV2 = await brain.versions.getContent(regularId, 2)
expect(regV1.title).toBe('Regular Doc')
expect(regV2.title).toBe('Updated Regular Doc')
})
})
describe('Edge Cases', () => {
it('should handle minimal content files', async () => {
// VFS requires non-empty data for embedding, so we use minimal content
await brain.vfs.writeFile('/minimal/file.txt', ' ') // Single space
const stat = await brain.vfs.stat('/minimal/file.txt')
const entityId = stat.entityId
await brain.versions.save(entityId, { tag: 'minimal' })
await brain.vfs.writeFile('/minimal/file.txt', 'Now has content')
await brain.versions.save(entityId, { tag: 'filled' })
// Restore to minimal
await brain.versions.restore(entityId, 'minimal')
const content = await brain.vfs.readFile('/minimal/file.txt')
expect(content.toString()).toBe(' ')
})
it('should handle large text files', async () => {
const largeContent1 = 'A'.repeat(100000) + ' version 1'
const largeContent2 = 'B'.repeat(100000) + ' version 2'
await brain.vfs.writeFile('/large/file.txt', largeContent1)
const stat = await brain.vfs.stat('/large/file.txt')
const entityId = stat.entityId
await brain.versions.save(entityId, { tag: 'v1' })
await brain.vfs.writeFile('/large/file.txt', largeContent2)
await brain.versions.save(entityId, { tag: 'v2' })
// Verify versions are different
const v1 = await brain.versions.getContent(entityId, 1)
const v2 = await brain.versions.getContent(entityId, 2)
expect(v1.data).toBe(largeContent1)
expect(v2.data).toBe(largeContent2)
})
it('should handle special characters in content', async () => {
const specialContent = 'Unicode: \u{1F600} \u{1F914} \u{1F4A1}\nNewlines\nAnd\ttabs'
await brain.vfs.writeFile('/special.txt', specialContent)
const stat = await brain.vfs.stat('/special.txt')
const entityId = stat.entityId
await brain.versions.save(entityId, { tag: 'special' })
await brain.vfs.writeFile('/special.txt', 'Plain text now')
await brain.versions.save(entityId, { tag: 'plain' })
// Restore special content
await brain.versions.restore(entityId, 'special')
const content = await brain.vfs.readFile('/special.txt')
expect(content.toString()).toBe(specialContent)
})
})
})

265
tests/transaction/integration/cow-transactions.test.ts
View file

@ -1,265 +0,0 @@
/**
* COW + Transactions Integration Tests
*
* Verifies that transactions work correctly with Copy-on-Write storage:
* - Branch isolation
* - Atomic commits
* - Rollback without affecting main branch
* - Content-addressable storage
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../../src/brainy.js'
import { NounType, VerbType } from '../../../src/types/graphTypes.js'
import { tmpdir } from 'os'
import { join } from 'path'
import { mkdirSync, rmSync } from 'fs'
describe('Transactions + COW Integration', () => {
let brain: Brainy
let testDir: string
beforeEach(async () => {
// Create unique test directory
testDir = join(tmpdir(), `brainy-cow-test-${Date.now()}`)
mkdirSync(testDir, { recursive: true })
// Initialize Brainy with COW enabled
brain = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
path: testDir
}
})
await brain.init()
// Enable COW if available
if (brain.cow) {
await brain.cow.init()
}
})
afterEach(async () => {
if (brain) {
await brain.shutdown()
}
if (testDir) {
rmSync(testDir, { recursive: true, force: true })
}
})
describe('Basic COW Operations', () => {
it('should commit transaction successfully on COW branch', async () => {
// Create branch
if (!brain.cow) {
console.log('COW not available, skipping test')
return
}
await brain.cow.createBranch('feature-branch')
await brain.cow.checkout('feature-branch')
// Add entity (should succeed)
const id = await brain.add({
data: { name: 'Test Entity' },
type: NounType.Thing
})
expect(id).toBeTruthy()
// Verify entity exists
const entity = await brain.get(id)
expect(entity).toBeTruthy()
expect(entity?.data.name).toBe('Test Entity')
})
it('should isolate transaction rollback to branch', async () => {
if (!brain.cow) {
console.log('COW not available, skipping test')
return
}
// Add entity on main branch
const mainId = await brain.add({
data: { name: 'Main Entity' },
type: NounType.Thing
})
// Create and checkout feature branch
await brain.cow.createBranch('feature-branch')
await brain.cow.checkout('feature-branch')
// Try to add entity that will fail
try {
await brain.add({
data: { name: 'Feature Entity' },
type: NounType.Thing,
// Force failure by providing invalid vector
vector: [] as any
})
} catch (e) {
// Expected to fail
}
// Switch back to main
await brain.cow.checkout('main')
// Main branch entity should still exist
const mainEntity = await brain.get(mainId)
expect(mainEntity).toBeTruthy()
expect(mainEntity?.data.name).toBe('Main Entity')
})
it('should handle atomic updates across COW branches', async () => {
if (!brain.cow) {
console.log('COW not available, skipping test')
return
}
// Add entity
const id = await brain.add({
data: { name: 'Original', version: 1 },
type: NounType.Thing
})
// Create branch
await brain.cow.createBranch('feature-branch')
await brain.cow.checkout('feature-branch')
// Update entity (atomic operation)
await brain.update({
id,
data: { name: 'Updated', version: 2 },
merge: false
})
// Verify update on feature branch
const featureEntity = await brain.get(id)
expect(featureEntity?.data.version).toBe(2)
// Switch to main
await brain.cow.checkout('main')
// Original should be unchanged on main
const mainEntity = await brain.get(id)
expect(mainEntity?.data.version).toBe(1)
})
})
describe('Transaction Rollback with COW', () => {
it('should rollback failed transaction without affecting COW branch', async () => {
if (!brain.cow) {
console.log('COW not available, skipping test')
return
}
await brain.cow.createBranch('test-branch')
await brain.cow.checkout('test-branch')
// Add first entity (will succeed)
const id1 = await brain.add({
data: { name: 'Entity 1' },
type: NounType.Thing
})
// Attempt to add with invalid data (will fail)
// This tests that the transaction rollback doesn't corrupt the branch
let failed = false
try {
await brain.add({
data: null as any, // Invalid
type: NounType.Thing
})
} catch (e) {
failed = true
}
expect(failed).toBe(true)
// First entity should still exist (transaction rollback worked)
const entity1 = await brain.get(id1)
expect(entity1).toBeTruthy()
})
})
describe('COW Branch Merging with Transactions', () => {
it('should preserve transaction atomicity during branch operations', async () => {
if (!brain.cow) {
console.log('COW not available, skipping test')
return
}
// Add entity on main
const mainId = await brain.add({
data: { name: 'Main Entity', branch: 'main' },
type: NounType.Thing
})
// Create feature branch
await brain.cow.createBranch('feature')
await brain.cow.checkout('feature')
// Add multiple entities in transaction (atomic)
const featureId1 = await brain.add({
data: { name: 'Feature 1', branch: 'feature' },
type: NounType.Thing
})
const featureId2 = await brain.add({
data: { name: 'Feature 2', branch: 'feature' },
type: NounType.Thing
})
// Create relationship (atomic)
await brain.relate({
from: featureId1,
to: featureId2,
type: VerbType.RelatesTo
})
// All operations should be atomic on feature branch
const feature1 = await brain.get(featureId1)
const feature2 = await brain.get(featureId2)
const relations = await brain.getRelations({ from: featureId1 })
expect(feature1).toBeTruthy()
expect(feature2).toBeTruthy()
expect(relations).toHaveLength(1)
// Switch back to main - feature entities should not exist
await brain.cow.checkout('main')
const mainCheck = await brain.get(featureId1)
expect(mainCheck).toBeNull()
})
})
describe('Content-Addressable Storage', () => {
it('should handle content-addressable storage with transactions', async () => {
if (!brain.cow) {
console.log('COW not available, skipping test')
return
}
// Add entity with specific data
const id = await brain.add({
data: { content: 'Test content', value: 123 },
type: NounType.Thing
})
// Update entity (creates new blob)
await brain.update({
id,
data: { content: 'Updated content', value: 456 }
})
// Get entity - should have updated content
const entity = await brain.get(id)
expect(entity?.data.content).toBe('Updated content')
expect(entity?.data.value).toBe(456)
// Transaction system should work with content-addressable storage
// (each version is a separate blob, rollback restores previous blob reference)
})
})
})

467
tests/typeAwareHNSWIndex.test.ts
View file

@ -1,467 +0,0 @@
/**
* TypeAwareHNSWIndex Unit Tests
*
* Comprehensive test suite for Phase 2 Type-Aware HNSW implementation.
* Tests cover:
* - Lazy initialization
* - Type routing (single/multi/all types)
* - Edge cases (empty array, null, invalid type)
* - Error handling
* - Memory isolation
* - Statistics
* - Configuration
*
* Total: 25 unit tests
*/
import { describe, it, expect, beforeEach } from 'vitest'
import { TypeAwareHNSWIndex } from '../src/hnsw/typeAwareHNSWIndex.js'
import type { NounType } from '../src/types/graphTypes.js'
import { euclideanDistance } from '../src/utils/index.js'
import { MemoryStorage } from '../src/storage/adapters/memoryStorage.js'
describe('TypeAwareHNSWIndex', () => {
let index: TypeAwareHNSWIndex
let storage: MemoryStorage
beforeEach(() => {
storage = new MemoryStorage()
index = new TypeAwareHNSWIndex(
{ M: 4, efConstruction: 50, efSearch: 20 },
euclideanDistance,
{ storage }
)
})
// ===================================================================
// 1. LAZY INITIALIZATION
// ===================================================================
describe('Lazy Initialization', () => {
it('should not create indexes upfront', () => {
expect(index.getActiveTypes()).toHaveLength(0)
expect(index.size()).toBe(0)
})
it('should create index only when first entity added', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 2, 3] },
'person' as NounType
)
expect(index.getActiveTypes()).toContain('person')
expect(index.getActiveTypes()).toHaveLength(1)
expect(index.sizeForType('person' as NounType)).toBe(1)
})
it('should create separate indexes for different types', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 2, 3] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [4, 5, 6] },
'document' as NounType
)
expect(index.getActiveTypes()).toHaveLength(2)
expect(index.getActiveTypes()).toContain('person')
expect(index.getActiveTypes()).toContain('document')
expect(index.sizeForType('person' as NounType)).toBe(1)
expect(index.sizeForType('document' as NounType)).toBe(1)
})
it('should not create index for types with no entities', () => {
expect(index.sizeForType('event' as NounType)).toBe(0)
expect(index.getActiveTypes()).not.toContain('event')
})
})
// ===================================================================
// 2. TYPE ROUTING
// ===================================================================
describe('Type Routing', () => {
beforeEach(async () => {
// Add entities of different types
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'person-2', vector: [1, 0.1, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
await index.addItem(
{ id: 'event-1', vector: [0, 0, 1] },
'event' as NounType
)
})
it('should search single type only (fast path)', async () => {
const results = await index.search([1, 0, 0], 2, 'person' as NounType)
expect(results).toHaveLength(2)
expect(results[0][0]).toBe('person-1') // Exact match
expect(results[1][0]).toBe('person-2') // Close match
})
it('should search multiple types', async () => {
const results = await index.search(
[1, 0, 0],
3,
['person', 'document'] as NounType[]
)
expect(results).toHaveLength(3)
const ids = results.map((r) => r[0])
expect(ids).toContain('person-1')
expect(ids).toContain('person-2')
expect(ids).toContain('doc-1')
expect(ids).not.toContain('event-1') // Not searched
})
it('should search all types when type not specified', async () => {
const results = await index.search([1, 0, 0], 4)
expect(results).toHaveLength(4)
const ids = results.map((r) => r[0])
expect(ids).toContain('person-1')
expect(ids).toContain('person-2')
expect(ids).toContain('doc-1')
expect(ids).toContain('event-1')
})
it('should return results sorted by distance', async () => {
const results = await index.search([1, 0, 0], 4)
// Distances should be increasing
for (let i = 0; i < results.length - 1; i++) {
expect(results[i][1]).toBeLessThanOrEqual(results[i + 1][1])
}
})
})
// ===================================================================
// 3. EDGE CASE HANDLING
// ===================================================================
describe('Edge Cases', () => {
it('should handle empty array in search() (fall through to all types)', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
const results = await index.search([1, 0, 0], 10, [] as NounType[])
// Should search all types (fallback behavior)
expect(results).toHaveLength(1)
expect(results[0][0]).toBe('person-1')
})
it('should throw on null item in addItem()', async () => {
await expect(
index.addItem(null as any, 'person' as NounType)
).rejects.toThrow('Invalid VectorDocument: item or vector is null/undefined')
})
it('should throw on undefined vector in addItem()', async () => {
await expect(
index.addItem({ id: 'test' } as any, 'person' as NounType)
).rejects.toThrow('Invalid VectorDocument: item or vector is null/undefined')
})
it('should throw on null type in addItem()', async () => {
await expect(
index.addItem({ id: 'test', vector: [1, 2, 3] }, null as any)
).rejects.toThrow('Type is required for type-aware indexing')
})
it('should throw on invalid type string', async () => {
await expect(
index.addItem(
{ id: 'test', vector: [1, 2, 3] },
'not-a-valid-noun-type-at-all' as any
)
).rejects.toThrow('Invalid NounType')
})
it('should handle search with no results', async () => {
const results = await index.search([1, 2, 3], 10, 'person' as NounType)
expect(results).toHaveLength(0)
})
it('should handle removeItem() for non-existent type', async () => {
const removed = await index.removeItem('test-id', 'person' as NounType)
expect(removed).toBe(false)
})
})
// ===================================================================
// 4. ADD/REMOVE/SEARCH OPERATIONS
// ===================================================================
describe('Operations', () => {
it('should add item and return ID', async () => {
const id = await index.addItem(
{ id: 'person-1', vector: [1, 2, 3] },
'person' as NounType
)
expect(id).toBe('person-1')
expect(index.size()).toBe(1)
})
it('should remove item from correct type', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 2, 3] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [4, 5, 6] },
'document' as NounType
)
const removed = await index.removeItem('person-1', 'person' as NounType)
expect(removed).toBe(true)
expect(index.sizeForType('person' as NounType)).toBe(0)
expect(index.sizeForType('document' as NounType)).toBe(1) // Unchanged
})
it('should search with filter function', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'person-2', vector: [1, 0.1, 0] },
'person' as NounType
)
const filter = async (id: string) => id === 'person-1'
const results = await index.search(
[1, 0, 0],
2,
'person' as NounType,
filter
)
expect(results).toHaveLength(1)
expect(results[0][0]).toBe('person-1')
})
})
// ===================================================================
// 5. MEMORY ISOLATION
// ===================================================================
describe('Memory Isolation', () => {
it('should maintain separate memory for each type', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
// Clear person type only
index.clearType('person' as NounType)
expect(index.sizeForType('person' as NounType)).toBe(0)
expect(index.sizeForType('document' as NounType)).toBe(1) // Unchanged
})
it('should clear all indexes', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
index.clear()
expect(index.size()).toBe(0)
expect(index.getActiveTypes()).toHaveLength(0)
})
})
// ===================================================================
// 6. SIZE AND STATISTICS
// ===================================================================
describe('Size and Statistics', () => {
it('should return total size across all types', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'person-2', vector: [1, 0.1, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
expect(index.size()).toBe(3)
})
it('should return size for specific type', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'person-2', vector: [1, 0.1, 0] },
'person' as NounType
)
expect(index.sizeForType('person' as NounType)).toBe(2)
expect(index.sizeForType('document' as NounType)).toBe(0)
})
it('should return comprehensive statistics', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
const stats = index.getStats()
expect(stats.totalNodes).toBe(2)
expect(stats.typeCount).toBe(2)
expect(stats.typeStats.has('person' as NounType)).toBe(true)
expect(stats.typeStats.has('document' as NounType)).toBe(true)
expect(stats.memoryReductionPercent).toBeGreaterThan(0)
})
it('should return stats for specific type', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
const stats = index.getStatsForType('person' as NounType)
expect(stats).not.toBeNull()
expect(stats!.nodeCount).toBe(1)
expect(stats!.memoryMB).toBeGreaterThanOrEqual(0)
})
it('should return null stats for non-existent type', () => {
const stats = index.getStatsForType('person' as NounType)
expect(stats).toBeNull()
})
it('should calculate memory reduction percentage', async () => {
// Add multiple entities to make calculation meaningful
for (let i = 0; i < 100; i++) {
await index.addItem(
{ id: `person-${i}`, vector: [Math.random(), Math.random(), Math.random()] },
'person' as NounType
)
}
const stats = index.getStats()
expect(stats.totalNodes).toBe(100)
expect(stats.estimatedMonolithicMemoryMB).toBeGreaterThan(0)
expect(stats.memoryReductionPercent).toBeGreaterThanOrEqual(0)
expect(stats.memoryReductionPercent).toBeLessThanOrEqual(100)
})
it('should handle stats with empty indexes', () => {
const stats = index.getStats()
expect(stats.totalNodes).toBe(0)
expect(stats.typeCount).toBe(0)
expect(stats.totalMemoryMB).toBe(0)
expect(stats.memoryReductionPercent).toBe(0)
})
})
// ===================================================================
// 7. CONFIGURATION
// ===================================================================
describe('Configuration', () => {
it('should return HNSW configuration', () => {
const config = index.getConfig()
expect(config.M).toBe(4)
expect(config.efConstruction).toBe(50)
expect(config.efSearch).toBe(20)
})
it('should return distance function', () => {
const distFn = index.getDistanceFunction()
expect(distFn).toBe(euclideanDistance)
})
it('should get parallelization setting', () => {
const parallel = index.getUseParallelization()
expect(parallel).toBe(true) // Default
})
it('should set parallelization for all indexes', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
index.setUseParallelization(false)
expect(index.getUseParallelization()).toBe(false)
})
})
// ===================================================================
// 8. ACTIVE TYPES
// ===================================================================
describe('Active Types', () => {
it('should return list of types with entities', async () => {
await index.addItem(
{ id: 'person-1', vector: [1, 0, 0] },
'person' as NounType
)
await index.addItem(
{ id: 'doc-1', vector: [0, 1, 0] },
'document' as NounType
)
const activeTypes = index.getActiveTypes()
expect(activeTypes).toHaveLength(2)
expect(activeTypes).toContain('person')
expect(activeTypes).toContain('document')
})
it('should return empty array when no types have entities', () => {
const activeTypes = index.getActiveTypes()
expect(activeTypes).toHaveLength(0)
})
})
})

30
tests/unit/storage/cow/BlobStorage.compression-policy.test.ts → tests/unit/storage/blobStorage.compression-policy.test.ts
View file

@ -2,7 +2,7 @@
* @module BlobStorage.compression-policy.test * @module BlobStorage.compression-policy.test
* @description 2.5.0 #32 — content-type-aware compression policy. * @description 2.5.0 #32 — content-type-aware compression policy.
* *
* COW's BlobStorage now consults `BlobWriteOptions.mimeType` in `auto` mode * BlobStorage consults `BlobWriteOptions.mimeType` in `auto` mode
* and skips zstd for MIME types known to be already heavily compressed (JPEG, * and skips zstd for MIME types known to be already heavily compressed (JPEG,
* PNG, MP4, MP3, ZIP, PDF, etc.). zstd over those formats is reliably a CPU * PNG, MP4, MP3, ZIP, PDF, etc.). zstd over those formats is reliably a CPU
* loss for no measurable byte savings. * loss for no measurable byte savings.
@ -24,11 +24,11 @@
import { describe, it, expect, beforeEach } from 'vitest' import { describe, it, expect, beforeEach } from 'vitest'
import { import {
BlobStorage, BlobStorage,
COWStorageAdapter, BlobStoreAdapter,
isAlreadyCompressedMimeType isAlreadyCompressedMimeType
} from '../../../../src/storage/cow/BlobStorage.js' } from '../../../src/storage/blobStorage.js'
class InMemoryCOWAdapter implements COWStorageAdapter { class InMemoryBlobAdapter implements BlobStoreAdapter {
private store = new Map<string, Buffer>() private store = new Map<string, Buffer>()
async get(key: string): Promise<Buffer | undefined> { return this.store.get(key) } async get(key: string): Promise<Buffer | undefined> { return this.store.get(key) }
async put(key: string, data: Buffer): Promise<void> { this.store.set(key, data) } async put(key: string, data: Buffer): Promise<void> { this.store.set(key, data) }
@ -45,12 +45,12 @@ class InMemoryCOWAdapter implements COWStorageAdapter {
} }
describe('BlobStorage compression policy (2.5.0 #32)', () => { describe('BlobStorage compression policy (2.5.0 #32)', () => {
let adapter: InMemoryCOWAdapter let adapter: InMemoryBlobAdapter
let blob: BlobStorage let blob: BlobStorage
beforeEach(() => { beforeEach(() => {
adapter = new InMemoryCOWAdapter() adapter = new InMemoryBlobAdapter()
blob = new BlobStorage(adapter, { enableCompression: true }) blob = new BlobStorage(adapter)
}) })
describe('isAlreadyCompressedMimeType helper', () => { describe('isAlreadyCompressedMimeType helper', () => {
@ -90,7 +90,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
it('skips compression for image/jpeg in auto mode (metadata records "none")', async () => { it('skips compression for image/jpeg in auto mode (metadata records "none")', async () => {
const hash = await blob.write(HIGHLY_COMPRESSIBLE, { const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
compression: 'auto', type: 'blob', mimeType: 'image/jpeg' compression: 'auto', mimeType: 'image/jpeg'
}) })
const meta = await blob.getMetadata(hash) const meta = await blob.getMetadata(hash)
expect(meta?.compression).toBe('none') expect(meta?.compression).toBe('none')
@ -101,7 +101,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
it('skips compression for video/mp4 in auto mode', async () => { it('skips compression for video/mp4 in auto mode', async () => {
const hash = await blob.write(HIGHLY_COMPRESSIBLE, { const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
compression: 'auto', type: 'blob', mimeType: 'video/mp4' compression: 'auto', mimeType: 'video/mp4'
}) })
const meta = await blob.getMetadata(hash) const meta = await blob.getMetadata(hash)
expect(meta?.compression).toBe('none') expect(meta?.compression).toBe('none')
@ -109,7 +109,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
it('skips compression for application/zip in auto mode', async () => { it('skips compression for application/zip in auto mode', async () => {
const hash = await blob.write(HIGHLY_COMPRESSIBLE, { const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
compression: 'auto', type: 'blob', mimeType: 'application/zip' compression: 'auto', mimeType: 'application/zip'
}) })
const meta = await blob.getMetadata(hash) const meta = await blob.getMetadata(hash)
expect(meta?.compression).toBe('none') expect(meta?.compression).toBe('none')
@ -122,7 +122,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
// falls back to 'none' — same outcome as a no-zstd install would see // falls back to 'none' — same outcome as a no-zstd install would see
// in production. Either way, the policy didn't block on text/plain. // in production. Either way, the policy didn't block on text/plain.
const hash = await blob.write(HIGHLY_COMPRESSIBLE, { const hash = await blob.write(HIGHLY_COMPRESSIBLE, {
compression: 'auto', type: 'blob', mimeType: 'text/plain' compression: 'auto', mimeType: 'text/plain'
}) })
const meta = await blob.getMetadata(hash) const meta = await blob.getMetadata(hash)
expect(meta?.compression === 'zstd' || meta?.compression === 'none').toBe(true) expect(meta?.compression === 'zstd' || meta?.compression === 'none').toBe(true)
@ -130,10 +130,10 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
it('decompresses transparently on read regardless of compression decision', async () => { it('decompresses transparently on read regardless of compression decision', async () => {
const jpegHash = await blob.write(HIGHLY_COMPRESSIBLE, { const jpegHash = await blob.write(HIGHLY_COMPRESSIBLE, {
compression: 'auto', type: 'blob', mimeType: 'image/jpeg' compression: 'auto', mimeType: 'image/jpeg'
}) })
const textHash = await blob.write(HIGHLY_COMPRESSIBLE, { const textHash = await blob.write(HIGHLY_COMPRESSIBLE, {
compression: 'auto', type: 'blob', mimeType: 'text/plain' compression: 'auto', mimeType: 'text/plain'
}) })
// Different mime-type policies, same bytes back on read. // Different mime-type policies, same bytes back on read.
expect((await blob.read(jpegHash)).equals(HIGHLY_COMPRESSIBLE)).toBe(true) expect((await blob.read(jpegHash)).equals(HIGHLY_COMPRESSIBLE)).toBe(true)
@ -145,7 +145,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
it('compression: "none" is honoured for text/plain (policy can\'t force compression on)', async () => { it('compression: "none" is honoured for text/plain (policy can\'t force compression on)', async () => {
const data = Buffer.alloc(4096, 0x43) const data = Buffer.alloc(4096, 0x43)
const hash = await blob.write(data, { const hash = await blob.write(data, {
compression: 'none', type: 'blob', mimeType: 'text/plain' compression: 'none', mimeType: 'text/plain'
}) })
const meta = await blob.getMetadata(hash) const meta = await blob.getMetadata(hash)
expect(meta?.compression).toBe('none') expect(meta?.compression).toBe('none')
@ -156,7 +156,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => {
it('compression: "zstd" is recorded as the intent for image/jpeg (policy applies only to auto)', async () => { it('compression: "zstd" is recorded as the intent for image/jpeg (policy applies only to auto)', async () => {
const data = Buffer.alloc(4096, 0x42) const data = Buffer.alloc(4096, 0x42)
const hash = await blob.write(data, { const hash = await blob.write(data, {
compression: 'zstd', type: 'blob', mimeType: 'image/jpeg' compression: 'zstd', mimeType: 'image/jpeg'
}) })
const meta = await blob.getMetadata(hash) const meta = await blob.getMetadata(hash)
// Explicit zstd → the policy DOESN'T short-circuit; the metadata // Explicit zstd → the policy DOESN'T short-circuit; the metadata

392
tests/unit/storage/cow/BlobStorage.test.ts → tests/unit/storage/blobStorage.test.ts
View file

@ -1,25 +1,28 @@
/** /**
* Comprehensive tests for BlobStorage * Comprehensive tests for BlobStorage (src/storage/blobStorage.ts)
* *
* Tests: * Tests:
* - Content-addressable storage (SHA-256) * - Content-addressable storage (SHA-256, integrity verification)
* - Deduplication * - Deduplication via reference counting
* - Compression (zstd) * - Compression (zstd, MIME-aware auto mode)
* - LRU caching * - LRU caching (bounded cache still serves correct bytes)
* - Batch operations * - Reference counting + delete-at-zero
* - Reference counting
* - Garbage collection
* - Error handling * - Error handling
* - Performance characteristics * - Performance characteristics
* - Wrapped-binary-data regressions (key-based dispatch)
*
* Cache-bypass pattern: the store has no cache-introspection API (the LRU is
* an internal optimization), so tests that must force a storage read create a
* FRESH BlobStorage over the same adapter — a cold cache by construction.
*/ */
import { describe, it, expect, beforeEach } from 'vitest' import { describe, it, expect, beforeEach } from 'vitest'
import { BlobStorage, COWStorageAdapter } from '../../../../src/storage/cow/BlobStorage.js' import { BlobStorage, BlobStoreAdapter } from '../../../src/storage/blobStorage.js'
/** /**
* Simple in-memory COW storage adapter for testing * Simple in-memory blob store adapter for testing
*/ */
class InMemoryCOWAdapter implements COWStorageAdapter { class InMemoryBlobAdapter implements BlobStoreAdapter {
private store = new Map<string, Buffer>() private store = new Map<string, Buffer>()
async get(key: string): Promise<Buffer | undefined> { async get(key: string): Promise<Buffer | undefined> {
@ -46,12 +49,12 @@ class InMemoryCOWAdapter implements COWStorageAdapter {
} }
describe('BlobStorage', () => { describe('BlobStorage', () => {
let adapter: COWStorageAdapter let adapter: BlobStoreAdapter
let blobStorage: BlobStorage let blobStorage: BlobStorage
beforeEach(() => { beforeEach(() => {
adapter = new InMemoryCOWAdapter() adapter = new InMemoryBlobAdapter()
blobStorage = new BlobStorage(adapter, { enableCompression: true }) blobStorage = new BlobStorage(adapter)
}) })
describe('Content-Addressable Storage', () => { describe('Content-Addressable Storage', () => {
@ -84,14 +87,13 @@ describe('BlobStorage', () => {
const data = Buffer.from('test data') const data = Buffer.from('test data')
const hash = await blobStorage.write(data) const hash = await blobStorage.write(data)
// Clear cache so read() fetches from storage // Corrupt the blob data behind the store's back
blobStorage.clearCache()
// Corrupt the blob data
await adapter.put(`blob:${hash}`, Buffer.from('corrupted')) await adapter.put(`blob:${hash}`, Buffer.from('corrupted'))
// Should detect corruption via hash verification // A fresh instance has a cold cache, so read() must hit storage and
await expect(blobStorage.read(hash)).rejects.toThrow('integrity check failed') // detect the corruption via hash verification.
const coldStore = new BlobStorage(adapter)
await expect(coldStore.read(hash)).rejects.toThrow('integrity check failed')
}) })
it('should check if blob exists', async () => { it('should check if blob exists', async () => {
@ -112,9 +114,9 @@ describe('BlobStorage', () => {
expect(hash1).toBe(hash2) expect(hash1).toBe(hash2)
const stats = blobStorage.getStats() // Exactly one stored blob + one metadata record
expect(stats.totalBlobs).toBe(1) expect(await adapter.list('blob:')).toHaveLength(1)
expect(stats.dedupSavings).toBe(data.length) expect(await adapter.list('blob-meta:')).toHaveLength(1)
}) })
it('should increment ref count on duplicate write', async () => { it('should increment ref count on duplicate write', async () => {
@ -127,15 +129,17 @@ describe('BlobStorage', () => {
expect(metadata?.refCount).toBe(2) expect(metadata?.refCount).toBe(2)
}) })
it('should track deduplication savings', async () => { it('should track every duplicate write in the reference count', async () => {
const data = Buffer.from('x'.repeat(1000)) const data = Buffer.from('x'.repeat(1000))
const hash1 = await blobStorage.write(data) const hash1 = await blobStorage.write(data)
const hash2 = await blobStorage.write(data) const hash2 = await blobStorage.write(data)
const hash3 = await blobStorage.write(data) const hash3 = await blobStorage.write(data)
const stats = blobStorage.getStats() expect(hash2).toBe(hash1)
expect(stats.dedupSavings).toBe(2000) // 2 duplicates × 1000 bytes expect(hash3).toBe(hash1)
const metadata = await blobStorage.getMetadata(hash1)
expect(metadata?.refCount).toBe(3)
}) })
}) })
@ -143,15 +147,12 @@ describe('BlobStorage', () => {
it('should compress large text data with zstd', async () => { it('should compress large text data with zstd', async () => {
const data = Buffer.from('a'.repeat(10000)) const data = Buffer.from('a'.repeat(10000))
const hash = await blobStorage.write(data, { const hash = await blobStorage.write(data, { compression: 'zstd' })
type: 'metadata',
compression: 'zstd'
})
const metadata = await blobStorage.getMetadata(hash) const metadata = await blobStorage.getMetadata(hash)
// zstd may not be available in test environment - falls back to 'none' // zstd may not be available in test environment - falls back to 'none'
// This is expected behavior (see BlobStorage initCompression fallback) // This is expected behavior (see BlobStorage.ensureCompressionReady fallback)
if (metadata?.compression === 'zstd') { if (metadata?.compression === 'zstd') {
expect(metadata.compressedSize).toBeLessThan(metadata.size) expect(metadata.compressedSize).toBeLessThan(metadata.size)
} else { } else {
@ -163,10 +164,7 @@ describe('BlobStorage', () => {
it('should decompress zstd data on read', async () => { it('should decompress zstd data on read', async () => {
const originalData = Buffer.from('test data '.repeat(100)) const originalData = Buffer.from('test data '.repeat(100))
const hash = await blobStorage.write(originalData, { const hash = await blobStorage.write(originalData, { compression: 'zstd' })
type: 'metadata',
compression: 'zstd'
})
const retrieved = await blobStorage.read(hash) const retrieved = await blobStorage.read(hash)
@ -176,36 +174,30 @@ describe('BlobStorage', () => {
it('should not compress small blobs', async () => { it('should not compress small blobs', async () => {
const data = Buffer.from('small') const data = Buffer.from('small')
const hash = await blobStorage.write(data, { const hash = await blobStorage.write(data, { compression: 'auto' })
compression: 'auto'
})
const metadata = await blobStorage.getMetadata(hash) const metadata = await blobStorage.getMetadata(hash)
expect(metadata?.compression).toBe('none') expect(metadata?.compression).toBe('none')
}) })
it('should not compress vector data (already dense)', async () => { it('should skip compression for already-compressed MIME types in auto mode', async () => {
const vectorData = Buffer.from(new Float32Array([1, 2, 3, 4, 5]))
const hash = await blobStorage.write(vectorData, {
type: 'vector',
compression: 'auto'
})
const metadata = await blobStorage.getMetadata(hash)
expect(metadata?.compression).toBe('none')
})
it('should auto-compress metadata/tree/commit types', async () => {
const data = Buffer.from('x'.repeat(5000)) const data = Buffer.from('x'.repeat(5000))
const hash = await blobStorage.write(data, { const hash = await blobStorage.write(data, {
type: 'metadata', compression: 'auto',
compression: 'auto' mimeType: 'image/jpeg'
}) })
const metadata = await blobStorage.getMetadata(hash)
expect(metadata?.compression).toBe('none')
})
it('should auto-compress large compressible payloads', async () => {
const data = Buffer.from('x'.repeat(5000))
const hash = await blobStorage.write(data, { compression: 'auto' })
const metadata = await blobStorage.getMetadata(hash) const metadata = await blobStorage.getMetadata(hash)
// Should compress if zstd is available // Should compress if zstd is available
@ -216,91 +208,37 @@ describe('BlobStorage', () => {
}) })
describe('LRU Caching', () => { describe('LRU Caching', () => {
it('should cache blob on read', async () => { it('should serve identical bytes from cache and from storage', async () => {
const data = Buffer.from('cached data') const data = Buffer.from('cached data')
const hash = await blobStorage.write(data) const hash = await blobStorage.write(data)
// First read (cache miss) // Warm read (write-through cache) and cold read (fresh instance)
await blobStorage.read(hash) const warm = await blobStorage.read(hash)
const cold = await new BlobStorage(adapter).read(hash)
// Second read (cache hit) expect(warm.equals(data)).toBe(true)
await blobStorage.read(hash) expect(cold.equals(data)).toBe(true)
const stats = blobStorage.getStats()
expect(stats.cacheHits).toBeGreaterThan(0)
}) })
it('should evict LRU entries when cache is full', async () => { it('should keep serving correct bytes when the cache evicts under pressure', async () => {
const smallCache = new BlobStorage(adapter, { const smallCache = new BlobStorage(adapter, { cacheMaxSize: 100 })
cacheMaxSize: 100, // Very small cache
enableCompression: false
})
// Write blobs that exceed cache size // Write blobs that exceed cache size (forces LRU eviction)
const blob1 = Buffer.from('x'.repeat(50)) const blobs = [
const blob2 = Buffer.from('y'.repeat(50)) Buffer.from('x'.repeat(50)),
const blob3 = Buffer.from('z'.repeat(50)) Buffer.from('y'.repeat(50)),
Buffer.from('z'.repeat(50))
const hash1 = await smallCache.write(blob1)
const hash2 = await smallCache.write(blob2)
const hash3 = await smallCache.write(blob3) // Should evict hash1
// Read all blobs
await smallCache.read(hash1)
await smallCache.read(hash2)
await smallCache.read(hash3)
// hash1 should have been evicted, causing cache miss
const stats = smallCache.getStats()
expect(stats.cacheMisses).toBeGreaterThan(0)
})
it('should clear cache on demand', async () => {
const data = Buffer.from('test')
const hash = await blobStorage.write(data)
await blobStorage.read(hash) // Cache it
blobStorage.clearCache()
await blobStorage.read(hash) // Should be cache miss
const stats = blobStorage.getStats()
expect(stats.cacheMisses).toBeGreaterThan(0)
})
})
describe('Batch Operations', () => {
it('should write multiple blobs in parallel', async () => {
const blobs: Array<[Buffer, any]> = [
[Buffer.from('blob1'), undefined],
[Buffer.from('blob2'), undefined],
[Buffer.from('blob3'), undefined]
] ]
const hashes: string[] = []
for (const blob of blobs) {
hashes.push(await smallCache.write(blob))
}
const hashes = await blobStorage.writeBatch(blobs) // Every blob still reads back correctly, evicted or not
for (let i = 0; i < blobs.length; i++) {
expect(hashes).toHaveLength(3) const retrieved = await smallCache.read(hashes[i])
expect(hashes[0]).toHaveLength(64) expect(retrieved.equals(blobs[i])).toBe(true)
expect(hashes[1]).toHaveLength(64) }
expect(hashes[2]).toHaveLength(64)
})
it('should read multiple blobs in parallel', async () => {
const data1 = Buffer.from('blob1')
const data2 = Buffer.from('blob2')
const data3 = Buffer.from('blob3')
const hash1 = await blobStorage.write(data1)
const hash2 = await blobStorage.write(data2)
const hash3 = await blobStorage.write(data3)
const blobs = await blobStorage.readBatch([hash1, hash2, hash3])
expect(blobs).toHaveLength(3)
expect(blobs[0].toString()).toBe('blob1')
expect(blobs[1].toString()).toBe('blob2')
expect(blobs[2].toString()).toBe('blob3')
}) })
}) })
@ -339,120 +277,28 @@ describe('BlobStorage', () => {
}) })
}) })
describe('Garbage Collection', () => {
it('should delete unreferenced blobs', async () => {
const blob1 = Buffer.from('referenced')
const blob2 = Buffer.from('unreferenced')
const hash1 = await blobStorage.write(blob1)
const hash2 = await blobStorage.write(blob2)
// Manually set refCount to 0 for unreferenced blob
// (In real COW usage, refCount tracks actual references from commits/trees)
// GC only deletes when refCount === 0 AND not in referenced set
const metadata2 = await blobStorage.getMetadata(hash2)
if (metadata2) {
metadata2.refCount = 0
await adapter.put(`blob-meta:${hash2}`, Buffer.from(JSON.stringify(metadata2)))
}
// Mark only hash1 as referenced
const referenced = new Set([hash1])
const deleted = await blobStorage.garbageCollect(referenced)
expect(deleted).toBeGreaterThan(0)
expect(await blobStorage.has(hash1)).toBe(true)
expect(await blobStorage.has(hash2)).toBe(false)
})
it('should not delete referenced blobs', async () => {
const blob1 = Buffer.from('ref1')
const blob2 = Buffer.from('ref2')
const hash1 = await blobStorage.write(blob1)
const hash2 = await blobStorage.write(blob2)
// Both referenced
const referenced = new Set([hash1, hash2])
const deleted = await blobStorage.garbageCollect(referenced)
expect(deleted).toBe(0)
expect(await blobStorage.has(hash1)).toBe(true)
expect(await blobStorage.has(hash2)).toBe(true)
})
})
describe('Metadata', () => { describe('Metadata', () => {
it('should store blob metadata', async () => { it('should store blob metadata', async () => {
const data = Buffer.from('test') const data = Buffer.from('test')
const hash = await blobStorage.write(data, { const hash = await blobStorage.write(data, { compression: 'none' })
type: 'metadata',
compression: 'none'
})
const metadata = await blobStorage.getMetadata(hash) const metadata = await blobStorage.getMetadata(hash)
expect(metadata?.hash).toBe(hash) expect(metadata?.hash).toBe(hash)
expect(metadata?.size).toBe(data.length) expect(metadata?.size).toBe(data.length)
expect(metadata?.type).toBe('metadata')
expect(metadata?.compression).toBe('none') expect(metadata?.compression).toBe('none')
expect(metadata?.createdAt).toBeGreaterThan(0) expect(metadata?.createdAt).toBeGreaterThan(0)
expect(metadata?.refCount).toBe(1) expect(metadata?.refCount).toBe(1)
}) })
})
describe('List Operations', () => { it('should return undefined metadata for unknown hashes', async () => {
it('should list all blobs', async () => { expect(await blobStorage.getMetadata('f'.repeat(64))).toBeUndefined()
const hash1 = await blobStorage.write(Buffer.from('blob1'))
const hash2 = await blobStorage.write(Buffer.from('blob2'))
const hash3 = await blobStorage.write(Buffer.from('blob3'))
const blobs = await blobStorage.listBlobs()
expect(blobs).toContain(hash1)
expect(blobs).toContain(hash2)
expect(blobs).toContain(hash3)
})
})
describe('Statistics', () => {
it('should track storage statistics', async () => {
const data1 = Buffer.from('x'.repeat(1000))
const data2 = Buffer.from('y'.repeat(2000))
await blobStorage.write(data1)
await blobStorage.write(data2)
const stats = blobStorage.getStats()
expect(stats.totalBlobs).toBe(2)
expect(stats.totalSize).toBe(3000)
expect(stats.avgBlobSize).toBe(1500)
})
it('should calculate compression ratio', async () => {
const data = Buffer.from('a'.repeat(10000))
await blobStorage.write(data, {
type: 'metadata',
compression: 'zstd'
})
const stats = blobStorage.getStats()
// Compression ratio should be > 1 if compressed
if (stats.compressedSize < stats.totalSize) {
expect(stats.compressionRatio).toBeGreaterThan(1)
}
}) })
}) })
describe('Error Handling', () => { describe('Error Handling', () => {
it('should throw on reading non-existent blob', async () => { it('should throw on reading non-existent blob', async () => {
// Use 'f' instead of '0' to avoid NULL_HASH sentinel value
await expect( await expect(
blobStorage.read('f'.repeat(64)) blobStorage.read('f'.repeat(64))
).rejects.toThrow('Blob metadata not found') ).rejects.toThrow('Blob metadata not found')
@ -462,14 +308,12 @@ describe('BlobStorage', () => {
const data = Buffer.from('test') const data = Buffer.from('test')
const hash = await blobStorage.write(data) const hash = await blobStorage.write(data)
// Clear cache so read() actually checks metadata // Delete metadata but keep blob bytes
blobStorage.clearCache()
// Delete metadata but keep blob
await adapter.delete(`blob-meta:${hash}`) await adapter.delete(`blob-meta:${hash}`)
// Use skipCache to ensure we check metadata // A fresh instance has a cold cache, so read() must consult metadata
await expect(blobStorage.read(hash, { skipCache: true })).rejects.toThrow('metadata not found') const coldStore = new BlobStorage(adapter)
await expect(coldStore.read(hash)).rejects.toThrow('metadata not found')
}) })
}) })
@ -510,10 +354,10 @@ describe('BlobStorage', () => {
}) })
}) })
describe('v5.10.0 Regression - Blob Integrity with Wrapped Data', () => { describe('Wrapped-binary regression (v5.10.0) - hash verification on unwrapped bytes', () => {
it('should handle wrapped binary data without hash mismatch (v5.10.0 fix)', async () => { it('should handle wrapped binary data without hash mismatch', async () => {
// Import TestWrappingAdapter that actually wraps data like production // Import TestWrappingAdapter that actually wraps data like production
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
const wrappingAdapter = new TestWrappingAdapter() const wrappingAdapter = new TestWrappingAdapter()
const testBlobStorage = new BlobStorage(wrappingAdapter) const testBlobStorage = new BlobStorage(wrappingAdapter)
@ -522,20 +366,17 @@ describe('BlobStorage', () => {
// Write blob (TestWrappingAdapter wraps as {_binary: true, data: "base64..."}) // Write blob (TestWrappingAdapter wraps as {_binary: true, data: "base64..."})
const hash = await testBlobStorage.write(originalData) const hash = await testBlobStorage.write(originalData)
// Clear cache to force re-read from storage (bypasses in-memory cache) // Fresh instance = cold cache: read() must fetch + unwrap from storage.
testBlobStorage.clearCache() // v5.10.0 bug: read() hashed the wrapped bytes instead of the unwrapped
// content; the fix runs unwrapBinaryData() before hash verification.
// v5.10.0 bug: This would fail with "Blob integrity check failed" const retrieved = await new BlobStorage(wrappingAdapter).read(hash)
// because BlobStorage.read() was hashing the wrapped data instead of unwrapped
// v5.10.1 fix: unwrapBinaryData() is called before hash verification
const retrieved = await testBlobStorage.read(hash)
expect(retrieved.equals(originalData)).toBe(true) expect(retrieved.equals(originalData)).toBe(true)
expect(retrieved.toString()).toBe('test content for v5.10.0 regression test') expect(retrieved.toString()).toBe('test content for v5.10.0 regression test')
}) })
it('should handle multiple wrapped blobs without integrity errors', async () => { it('should handle multiple wrapped blobs without integrity errors', async () => {
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
const wrappingAdapter = new TestWrappingAdapter() const wrappingAdapter = new TestWrappingAdapter()
const testBlobStorage = new BlobStorage(wrappingAdapter) const testBlobStorage = new BlobStorage(wrappingAdapter)
@ -554,19 +395,17 @@ describe('BlobStorage', () => {
hashes.push(hash) hashes.push(hash)
} }
// Clear cache // Cold-cache instance forces storage reads
testBlobStorage.clearCache() const coldStore = new BlobStorage(wrappingAdapter)
// Read all blobs (would fail in v5.10.0)
for (let i = 0; i < hashes.length; i++) { for (let i = 0; i < hashes.length; i++) {
const retrieved = await testBlobStorage.read(hashes[i]) const retrieved = await coldStore.read(hashes[i])
expect(retrieved.equals(testData[i])).toBe(true) expect(retrieved.equals(testData[i])).toBe(true)
} }
}) })
it('should work even if adapter fails to unwrap (defense-in-depth)', async () => { it('should work even if adapter fails to unwrap (defense-in-depth)', async () => {
// Create a buggy adapter that doesn't unwrap properly // Create a buggy adapter that doesn't unwrap properly
class BuggyAdapter implements COWStorageAdapter { class BuggyAdapter implements BlobStoreAdapter {
private storage = new Map<string, any>() private storage = new Map<string, any>()
async get(key: string): Promise<any | undefined> { async get(key: string): Promise<any | undefined> {
@ -601,21 +440,20 @@ describe('BlobStorage', () => {
const originalData = Buffer.from('test defense-in-depth') const originalData = Buffer.from('test defense-in-depth')
const hash = await testBlobStorage.write(originalData) const hash = await testBlobStorage.write(originalData)
testBlobStorage.clearCache() // Even with buggy adapter (and a cold cache), read() should unwrap and
// verify correctly
// Even with buggy adapter, BlobStorage.read() should unwrap and verify correctly const retrieved = await new BlobStorage(buggyAdapter).read(hash)
const retrieved = await testBlobStorage.read(hash)
expect(retrieved.equals(originalData)).toBe(true) expect(retrieved.equals(originalData)).toBe(true)
}) })
}) })
describe('v6.2.0 Regression - Key-Based Dispatch (Permanent Fix)', () => { describe('Key-based dispatch regression (v6.2.0)', () => {
it('should handle JSON-like compressed data without integrity failures', async () => { it('should handle JSON-like compressed data without integrity failures', async () => {
// THE KILLER TEST CASE: Data that looks like JSON when compressed // THE KILLER TEST CASE: Data that looks like JSON when compressed
// This would fail with v5.10.1 wrapBinaryData() guessing approach // This would fail with v5.10.1 wrapBinaryData() guessing approach
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
const wrappingAdapter = new TestWrappingAdapter() const wrappingAdapter = new TestWrappingAdapter()
const testBlobStorage = new BlobStorage(wrappingAdapter, { enableCompression: true }) const testBlobStorage = new BlobStorage(wrappingAdapter)
// Create JSON data that will be compressed // Create JSON data that will be compressed
const jsonData = { nested: { key: 'value', array: [1, 2, 3] }, repeated: 'x'.repeat(1000) } const jsonData = { nested: { key: 'value', array: [1, 2, 3] }, repeated: 'x'.repeat(1000) }
@ -624,54 +462,23 @@ describe('BlobStorage', () => {
// Write blob (compression happens, might create JSON-parseable bytes) // Write blob (compression happens, might create JSON-parseable bytes)
const hash = await testBlobStorage.write(originalData) const hash = await testBlobStorage.write(originalData)
// Clear cache to force re-read from storage
testBlobStorage.clearCache()
// v5.10.1 bug: If compressed bytes accidentally parse as JSON, // v5.10.1 bug: If compressed bytes accidentally parse as JSON,
// wrapBinaryData() would store parsed object instead of wrapped binary // wrapBinaryData() would store parsed object instead of wrapped binary
// On read: JSON.stringify(object) !== original compressed bytes → hash mismatch // On read: JSON.stringify(object) !== original compressed bytes → hash mismatch
// //
// v6.2.0 fix: Key-based dispatch eliminates guessing // v6.2.0 fix: Key-based dispatch eliminates guessing
// 'blob:hash' → Always wrapped as binary, never parsed // 'blob:hash' → Always wrapped as binary, never parsed
const retrieved = await testBlobStorage.read(hash) const retrieved = await new BlobStorage(wrappingAdapter).read(hash)
// Hash MUST match // Hash MUST match
expect(BlobStorage.hash(retrieved)).toBe(hash) expect(BlobStorage.hash(retrieved)).toBe(hash)
expect(retrieved.equals(originalData)).toBe(true) expect(retrieved.equals(originalData)).toBe(true)
}) })
it('should correctly dispatch all key types', async () => {
// Verify key-based dispatch works for all COW key patterns
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js')
const wrappingAdapter = new TestWrappingAdapter()
const testBlobStorage = new BlobStorage(wrappingAdapter)
// Test binary blob keys
const blobData = Buffer.from('binary blob content')
const blobHash = await testBlobStorage.write(blobData, { type: 'blob' })
testBlobStorage.clearCache()
const retrievedBlob = await testBlobStorage.read(blobHash)
expect(retrievedBlob.equals(blobData)).toBe(true)
// Test commit keys
const commitData = Buffer.from('commit content')
const commitHash = await testBlobStorage.write(commitData, { type: 'commit' })
testBlobStorage.clearCache()
const retrievedCommit = await testBlobStorage.read(commitHash)
expect(retrievedCommit.equals(commitData)).toBe(true)
// Test tree keys
const treeData = Buffer.from('tree content')
const treeHash = await testBlobStorage.write(treeData, { type: 'tree' })
testBlobStorage.clearCache()
const retrievedTree = await testBlobStorage.read(treeHash)
expect(retrievedTree.equals(treeData)).toBe(true)
})
it('should handle metadata keys correctly', async () => { it('should handle metadata keys correctly', async () => {
// Verify that key-based dispatch correctly handles metadata keys // Verify that key-based dispatch correctly handles metadata keys
// This test verifies the dispatch logic, not the full read/write cycle // This test verifies the dispatch logic, not the full read/write cycle
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
const wrappingAdapter = new TestWrappingAdapter() const wrappingAdapter = new TestWrappingAdapter()
// Test metadata key dispatch: should parse as JSON // Test metadata key dispatch: should parse as JSON
@ -688,10 +495,10 @@ describe('BlobStorage', () => {
expect(retrieved.size).toBe(42) expect(retrieved.size).toBe(42)
}) })
it('should never call wrapBinaryData on write path', async () => { it('should never parse blob bytes as JSON on the write path', async () => {
// Verify that baseStorage COW adapter uses key-based dispatch, // Verify that the blob bridge uses key-based dispatch,
// NOT wrapBinaryData() guessing // NOT wrapBinaryData() guessing
const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js')
const wrappingAdapter = new TestWrappingAdapter() const wrappingAdapter = new TestWrappingAdapter()
const testBlobStorage = new BlobStorage(wrappingAdapter) const testBlobStorage = new BlobStorage(wrappingAdapter)
@ -699,8 +506,7 @@ describe('BlobStorage', () => {
const problematicData = Buffer.from('[{"looks":"like","valid":"json"}]') const problematicData = Buffer.from('[{"looks":"like","valid":"json"}]')
const hash = await testBlobStorage.write(problematicData) const hash = await testBlobStorage.write(problematicData)
testBlobStorage.clearCache() const retrieved = await new BlobStorage(wrappingAdapter).read(hash)
const retrieved = await testBlobStorage.read(hash)
// Should retrieve exact same bytes (no JSON parsing occurred) // Should retrieve exact same bytes (no JSON parsing occurred)
expect(retrieved.equals(problematicData)).toBe(true) expect(retrieved.equals(problematicData)).toBe(true)

538
tests/unit/versioning/VersionDiff.test.ts
View file

@ -1,538 +0,0 @@
/**
* VersionDiff Unit Tests (v5.3.0)
*
* Tests deep object comparison:
* - Added fields
* - Removed fields
* - Modified fields
* - Type changes
* - Nested object comparison
* - Array comparison
*/
import { describe, it, expect } from 'vitest'
import { compareEntityVersions } from '../../../src/versioning/VersionDiff.js'
import type { NounMetadata } from '../../../src/coreTypes.js'
describe('VersionDiff', () => {
describe('compareEntityVersions()', () => {
it('should detect added fields', () => {
const from: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: { email: 'alice@example.com' }
}
const to: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
email: 'alice@example.com',
city: 'NYC',
age: 30
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'user-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.added.length).toBe(2)
expect(diff.added.some(c => c.path.includes('city'))).toBe(true)
expect(diff.added.some(c => c.path.includes('age'))).toBe(true)
expect(diff.added.find(c => c.path.includes('city'))?.newValue).toBe('NYC')
})
it('should detect removed fields', () => {
const from: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
email: 'alice@example.com',
city: 'NYC',
age: 30
}
}
const to: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: { email: 'alice@example.com' }
}
const diff = compareEntityVersions(from, to, {
entityId: 'user-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.removed.length).toBe(2)
expect(diff.removed.some(c => c.path.includes('city'))).toBe(true)
expect(diff.removed.some(c => c.path.includes('age'))).toBe(true)
expect(diff.removed.find(c => c.path.includes('city'))?.oldValue).toBe('NYC')
})
it('should detect modified fields', () => {
const from: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
email: 'alice@example.com',
status: 'active'
}
}
const to: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice Smith',
metadata: {
email: 'alice.smith@example.com',
status: 'active'
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'user-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.modified.length).toBe(2)
const nameChange = diff.modified.find(c => c.path.includes('name'))
expect(nameChange?.oldValue).toBe('Alice')
expect(nameChange?.newValue).toBe('Alice Smith')
const emailChange = diff.modified.find(c => c.path.includes('email'))
expect(emailChange?.oldValue).toBe('alice@example.com')
expect(emailChange?.newValue).toBe('alice.smith@example.com')
})
it('should detect type changes', () => {
const from: NounMetadata = {
id: 'config-1',
type: 'thing',
name: 'Config',
metadata: { value: '100' }
}
const to: NounMetadata = {
id: 'config-1',
type: 'thing',
name: 'Config',
metadata: { value: 100 }
}
const diff = compareEntityVersions(from, to, {
entityId: 'config-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.typeChanged.length).toBe(1)
const typeChange = diff.typeChanged[0]
expect(typeChange.path).toContain('value')
expect(typeChange.oldType).toBe('string')
expect(typeChange.newType).toBe('number')
expect(typeChange.oldValue).toBe('100')
expect(typeChange.newValue).toBe(100)
})
it('should detect identical versions', () => {
const entity: NounMetadata = {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Unchanged' }
}
const diff = compareEntityVersions(entity, entity, {
entityId: 'doc-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.identical).toBe(true)
expect(diff.totalChanges).toBe(0)
expect(diff.added).toHaveLength(0)
expect(diff.removed).toHaveLength(0)
expect(diff.modified).toHaveLength(0)
})
it('should handle nested object changes', () => {
const from: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
address: {
street: '123 Main St',
city: 'NYC'
}
}
}
const to: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
address: {
street: '456 Oak Ave',
city: 'NYC',
zip: '10001'
}
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'user-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.modified.some(c => c.path.includes('address.street'))).toBe(true)
expect(diff.added.some(c => c.path.includes('address.zip'))).toBe(true)
const streetChange = diff.modified.find(c => c.path.includes('address.street'))
expect(streetChange?.oldValue).toBe('123 Main St')
expect(streetChange?.newValue).toBe('456 Oak Ave')
})
it('should handle array changes', () => {
const from: NounMetadata = {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: {
tags: ['a', 'b', 'c']
}
}
const to: NounMetadata = {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: {
tags: ['a', 'b', 'd']
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'doc-1',
fromVersion: 1,
toVersion: 2
})
// Array element change detected as modification
expect(diff.totalChanges).toBeGreaterThan(0)
})
it('should handle null and undefined', () => {
const from: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {
a: null,
b: undefined,
c: 'value'
}
}
const to: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {
a: 'now-value',
b: null,
c: 'value'
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'test-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.totalChanges).toBeGreaterThan(0)
})
it('should ignore specified fields', () => {
const from: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
email: 'alice@example.com',
lastModified: 1000,
internal: 'ignore-me'
}
}
const to: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice Smith',
metadata: {
email: 'alice@example.com',
lastModified: 2000,
internal: 'changed'
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'user-1',
fromVersion: 1,
toVersion: 2,
ignoreFields: ['lastModified', 'internal']
})
// Should only detect name change
expect(diff.totalChanges).toBe(1)
expect(diff.modified.some(c => c.path.includes('name'))).toBe(true)
expect(diff.modified.some(c => c.path.includes('lastModified'))).toBe(false)
expect(diff.modified.some(c => c.path.includes('internal'))).toBe(false)
})
it('should respect maxDepth option', () => {
const from: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {
level1: {
level2: {
level3: {
level4: 'deep-value'
}
}
}
}
}
const to: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {
level1: {
level2: {
level3: {
level4: 'changed-value'
}
}
}
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'test-1',
fromVersion: 1,
toVersion: 2,
maxDepth: 2 // Stop at level2
})
// Should detect change at metadata.level1.level2 level
expect(diff.totalChanges).toBeGreaterThan(0)
})
it('should handle empty objects', () => {
const from: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
}
const to: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: { value: 123 }
}
const diff = compareEntityVersions(from, to, {
entityId: 'test-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.added.length).toBe(1)
expect(diff.added[0].newValue).toBe(123)
})
it('should calculate total changes correctly', () => {
const from: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {
email: 'alice@example.com',
status: 'active',
age: 30
}
}
const to: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice Smith',
metadata: {
email: 'alice.smith@example.com',
city: 'NYC'
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'user-1',
fromVersion: 1,
toVersion: 2
})
const expectedTotal =
diff.added.length +
diff.removed.length +
diff.modified.length +
diff.typeChanged.length
expect(diff.totalChanges).toBe(expectedTotal)
})
it('should handle complex real-world changes', () => {
const from: NounMetadata = {
id: 'project-1',
type: 'thing',
name: 'My Project',
metadata: {
description: 'Old description',
status: 'draft',
team: ['alice', 'bob'],
config: {
theme: 'light',
notifications: true
},
createdAt: 1000
}
}
const to: NounMetadata = {
id: 'project-1',
type: 'thing',
name: 'My Awesome Project',
metadata: {
description: 'New description',
status: 'active',
team: ['alice', 'bob', 'charlie'],
config: {
theme: 'dark',
notifications: true,
language: 'en'
},
createdAt: 1000,
updatedAt: 2000
}
}
const diff = compareEntityVersions(from, to, {
entityId: 'project-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.totalChanges).toBeGreaterThan(0)
expect(diff.modified.length).toBeGreaterThan(0)
expect(diff.added.length).toBeGreaterThan(0)
const nameChange = diff.modified.find(c => c.path.includes('name'))
expect(nameChange?.newValue).toBe('My Awesome Project')
})
it('should handle boolean changes', () => {
const from: NounMetadata = {
id: 'feature-1',
type: 'thing',
name: 'Feature',
metadata: { enabled: false }
}
const to: NounMetadata = {
id: 'feature-1',
type: 'thing',
name: 'Feature',
metadata: { enabled: true }
}
const diff = compareEntityVersions(from, to, {
entityId: 'feature-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.modified.length).toBe(1)
expect(diff.modified[0].oldValue).toBe(false)
expect(diff.modified[0].newValue).toBe(true)
})
it('should handle number changes', () => {
const from: NounMetadata = {
id: 'counter-1',
type: 'thing',
name: 'Counter',
metadata: { count: 10 }
}
const to: NounMetadata = {
id: 'counter-1',
type: 'thing',
name: 'Counter',
metadata: { count: 20 }
}
const diff = compareEntityVersions(from, to, {
entityId: 'counter-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.modified.length).toBe(1)
expect(diff.modified[0].oldValue).toBe(10)
expect(diff.modified[0].newValue).toBe(20)
})
it('should handle date/timestamp changes', () => {
const from: NounMetadata = {
id: 'event-1',
type: 'thing',
name: 'Event',
metadata: { timestamp: 1000 }
}
const to: NounMetadata = {
id: 'event-1',
type: 'thing',
name: 'Event',
metadata: { timestamp: 2000 }
}
const diff = compareEntityVersions(from, to, {
entityId: 'event-1',
fromVersion: 1,
toVersion: 2
})
expect(diff.modified.length).toBe(1)
expect(diff.modified[0].path).toContain('timestamp')
})
})
})

710
tests/unit/versioning/VersionManager.test.ts
View file

@ -1,710 +0,0 @@
/**
* VersionManager Unit Tests (v5.3.0)
*
* Tests the core versioning engine in isolation:
* - Save versions
* - Restore versions
* - List versions
* - Compare versions
* - Prune versions
* - Content deduplication
* - Branch awareness
*/
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { VersionManager } from '../../../src/versioning/VersionManager.js'
import type { NounMetadata } from '../../../src/coreTypes.js'
describe('VersionManager', () => {
let manager: VersionManager
let mockBrain: any
let mockStorage: Map<string, any>
let mockIndex: Map<string, any>
beforeEach(() => {
// Mock storage
mockStorage = new Map()
mockIndex = new Map()
// Mock Brainy instance
mockBrain = {
currentBranch: 'main',
storageAdapter: {
// v6.3.0: VersionStorage now uses saveMetadata/getMetadata for key-value storage
saveMetadata: vi.fn(async (key: string, data: any) => {
mockStorage.set(key, JSON.stringify(data))
}),
getMetadata: vi.fn(async (key: string) => {
const data = mockStorage.get(key)
if (!data) return null
return JSON.parse(data)
}),
// Legacy methods for compatibility
exists: vi.fn(async (path: string) => mockStorage.has(path)),
writeFile: vi.fn(async (path: string, data: string) => {
mockStorage.set(path, data)
}),
readFile: vi.fn(async (path: string) => {
const data = mockStorage.get(path)
if (!data) throw new Error('File not found')
return data
}),
deleteFile: vi.fn(async (path: string) => {
mockStorage.delete(path)
})
},
refManager: {
getRef: vi.fn(async (branch: string) => ({
name: `refs/heads/${branch}`,
commitHash: 'commit-hash-123',
type: 'branch'
})),
setRef: vi.fn(async () => {}),
resolveRef: vi.fn(async () => 'commit-hash-123')
},
getNounMetadata: vi.fn(async (id: string) => {
const entity = mockIndex.get(id)
// Return null for version entities (_version:...) that don't exist
// Throw for regular entities that don't exist
if (!entity) {
if (id.startsWith('_version:')) {
return null // Version doesn't exist - let VersionIndex handle it
}
throw new Error(`Entity ${id} not found`)
}
return entity
}),
saveNounMetadata: vi.fn(async (id: string, data: NounMetadata) => {
mockIndex.set(id, data)
}),
deleteNounMetadata: vi.fn(async (id: string) => {
mockIndex.delete(id)
}),
searchByMetadata: vi.fn(async (query: any) => {
const results: any[] = []
for (const [id, entity] of mockIndex.entries()) {
let matches = true
for (const [key, value] of Object.entries(query)) {
// Handle top-level properties (like 'type')
if (key === 'type') {
if (entity.type !== value) {
matches = false
break
}
continue // Skip metadata check for 'type'
}
// Check metadata properties with glob pattern support
const entityValue = entity.metadata?.[key]
// Support simple glob patterns (e.g., 'v1.*' matches 'v1.0.0', 'v1.1.0')
if (typeof value === 'string' && value.includes('*')) {
const pattern = new RegExp('^' + value.replace(/\*/g, '.*') + '$')
if (!entityValue || !pattern.test(String(entityValue))) {
matches = false
break
}
} else if (entityValue !== value) {
matches = false
break
}
}
if (matches) {
results.push({ id, ...entity })
}
}
return results
}),
commit: vi.fn(async () => 'commit-hash-123'),
// v6.3.0: VersionManager.restore() uses brain.update() to refresh all indexes
update: vi.fn(async (opts: { id: string, data?: any, type?: string, metadata?: any, confidence?: number, weight?: number, merge?: boolean }) => {
// Simulate update by merging metadata into the entity
const existing = mockIndex.get(opts.id)
if (!existing) {
throw new Error(`Entity ${opts.id} not found for update`)
}
const updated = {
...existing,
data: opts.data !== undefined ? opts.data : existing.data,
type: opts.type || existing.type,
confidence: opts.confidence !== undefined ? opts.confidence : existing.confidence,
weight: opts.weight !== undefined ? opts.weight : existing.weight,
...(opts.merge === false ? opts.metadata : { ...existing.metadata, ...opts.metadata })
}
mockIndex.set(opts.id, updated)
})
}
manager = new VersionManager(mockBrain)
})
describe('save()', () => {
it('should save a new version', async () => {
// Add test entity
mockIndex.set('user-123', {
id: 'user-123',
type: 'user',
name: 'Alice',
metadata: { email: 'alice@example.com' }
})
const version = await manager.save('user-123', {
tag: 'v1.0',
description: 'Initial version'
})
expect(version.version).toBe(1)
expect(version.entityId).toBe('user-123')
expect(version.branch).toBe('main')
expect(version.tag).toBe('v1.0')
expect(version.description).toBe('Initial version')
expect(version.contentHash).toBeDefined()
expect(version.commitHash).toBe('commit-hash-123')
})
it('should increment version numbers', async () => {
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Version 1' }
})
const v1 = await manager.save('doc-1', { tag: 'v1' })
expect(v1.version).toBe(1)
// Update entity
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Version 2' }
})
const v2 = await manager.save('doc-1', { tag: 'v2' })
expect(v2.version).toBe(2)
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Version 3' }
})
const v3 = await manager.save('doc-1', { tag: 'v3' })
expect(v3.version).toBe(3)
})
it('should deduplicate identical content', async () => {
mockIndex.set('note-1', {
id: 'note-1',
type: 'document',
name: 'Note',
metadata: { content: 'Unchanged' }
})
const v1 = await manager.save('note-1', { tag: 'v1' })
const v2 = await manager.save('note-1', { tag: 'v2' })
// Should return same version (content unchanged)
expect(v2.version).toBe(v1.version)
expect(v2.contentHash).toBe(v1.contentHash)
})
it('should throw if entity does not exist', async () => {
await expect(
manager.save('nonexistent', { tag: 'v1' })
).rejects.toThrow('Entity nonexistent not found')
})
it('should save without tag or description', async () => {
mockIndex.set('test-1', {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
})
const version = await manager.save('test-1')
expect(version.version).toBe(1)
expect(version.tag).toBeUndefined()
expect(version.description).toBeUndefined()
})
})
describe('restore()', () => {
it('should restore entity to specific version', async () => {
// Save version 1
mockIndex.set('config-1', {
id: 'config-1',
type: 'thing',
name: 'Config',
metadata: { theme: 'light', version: 1 }
})
await manager.save('config-1', { tag: 'v1' })
// Update to version 2
mockIndex.set('config-1', {
id: 'config-1',
type: 'thing',
name: 'Config',
metadata: { theme: 'dark', version: 2 }
})
await manager.save('config-1', { tag: 'v2' })
// Restore to v1
await manager.restore('config-1', 1)
// v6.3.0: restore() uses brain.update() to refresh all indexes
expect(mockBrain.update).toHaveBeenCalledWith(
expect.objectContaining({
id: 'config-1',
merge: false // Replace, don't merge
})
)
})
it('should restore by tag', async () => {
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { status: 'alpha' }
})
await manager.save('app-1', { tag: 'alpha' })
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { status: 'beta' }
})
await manager.save('app-1', { tag: 'beta' })
// Restore to alpha
await manager.restore('app-1', 'alpha')
// v6.3.0: restore() uses brain.update() to refresh all indexes
expect(mockBrain.update).toHaveBeenCalledWith(
expect.objectContaining({
id: 'app-1',
merge: false // Replace, don't merge
})
)
})
it('should throw if version not found', async () => {
mockIndex.set('test-1', {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
})
await manager.save('test-1')
await expect(
manager.restore('test-1', 999)
).rejects.toThrow('Version 999 not found')
})
})
describe('list()', () => {
it('should list all versions for entity', async () => {
mockIndex.set('log-1', {
id: 'log-1',
type: 'document',
name: 'Log',
metadata: { entry: 1 }
})
await manager.save('log-1', { tag: 'v1' })
mockIndex.set('log-1', {
id: 'log-1',
type: 'document',
name: 'Log',
metadata: { entry: 2 }
})
await manager.save('log-1', { tag: 'v2' })
mockIndex.set('log-1', {
id: 'log-1',
type: 'document',
name: 'Log',
metadata: { entry: 3 }
})
await manager.save('log-1', { tag: 'v3' })
const versions = await manager.list('log-1')
expect(versions).toHaveLength(3)
expect(versions[0].version).toBe(3) // Newest first
expect(versions[1].version).toBe(2)
expect(versions[2].version).toBe(1)
})
it('should filter by exact tag', async () => {
// v6.3.0: Tag filtering is exact match only (no glob patterns)
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { v: 1 }
})
await manager.save('app-1', { tag: 'v1.0.0' })
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { v: 2 }
})
await manager.save('app-1', { tag: 'v1.1.0' })
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { v: 3 }
})
await manager.save('app-1', { tag: 'v2.0.0' })
// v6.3.0: Exact tag match only
const v1Versions = await manager.list('app-1', { tag: 'v1.0.0' })
expect(v1Versions).toHaveLength(1)
expect(v1Versions[0].tag).toBe('v1.0.0')
// Get all versions
const allVersions = await manager.list('app-1')
expect(allVersions).toHaveLength(3)
})
it('should limit results', async () => {
mockIndex.set('data-1', {
id: 'data-1',
type: 'thing',
name: 'Data',
metadata: { v: 1 }
})
for (let i = 1; i <= 5; i++) {
mockIndex.set('data-1', {
id: 'data-1',
type: 'thing',
name: 'Data',
metadata: { v: i }
})
await manager.save('data-1', { tag: `v${i}` })
}
const versions = await manager.list('data-1', { limit: 3 })
expect(versions).toHaveLength(3)
expect(versions[0].version).toBe(5)
expect(versions[1].version).toBe(4)
expect(versions[2].version).toBe(3)
})
it('should return empty array if no versions', async () => {
mockIndex.set('new-1', {
id: 'new-1',
type: 'thing',
name: 'New',
metadata: {}
})
const versions = await manager.list('new-1')
expect(versions).toHaveLength(0)
})
})
describe('compare()', () => {
it('should compare two versions', async () => {
mockIndex.set('user-1', {
id: 'user-1',
type: 'user',
name: 'Bob',
metadata: { email: 'bob@example.com', age: 30 }
})
await manager.save('user-1', { tag: 'v1' })
mockIndex.set('user-1', {
id: 'user-1',
type: 'user',
name: 'Robert',
metadata: { email: 'robert@example.com', age: 30, city: 'NYC' }
})
await manager.save('user-1', { tag: 'v2' })
const diff = await manager.compare('user-1', 1, 2)
expect(diff.totalChanges).toBeGreaterThan(0)
expect(diff.modified.length).toBeGreaterThan(0)
expect(diff.added.length).toBeGreaterThan(0)
const nameChange = diff.modified.find(c => c.path.includes('name'))
expect(nameChange?.oldValue).toBe('Bob')
expect(nameChange?.newValue).toBe('Robert')
const cityAdd = diff.added.find(c => c.path.includes('city'))
expect(cityAdd?.newValue).toBe('NYC')
})
it('should compare by tags', async () => {
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Alpha' }
})
await manager.save('doc-1', { tag: 'alpha' })
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Beta' }
})
await manager.save('doc-1', { tag: 'beta' })
const diff = await manager.compare('doc-1', 'alpha', 'beta')
expect(diff.modified.length).toBeGreaterThan(0)
const contentChange = diff.modified.find(c => c.path.includes('content'))
expect(contentChange?.oldValue).toBe('Alpha')
expect(contentChange?.newValue).toBe('Beta')
})
it('should detect identical versions', async () => {
mockIndex.set('same-1', {
id: 'same-1',
type: 'thing',
name: 'Same',
metadata: { value: 100 }
})
await manager.save('same-1', { tag: 'v1' })
await manager.save('same-1', { tag: 'v2' }) // Content unchanged
const diff = await manager.compare('same-1', 1, 1)
expect(diff.identical).toBe(true)
expect(diff.totalChanges).toBe(0)
})
})
describe('prune()', () => {
it('should prune old versions keeping recent', async () => {
mockIndex.set('log-1', {
id: 'log-1',
type: 'document',
name: 'Log',
metadata: { entry: 1 }
})
for (let i = 1; i <= 10; i++) {
mockIndex.set('log-1', {
id: 'log-1',
type: 'document',
name: 'Log',
metadata: { entry: i }
})
await manager.save('log-1', { tag: `v${i}` })
}
const result = await manager.prune('log-1', {
keepRecent: 5,
keepTagged: false
})
expect(result.deleted).toBe(5)
expect(result.kept).toBe(5)
const remaining = await manager.list('log-1')
expect(remaining).toHaveLength(5)
expect(remaining[0].version).toBe(10) // Most recent
})
it('should keep tagged versions', async () => {
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { v: 1 }
})
await manager.save('app-1', { tag: 'release' })
for (let i = 2; i <= 10; i++) {
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { v: i }
})
await manager.save('app-1') // No tag
}
const result = await manager.prune('app-1', {
keepRecent: 3,
keepTagged: true
})
const remaining = await manager.list('app-1')
const releaseVersion = remaining.find(v => v.tag === 'release')
expect(releaseVersion).toBeDefined()
})
it('should respect keepAfter timestamp', async () => {
mockIndex.set('data-1', {
id: 'data-1',
type: 'thing',
name: 'Data',
metadata: { v: 1 }
})
const now = Date.now()
const oneDayAgo = now - 24 * 60 * 60 * 1000
await manager.save('data-1', { tag: 'old' })
// Simulate newer versions
for (let i = 2; i <= 5; i++) {
mockIndex.set('data-1', {
id: 'data-1',
type: 'thing',
name: 'Data',
metadata: { v: i }
})
await manager.save('data-1', { tag: `new${i}` })
}
const result = await manager.prune('data-1', {
keepAfter: oneDayAgo,
keepTagged: false
})
expect(result.deleted).toBeGreaterThanOrEqual(0)
})
})
describe('getVersion()', () => {
it('should get specific version by number', async () => {
mockIndex.set('test-1', {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: { v: 1 }
})
const v1 = await manager.save('test-1', { tag: 'v1' })
mockIndex.set('test-1', {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: { v: 2 }
})
await manager.save('test-1', { tag: 'v2' })
const version = await manager.getVersion('test-1', 1)
expect(version).toBeDefined()
expect(version?.version).toBe(1)
expect(version?.tag).toBe('v1')
})
it('should return null if version not found', async () => {
mockIndex.set('test-1', {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
})
await manager.save('test-1')
const version = await manager.getVersion('test-1', 999)
expect(version).toBeNull()
})
})
describe('getVersionByTag()', () => {
it('should get version by tag', async () => {
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { status: 'alpha' }
})
await manager.save('app-1', { tag: 'alpha' })
mockIndex.set('app-1', {
id: 'app-1',
type: 'thing',
name: 'App',
metadata: { status: 'beta' }
})
await manager.save('app-1', { tag: 'beta' })
const betaVersion = await manager.getVersionByTag('app-1', 'beta')
expect(betaVersion).toBeDefined()
expect(betaVersion?.tag).toBe('beta')
})
it('should return null if tag not found', async () => {
mockIndex.set('test-1', {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
})
await manager.save('test-1', { tag: 'v1' })
const version = await manager.getVersionByTag('test-1', 'nonexistent')
expect(version).toBeNull()
})
})
describe('getVersionCount()', () => {
it('should count versions', async () => {
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { v: 1 }
})
expect(await manager.getVersionCount('doc-1')).toBe(0)
await manager.save('doc-1')
expect(await manager.getVersionCount('doc-1')).toBe(1)
mockIndex.set('doc-1', {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { v: 2 }
})
await manager.save('doc-1')
expect(await manager.getVersionCount('doc-1')).toBe(2)
})
})
describe('Branch Awareness', () => {
it('should track versions per branch', async () => {
mockIndex.set('branch-test', {
id: 'branch-test',
type: 'thing',
name: 'Test',
metadata: {}
})
// Save on main
mockBrain.currentBranch = 'main'
const mainV1 = await manager.save('branch-test', { tag: 'main-v1' })
expect(mainV1.branch).toBe('main')
// Save on feature
mockBrain.currentBranch = 'feature'
const featureV1 = await manager.save('branch-test', { tag: 'feature-v1' })
expect(featureV1.branch).toBe('feature')
// Versions should be independent
expect(mainV1.version).toBe(1)
expect(featureV1.version).toBe(1) // Each branch starts at 1
})
})
})

567
tests/unit/versioning/VersionStorage.test.ts
View file

@ -1,567 +0,0 @@
/**
* VersionStorage Unit Tests (v5.3.0)
*
* Tests content-addressable storage:
* - SHA-256 content hashing
* - Content deduplication
* - Storage adapter integration
* - Save/load/delete operations
*/
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { VersionStorage } from '../../../src/versioning/VersionStorage.js'
import type { NounMetadata } from '../../../src/coreTypes.js'
import type { EntityVersion } from '../../../src/versioning/VersionManager.js'
describe('VersionStorage', () => {
let storage: VersionStorage
let mockBrain: any
let mockFiles: Map<string, string>
beforeEach(() => {
mockFiles = new Map()
mockBrain = {
storageAdapter: {
// v6.3.0: VersionStorage now uses saveMetadata/getMetadata instead of writeFile/readFile
saveMetadata: vi.fn(async (key: string, data: any) => {
mockFiles.set(key, JSON.stringify(data))
}),
getMetadata: vi.fn(async (key: string) => {
const data = mockFiles.get(key)
if (!data) return null
return JSON.parse(data)
}),
// Legacy methods for tests that still use them
exists: vi.fn(async (path: string) => mockFiles.has(path)),
writeFile: vi.fn(async (path: string, data: string) => {
mockFiles.set(path, data)
}),
readFile: vi.fn(async (path: string) => {
const data = mockFiles.get(path)
if (!data) throw new Error('File not found')
return data
}),
deleteFile: vi.fn(async (path: string) => {
mockFiles.delete(path)
})
}
}
storage = new VersionStorage(mockBrain)
})
describe('hashEntity()', () => {
it('should generate consistent SHA-256 hashes', () => {
const entity: NounMetadata = {
id: 'user-123',
type: 'user',
name: 'Alice',
metadata: { email: 'alice@example.com' }
}
const hash1 = storage.hashEntity(entity)
const hash2 = storage.hashEntity(entity)
expect(hash1).toBe(hash2)
expect(hash1).toHaveLength(64) // SHA-256 = 64 hex chars
})
it('should generate different hashes for different content', () => {
const entity1: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: {}
}
const entity2: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Bob',
metadata: {}
}
const hash1 = storage.hashEntity(entity1)
const hash2 = storage.hashEntity(entity2)
expect(hash1).not.toBe(hash2)
})
it('should ignore property order (stable JSON)', () => {
const entity1: NounMetadata = {
id: 'user-1',
type: 'user',
name: 'Alice',
metadata: { a: 1, b: 2, c: 3 }
}
const entity2: NounMetadata = {
type: 'user',
metadata: { c: 3, b: 2, a: 1 },
name: 'Alice',
id: 'user-1'
}
const hash1 = storage.hashEntity(entity1)
const hash2 = storage.hashEntity(entity2)
expect(hash1).toBe(hash2)
})
it('should handle nested objects', () => {
const entity: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {
nested: {
deep: {
value: 'hello'
}
}
}
}
const hash = storage.hashEntity(entity)
expect(hash).toHaveLength(64)
})
it('should handle arrays', () => {
const entity: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {
tags: ['a', 'b', 'c']
}
}
const hash = storage.hashEntity(entity)
expect(hash).toHaveLength(64)
})
it('should handle null and undefined', () => {
const entity1: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: { value: null }
}
const entity2: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: { value: undefined }
}
const hash1 = storage.hashEntity(entity1)
const hash2 = storage.hashEntity(entity2)
expect(hash1).not.toBe(hash2)
})
})
describe('saveVersion()', () => {
it('should save version content to storage', async () => {
const entity: NounMetadata = {
id: 'user-123',
type: 'user',
name: 'Alice',
metadata: { email: 'alice@example.com' }
}
const version: EntityVersion = {
version: 1,
entityId: 'user-123',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
await storage.saveVersion(version, entity)
// v6.3.0: Uses __system_version_ prefix for saveMetadata keys
const expectedKey = `__system_version_user-123_${version.contentHash}`
expect(mockFiles.has(expectedKey)).toBe(true)
const savedData = mockFiles.get(expectedKey)
expect(savedData).toBeDefined()
expect(JSON.parse(savedData!)).toEqual(entity)
})
it('should deduplicate identical content', async () => {
const entity: NounMetadata = {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Same content' }
}
const contentHash = storage.hashEntity(entity)
const version1: EntityVersion = {
version: 1,
entityId: 'doc-1',
branch: 'main',
commitHash: 'commit-1',
timestamp: Date.now(),
contentHash
}
const version2: EntityVersion = {
version: 2,
entityId: 'doc-1',
branch: 'main',
commitHash: 'commit-2',
timestamp: Date.now() + 1000,
contentHash // Same hash!
}
await storage.saveVersion(version1, entity)
const writeCallCount1 = mockBrain.storageAdapter.writeFile.mock.calls.length
await storage.saveVersion(version2, entity)
const writeCallCount2 = mockBrain.storageAdapter.writeFile.mock.calls.length
// Should not write again (deduplication)
expect(writeCallCount2).toBe(writeCallCount1)
})
it('should store different content separately', async () => {
const entity1: NounMetadata = {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Version 1' }
}
const entity2: NounMetadata = {
id: 'doc-1',
type: 'document',
name: 'Doc',
metadata: { content: 'Version 2' }
}
const version1: EntityVersion = {
version: 1,
entityId: 'doc-1',
branch: 'main',
commitHash: 'commit-1',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity1)
}
const version2: EntityVersion = {
version: 2,
entityId: 'doc-1',
branch: 'main',
commitHash: 'commit-2',
timestamp: Date.now() + 1000,
contentHash: storage.hashEntity(entity2)
}
await storage.saveVersion(version1, entity1)
await storage.saveVersion(version2, entity2)
// v6.3.0: Uses __system_version_ prefix for saveMetadata keys
const key1 = `__system_version_doc-1_${version1.contentHash}`
const key2 = `__system_version_doc-1_${version2.contentHash}`
expect(mockFiles.has(key1)).toBe(true)
expect(mockFiles.has(key2)).toBe(true)
expect(key1).not.toBe(key2)
})
})
describe('loadVersion()', () => {
it('should load version content from storage', async () => {
const entity: NounMetadata = {
id: 'user-123',
type: 'user',
name: 'Alice',
metadata: { email: 'alice@example.com' }
}
const version: EntityVersion = {
version: 1,
entityId: 'user-123',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
// Save first
await storage.saveVersion(version, entity)
// Load
const loaded = await storage.loadVersion(version)
expect(loaded).toEqual(entity)
})
it('should return null if version not found', async () => {
const version: EntityVersion = {
version: 1,
entityId: 'nonexistent',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: 'fake-hash'
}
const loaded = await storage.loadVersion(version)
expect(loaded).toBeNull()
})
it('should handle complex nested data', async () => {
const entity: NounMetadata = {
id: 'complex-1',
type: 'thing',
name: 'Complex',
metadata: {
nested: {
array: [1, 2, 3],
object: { key: 'value' }
},
tags: ['a', 'b', 'c']
}
}
const version: EntityVersion = {
version: 1,
entityId: 'complex-1',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
await storage.saveVersion(version, entity)
const loaded = await storage.loadVersion(version)
expect(loaded).toEqual(entity)
})
})
describe('deleteVersion()', () => {
it('should handle version deletion (content-addressed, no-op)', async () => {
// v6.3.0: Version content is content-addressed and may be shared by multiple versions.
// The deleteVersion method is a no-op - it doesn't actually delete the content.
// Version INDEX is deleted via VersionIndex.removeVersion().
// A GC process could clean up unreferenced content in the future.
const entity: NounMetadata = {
id: 'user-123',
type: 'user',
name: 'Alice',
metadata: {}
}
const version: EntityVersion = {
version: 1,
entityId: 'user-123',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
// Save
await storage.saveVersion(version, entity)
const key = `__system_version_user-123_${version.contentHash}`
expect(mockFiles.has(key)).toBe(true)
// Delete is a no-op for content (content-addressed, may be shared)
await storage.deleteVersion(version)
// Content is NOT deleted to avoid breaking other versions with same content hash
expect(mockFiles.has(key)).toBe(true) // v6.3.0: Content preserved
})
it('should handle deleting non-existent version gracefully', async () => {
const version: EntityVersion = {
version: 1,
entityId: 'nonexistent',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: 'fake-hash'
}
// Should not throw
await storage.deleteVersion(version)
})
})
describe('Storage Adapter Integration', () => {
it('should work with memory storage adapter', async () => {
const entity: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: { value: 123 }
}
const version: EntityVersion = {
version: 1,
entityId: 'test-1',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
await storage.saveVersion(version, entity)
const loaded = await storage.loadVersion(version)
expect(loaded).toEqual(entity)
})
it('should use adapter.saveMetadata API', async () => {
// v6.3.0: VersionStorage now uses saveMetadata/getMetadata exclusively
const testStorage = new Map<string, string>()
mockBrain.storageAdapter = {
saveMetadata: vi.fn(async (key: string, data: any) => {
testStorage.set(key, JSON.stringify(data))
}),
getMetadata: vi.fn(async (key: string) => {
const data = testStorage.get(key)
if (!data) return null
return JSON.parse(data)
})
}
storage = new VersionStorage(mockBrain)
const entity: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
}
const version: EntityVersion = {
version: 1,
entityId: 'test-1',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
await storage.saveVersion(version, entity)
expect(mockBrain.storageAdapter.saveMetadata).toHaveBeenCalled()
const loaded = await storage.loadVersion(version)
expect(mockBrain.storageAdapter.getMetadata).toHaveBeenCalled()
expect(loaded).toEqual(entity)
})
it('should throw if storage adapter missing', async () => {
mockBrain.storageAdapter = null
storage = new VersionStorage(mockBrain)
const entity: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
}
const version: EntityVersion = {
version: 1,
entityId: 'test-1',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
await expect(
storage.saveVersion(version, entity)
).rejects.toThrow('Storage adapter not available')
})
it('should throw if adapter does not support required operations', async () => {
mockBrain.storageAdapter = {} // No methods (no saveMetadata)
storage = new VersionStorage(mockBrain)
const entity: NounMetadata = {
id: 'test-1',
type: 'thing',
name: 'Test',
metadata: {}
}
const version: EntityVersion = {
version: 1,
entityId: 'test-1',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash: storage.hashEntity(entity)
}
// v6.3.0: Error from calling undefined saveMetadata
await expect(
storage.saveVersion(version, entity)
).rejects.toThrow() // TypeError: adapter.saveMetadata is not a function
})
})
describe('Key Generation', () => {
it('should generate correct storage keys', async () => {
const entity: NounMetadata = {
id: 'user-123',
type: 'user',
name: 'Alice',
metadata: {}
}
const contentHash = storage.hashEntity(entity)
const version: EntityVersion = {
version: 1,
entityId: 'user-123',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash
}
await storage.saveVersion(version, entity)
// v6.3.0: Uses __system_version_ prefix for saveMetadata keys
const expectedKey = `__system_version_user-123_${contentHash}`
expect(mockFiles.has(expectedKey)).toBe(true)
})
it('should handle entity IDs with special characters', async () => {
const entity: NounMetadata = {
id: 'user:123:profile',
type: 'user',
name: 'Alice',
metadata: {}
}
const contentHash = storage.hashEntity(entity)
const version: EntityVersion = {
version: 1,
entityId: 'user:123:profile',
branch: 'main',
commitHash: 'commit-123',
timestamp: Date.now(),
contentHash
}
await storage.saveVersion(version, entity)
// v6.3.0: Uses __system_version_ prefix for saveMetadata keys
const expectedKey = `__system_version_user:123:profile_${contentHash}`
expect(mockFiles.has(expectedKey)).toBe(true)
})
})
})