soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: ID-first storage architecture + remove memory-unsafe APIs (v6.0.0)

Browse source 
BREAKING CHANGES:

**ID-First Storage Paths**
- Direct O(1) entity access without type lookups
- Before: entities/nouns/{TYPE}/metadata/{SHARD}/{ID}.json
- After:  entities/nouns/{SHARD}/{ID}/metadata.json
- Migration handled automatically on first init()

**Removed Memory-Unsafe APIs**
- Removed brain.merge() - loaded all entities into memory
- Removed brain.diff() - loaded all entities into memory
- Removed brain.data().backup() - loaded all entities into memory
- Removed brain.data().restore() - depended on backup()
- Removed CLI commands: backup, restore, cow merge

**Migration Paths**
- merge() → Use checkout() or manually copy entities with pagination
- diff() → Use asOf() with manual paginated comparison
- backup() → Use fork() for instant COW snapshots
- restore() → Use checkout() to switch to snapshot branch

Core Improvements:
- ✅ All 8 storage adapters properly call super.init()
- ✅ GraphAdjacencyIndex integration in BaseStorage.init()
- ✅ Fixed ID-first path bugs (vector.json → vectors.json)
- ✅ Fixed MemoryStorage.initializeCounts() for ID-first paths
- ✅ New VFS APIs: du(), access(), find()
- ✅ Comprehensive documentation with migration guides

Storage Adapters Fixed:
- MemoryStorage, FileSystemStorage, AzureBlobStorage
- GCSStorage, R2Storage, S3CompatibleStorage
- OPFSStorage, HistoricalStorageAdapter

Files Changed: 28 files, +1,075/-1,933 lines (net -858)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-11-19 16:46:11 -08:00
parent 9730ca41e5
commit 42ae5be455
28 changed files with 1079 additions and 1937 deletions
Download patch file Download diff file Expand all files Collapse all files

455
src/brainy.ts
View file

@ -2924,445 +2924,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
/**
* Merge a source branch into target branch
* @param sourceBranch - Branch to merge from
* @param targetBranch - Branch to merge into
* @param options - Merge options (strategy, author, onConflict)
* @returns Merge result with statistics
*
* @example
* ```typescript
* const result = await brain.merge('experiment', 'main', {
* strategy: 'last-write-wins',
* author: 'dev@example.com'
* })
* console.log(result) // { added: 5, modified: 3, deleted: 1, conflicts: 0 }
* ```
*/
async merge(
sourceBranch: string,
targetBranch: string,
options?: {
strategy?: 'last-write-wins' | 'first-write-wins' | 'custom'
author?: string
onConflict?: (entityA: any, entityB: any) => Promise<any>
}
): Promise<{
added: number
modified: number
deleted: number
conflicts: number
}> {
await this.ensureInitialized()
return this.augmentationRegistry.execute(
'merge',
{ sourceBranch, targetBranch, options },
async () => {
if (!('refManager' in this.storage) || !('blobStorage' in this.storage)) {
throw new Error('Merge requires COW-enabled storage (v5.0.0+)')
}
const strategy = options?.strategy || 'last-write-wins'
let added = 0
let modified = 0
let deleted = 0
let conflicts = 0
// Verify both branches exist
const branches = await this.listBranches()
if (!branches.includes(sourceBranch)) {
throw new Error(`Source branch '${sourceBranch}' does not exist`)
}
if (!branches.includes(targetBranch)) {
throw new Error(`Target branch '${targetBranch}' does not exist`)
}
// 1. Create temporary fork of source branch to read from
const sourceFork = await this.fork(`${sourceBranch}-merge-temp-${Date.now()}`)
await sourceFork.checkout(sourceBranch)
// 2. Save current branch and checkout target
const currentBranch = await this.getCurrentBranch()
if (currentBranch !== targetBranch) {
await this.checkout(targetBranch)
}
try {
// 3. Get all entities from source and target
const sourceResults = await sourceFork.find({})
const targetResults = await this.find({})
// Create maps for faster lookup
const targetMap = new Map(targetResults.map(r => [r.entity.id, r.entity]))
// 4. Merge entities
for (const sourceResult of sourceResults) {
const sourceEntity = sourceResult.entity
const targetEntity = targetMap.get(sourceEntity.id)
if (!targetEntity) {
// NEW entity in source - ADD to target
await this.add({
id: sourceEntity.id,
type: sourceEntity.type,
data: sourceEntity.data,
vector: sourceEntity.vector
})
added++
} else {
// Entity exists in both branches - check for conflicts
const sourceTime = sourceEntity.updatedAt || sourceEntity.createdAt || 0
const targetTime = targetEntity.updatedAt || targetEntity.createdAt || 0
// If timestamps are identical, no change needed
if (sourceTime === targetTime) {
continue
}
// Apply merge strategy
if (strategy === 'last-write-wins') {
if (sourceTime > targetTime) {
// Source is newer, update target
await this.update({ id: sourceEntity.id, data: sourceEntity.data })
modified++
}
// else target is newer, keep target
} else if (strategy === 'first-write-wins') {
if (sourceTime < targetTime) {
// Source is older, update target
await this.update({ id: sourceEntity.id, data: sourceEntity.data })
modified++
}
} else if (strategy === 'custom' && options?.onConflict) {
// Custom conflict resolution
const resolved = await options.onConflict(targetEntity, sourceEntity)
await this.update({ id: sourceEntity.id, data: resolved.data })
modified++
conflicts++
} else {
// Conflict detected but no resolution strategy
conflicts++
}
}
}
// 5. Merge relationships (verbs)
const sourceVerbsResult = await sourceFork.storage.getVerbs({})
const targetVerbsResult = await this.storage.getVerbs({})
const sourceVerbs = sourceVerbsResult.items || []
const targetVerbs = targetVerbsResult.items || []
// Create set of existing target relationships for deduplication
const targetRelSet = new Set(
targetVerbs.map((v: any) => `${v.sourceId}-${v.verb}-${v.targetId}`)
)
// Add relationships that don't exist in target
for (const sourceVerb of sourceVerbs) {
const key = `${sourceVerb.sourceId}-${sourceVerb.verb}-${sourceVerb.targetId}`
if (!targetRelSet.has(key)) {
// Only add if both entities exist in target
const hasSource = targetMap.has(sourceVerb.sourceId)
const hasTarget = targetMap.has(sourceVerb.targetId)
if (hasSource && hasTarget) {
await this.relate({
from: sourceVerb.sourceId,
to: sourceVerb.targetId,
type: sourceVerb.verb as any,
weight: sourceVerb.weight,
metadata: sourceVerb.metadata as any
})
}
}
}
// 6. Create merge commit
if ('commitLog' in this.storage) {
await this.commit({
message: `Merge ${sourceBranch} into ${targetBranch}`,
author: options?.author || 'system',
metadata: {
mergeType: 'branch',
source: sourceBranch,
target: targetBranch,
strategy,
stats: { added, modified, deleted, conflicts }
}
})
}
} finally {
// 7. Clean up temporary fork (just delete the temp branch)
try {
const tempBranchName = `${sourceBranch}-merge-temp-${Date.now()}`
const branches = await this.listBranches()
if (branches.includes(tempBranchName)) {
await this.deleteBranch(tempBranchName)
}
} catch (err) {
// Ignore cleanup errors
}
// Restore original branch if needed
if (currentBranch !== targetBranch) {
await this.checkout(currentBranch)
}
}
return { added, modified, deleted, conflicts }
}
)
}
/**
* Compare differences between two branches (like git diff)
* @param sourceBranch - Branch to compare from (defaults to current branch)
* @param targetBranch - Branch to compare to (defaults to 'main')
* @returns Diff result showing added, modified, and deleted entities/relationships
*
* @example
* ```typescript
* // Compare current branch with main
* const diff = await brain.diff()
*
* // Compare two specific branches
* const diff = await brain.diff('experiment', 'main')
* console.log(diff)
* // {
* // entities: { added: 5, modified: 3, deleted: 1 },
* // relationships: { added: 10, modified: 2, deleted: 0 }
* // }
* ```
*/
async diff(
sourceBranch?: string,
targetBranch?: string
): Promise<{
entities: {
added: Array<{ id: string; type: string; data?: any }>
modified: Array<{ id: string; type: string; changes: string[] }>
deleted: Array<{ id: string; type: string }>
}
relationships: {
added: Array<{ from: string; to: string; type: string }>
modified: Array<{ from: string; to: string; type: string; changes: string[] }>
deleted: Array<{ from: string; to: string; type: string }>
}
summary: {
entitiesAdded: number
entitiesModified: number
entitiesDeleted: number
relationshipsAdded: number
relationshipsModified: number
relationshipsDeleted: number
}
}> {
await this.ensureInitialized()
return this.augmentationRegistry.execute(
'diff',
{ sourceBranch, targetBranch },
async () => {
// Default branches
const source = sourceBranch || (await this.getCurrentBranch())
const target = targetBranch || 'main'
const currentBranch = await this.getCurrentBranch()
// If source is current branch, use this instance directly (no fork needed)
let sourceFork: Brainy<T>
let sourceForkCreated = false
if (source === currentBranch) {
sourceFork = this
} else {
sourceFork = await this.fork(`temp-diff-source-${Date.now()}`)
sourceForkCreated = true
try {
await sourceFork.checkout(source)
} catch (err) {
// If checkout fails, branch may not exist - just use current state
}
}
// If target is current branch, use this instance directly (no fork needed)
let targetFork: Brainy<T>
let targetForkCreated = false
if (target === currentBranch) {
targetFork = this
} else {
targetFork = await this.fork(`temp-diff-target-${Date.now()}`)
targetForkCreated = true
try {
await targetFork.checkout(target)
} catch (err) {
// If checkout fails, branch may not exist - just use current state
}
}
try {
// Get all entities from both branches
const sourceResults = await sourceFork.find({})
const targetResults = await targetFork.find({})
// Create maps for lookup
const sourceMap = new Map(sourceResults.map(r => [r.entity.id, r.entity]))
const targetMap = new Map(targetResults.map(r => [r.entity.id, r.entity]))
// Track differences
const entitiesAdded: any[] = []
const entitiesModified: any[] = []
const entitiesDeleted: any[] = []
// Find added and modified entities
for (const [id, sourceEntity] of sourceMap.entries()) {
const targetEntity = targetMap.get(id)
if (!targetEntity) {
// Entity exists in source but not target = ADDED
entitiesAdded.push({
id: sourceEntity.id,
type: sourceEntity.type,
data: sourceEntity.data
})
} else {
// Entity exists in both - check for modifications
const changes: string[] = []
if (sourceEntity.data !== targetEntity.data) {
changes.push('data')
}
if ((sourceEntity.updatedAt || 0) !== (targetEntity.updatedAt || 0)) {
changes.push('updatedAt')
}
if (changes.length > 0) {
entitiesModified.push({
id: sourceEntity.id,
type: sourceEntity.type,
changes
})
}
}
}
// Find deleted entities (in target but not in source)
for (const [id, targetEntity] of targetMap.entries()) {
if (!sourceMap.has(id)) {
entitiesDeleted.push({
id: targetEntity.id,
type: targetEntity.type
})
}
}
// Compare relationships
const sourceVerbsResult = await sourceFork.storage.getVerbs({})
const targetVerbsResult = await targetFork.storage.getVerbs({})
const sourceVerbs = sourceVerbsResult.items || []
const targetVerbs = targetVerbsResult.items || []
const sourceRelMap = new Map(
sourceVerbs.map((v: any) => [`${v.sourceId}-${v.verb}-${v.targetId}`, v])
)
const targetRelMap = new Map(
targetVerbs.map((v: any) => [`${v.sourceId}-${v.verb}-${v.targetId}`, v])
)
const relationshipsAdded: any[] = []
const relationshipsModified: any[] = []
const relationshipsDeleted: any[] = []
// Find added and modified relationships
for (const [key, sourceVerb] of sourceRelMap.entries()) {
const targetVerb = targetRelMap.get(key)
if (!targetVerb) {
// Relationship exists in source but not target = ADDED
relationshipsAdded.push({
from: sourceVerb.sourceId,
to: sourceVerb.targetId,
type: sourceVerb.verb
})
} else {
// Relationship exists in both - check for modifications
const changes: string[] = []
if ((sourceVerb.weight || 0) !== (targetVerb.weight || 0)) {
changes.push('weight')
}
if (JSON.stringify(sourceVerb.metadata) !== JSON.stringify(targetVerb.metadata)) {
changes.push('metadata')
}
if (changes.length > 0) {
relationshipsModified.push({
from: sourceVerb.sourceId,
to: sourceVerb.targetId,
type: sourceVerb.verb,
changes
})
}
}
}
// Find deleted relationships
for (const [key, targetVerb] of targetRelMap.entries()) {
if (!sourceRelMap.has(key)) {
relationshipsDeleted.push({
from: targetVerb.sourceId,
to: targetVerb.targetId,
type: targetVerb.verb
})
}
}
return {
entities: {
added: entitiesAdded,
modified: entitiesModified,
deleted: entitiesDeleted
},
relationships: {
added: relationshipsAdded,
modified: relationshipsModified,
deleted: relationshipsDeleted
},
summary: {
entitiesAdded: entitiesAdded.length,
entitiesModified: entitiesModified.length,
entitiesDeleted: entitiesDeleted.length,
relationshipsAdded: relationshipsAdded.length,
relationshipsModified: relationshipsModified.length,
relationshipsDeleted: relationshipsDeleted.length
}
}
} finally {
// Clean up temporary forks (only if we created them)
try {
const branches = await this.listBranches()
if (sourceForkCreated && sourceFork !== this) {
const sourceBranchName = await sourceFork.getCurrentBranch()
if (branches.includes(sourceBranchName)) {
await this.deleteBranch(sourceBranchName)
}
}
if (targetForkCreated && targetFork !== this) {
const targetBranchName = await targetFork.getCurrentBranch()
if (branches.includes(targetBranchName)) {
await this.deleteBranch(targetBranchName)
}
}
} catch (err) {
// Ignore cleanup errors
}
}
}
)
}
/**
* Delete a branch/fork
* @param branch - Branch name to delete
@ -4107,22 +3668,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.metadataIndex.getFieldsForType(nounType)
}
/**
* Get comprehensive type-field affinity statistics
* Useful for understanding data patterns and NLP optimization
*/
async getTypeFieldAffinityStats(): Promise<{
totalTypes: number
averageFieldsPerType: number
typeBreakdown: Record<string, {
totalEntities: number
uniqueFields: number
topFields: Array<{field: string; affinity: number}>
}>
}> {
await this.ensureInitialized()
return this.metadataIndex.getTypeFieldAffinityStats()
}
/**
* Create a streaming pipeline