feat: ID-first storage architecture + remove memory-unsafe APIs (v6.0.0)
BREAKING CHANGES:
**ID-First Storage Paths**
- Direct O(1) entity access without type lookups
- Before: entities/nouns/{TYPE}/metadata/{SHARD}/{ID}.json
- After: entities/nouns/{SHARD}/{ID}/metadata.json
- Migration handled automatically on first init()
**Removed Memory-Unsafe APIs**
- Removed brain.merge() - loaded all entities into memory
- Removed brain.diff() - loaded all entities into memory
- Removed brain.data().backup() - loaded all entities into memory
- Removed brain.data().restore() - depended on backup()
- Removed CLI commands: backup, restore, cow merge
**Migration Paths**
- merge() → Use checkout() or manually copy entities with pagination
- diff() → Use asOf() with manual paginated comparison
- backup() → Use fork() for instant COW snapshots
- restore() → Use checkout() to switch to snapshot branch
Core Improvements:
- ✅ All 8 storage adapters properly call super.init()
- ✅ GraphAdjacencyIndex integration in BaseStorage.init()
- ✅ Fixed ID-first path bugs (vector.json → vectors.json)
- ✅ Fixed MemoryStorage.initializeCounts() for ID-first paths
- ✅ New VFS APIs: du(), access(), find()
- ✅ Comprehensive documentation with migration guides
Storage Adapters Fixed:
- MemoryStorage, FileSystemStorage, AzureBlobStorage
- GCSStorage, R2Storage, S3CompatibleStorage
- OPFSStorage, HistoricalStorageAdapter
Files Changed: 28 files, +1,075/-1,933 lines (net -858)
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
9730ca41e5
commit
42ae5be455
28 changed files with 1079 additions and 1937 deletions
|
|
@ -81,296 +81,6 @@ export class DataAPI {
|
|||
this.brain = brain
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a backup of all data
|
||||
*/
|
||||
async backup(options: BackupOptions = {}): Promise<BackupData | { compressed: boolean; data: string; originalSize: number; compressedSize: number }> {
|
||||
const {
|
||||
includeVectors = true,
|
||||
compress = false,
|
||||
format = 'json'
|
||||
} = options
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
// Get all entities
|
||||
const nounsResult = await this.storage.getNouns({
|
||||
pagination: { limit: 1000000 }
|
||||
})
|
||||
const entities: BackupData['entities'] = []
|
||||
|
||||
for (const noun of nounsResult.items) {
|
||||
const entity = {
|
||||
id: noun.id,
|
||||
vector: includeVectors ? noun.vector : undefined,
|
||||
type: noun.type || NounType.Thing, // v4.8.0: type at top-level
|
||||
metadata: noun.metadata,
|
||||
service: noun.service // v4.8.0: service at top-level
|
||||
}
|
||||
entities.push(entity)
|
||||
}
|
||||
|
||||
// Get all relations
|
||||
const verbsResult = await this.storage.getVerbs({
|
||||
pagination: { limit: 1000000 }
|
||||
})
|
||||
const relations: BackupData['relations'] = []
|
||||
|
||||
for (const verb of verbsResult.items) {
|
||||
relations.push({
|
||||
id: verb.id,
|
||||
from: verb.sourceId,
|
||||
to: verb.targetId,
|
||||
type: verb.verb as string,
|
||||
weight: verb.weight || 1.0, // v4.8.0: weight at top-level
|
||||
metadata: verb.metadata
|
||||
})
|
||||
}
|
||||
|
||||
// Create backup data
|
||||
const backupData: BackupData = {
|
||||
version: '3.0.0',
|
||||
timestamp: Date.now(),
|
||||
entities,
|
||||
relations,
|
||||
stats: {
|
||||
entityCount: entities.length,
|
||||
relationCount: relations.length,
|
||||
vectorDimensions: entities[0]?.vector?.length
|
||||
}
|
||||
}
|
||||
|
||||
// Compress if requested
|
||||
if (compress) {
|
||||
// Import zlib for compression
|
||||
const { gzipSync } = await import('node:zlib')
|
||||
const jsonString = JSON.stringify(backupData)
|
||||
const compressed = gzipSync(Buffer.from(jsonString))
|
||||
|
||||
return {
|
||||
compressed: true,
|
||||
data: compressed.toString('base64'),
|
||||
originalSize: jsonString.length,
|
||||
compressedSize: compressed.length
|
||||
}
|
||||
}
|
||||
|
||||
return backupData
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore data from a backup
|
||||
*
|
||||
* v4.11.1: CRITICAL FIX - Now uses brain.addMany() and brain.relateMany()
|
||||
* Previous implementation only wrote to storage cache without updating indexes,
|
||||
* causing complete data loss on restart. This fix ensures:
|
||||
* - All 5 indexes updated (HNSW, metadata, adjacency, sparse, type-aware)
|
||||
* - Proper persistence to disk/cloud storage
|
||||
* - Storage-aware batching for optimal performance
|
||||
* - Atomic writes to prevent corruption
|
||||
* - Data survives instance restart
|
||||
*/
|
||||
async restore(params: {
|
||||
backup: BackupData
|
||||
merge?: boolean
|
||||
overwrite?: boolean
|
||||
validate?: boolean
|
||||
onProgress?: (completed: number, total: number) => void
|
||||
}): Promise<{
|
||||
entitiesRestored: number
|
||||
relationshipsRestored: number
|
||||
relationshipsSkipped: number
|
||||
errors: Array<{ type: 'entity' | 'relation'; id: string; error: string }>
|
||||
}> {
|
||||
const { backup, merge = false, overwrite = false, validate = true, onProgress } = params
|
||||
|
||||
const result = {
|
||||
entitiesRestored: 0,
|
||||
relationshipsRestored: 0,
|
||||
relationshipsSkipped: 0,
|
||||
errors: [] as Array<{ type: 'entity' | 'relation'; id: string; error: string }>
|
||||
}
|
||||
|
||||
// Validate backup format
|
||||
if (validate) {
|
||||
if (!backup.version || !backup.entities || !backup.relations) {
|
||||
throw new Error('Invalid backup format: missing version, entities, or relations')
|
||||
}
|
||||
}
|
||||
|
||||
// Validate brain instance is available (required for v4.11.1+ restore)
|
||||
if (!this.brain) {
|
||||
throw new Error(
|
||||
'Restore requires brain instance. DataAPI must be initialized with brain reference. ' +
|
||||
'Use: await brain.data() instead of constructing DataAPI directly.'
|
||||
)
|
||||
}
|
||||
|
||||
// Clear existing data if not merging
|
||||
if (!merge && overwrite) {
|
||||
await this.clear({ entities: true, relations: true })
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Phase 1: Restore entities using addMany()
|
||||
// v4.11.1: Uses proper persistence path through brain.addMany()
|
||||
// ============================================
|
||||
|
||||
// Prepare entity parameters for addMany()
|
||||
const entityParams = backup.entities
|
||||
.filter(entity => {
|
||||
// Skip existing entities when merging without overwrite
|
||||
if (merge && !overwrite) {
|
||||
// Note: We'll rely on addMany's internal duplicate handling
|
||||
// rather than checking each entity individually (performance)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
})
|
||||
.map(entity => {
|
||||
// Extract data field from metadata (backup format compatibility)
|
||||
// Backup stores the original data in metadata.data
|
||||
const data = entity.metadata?.data || entity.id
|
||||
|
||||
return {
|
||||
id: entity.id,
|
||||
data, // Required field for brainy.add()
|
||||
type: entity.type,
|
||||
metadata: entity.metadata || {},
|
||||
vector: entity.vector, // Preserve original vectors from backup
|
||||
service: entity.service,
|
||||
// Preserve confidence and weight if available
|
||||
confidence: entity.metadata?.confidence,
|
||||
weight: entity.metadata?.weight
|
||||
}
|
||||
})
|
||||
|
||||
// Restore entities in batches using storage-aware batching (v4.11.0)
|
||||
// v4.11.1: Track successful entity IDs to prevent orphaned relationships
|
||||
const successfulEntityIds = new Set<string>()
|
||||
|
||||
if (entityParams.length > 0) {
|
||||
try {
|
||||
const addResult = await this.brain.addMany({
|
||||
items: entityParams,
|
||||
continueOnError: true,
|
||||
onProgress: (done: number, total: number) => {
|
||||
onProgress?.(done, backup.entities.length + backup.relations.length)
|
||||
}
|
||||
})
|
||||
|
||||
result.entitiesRestored = addResult.successful.length
|
||||
|
||||
// Build Set of successfully restored entity IDs (v4.11.1 Bug Fix)
|
||||
addResult.successful.forEach((entityId: string) => {
|
||||
successfulEntityIds.add(entityId)
|
||||
})
|
||||
|
||||
// Track errors
|
||||
addResult.failed.forEach((failure: any) => {
|
||||
result.errors.push({
|
||||
type: 'entity',
|
||||
id: failure.item?.id || 'unknown',
|
||||
error: failure.error || 'Unknown error'
|
||||
})
|
||||
})
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to restore entities: ${(error as Error).message}`)
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Phase 2: Restore relationships using relateMany()
|
||||
// v4.11.1: CRITICAL FIX - Filter orphaned relationships
|
||||
// ============================================
|
||||
|
||||
// Prepare relationship parameters - filter out orphaned references
|
||||
const relationParams = backup.relations
|
||||
.filter(relation => {
|
||||
// Skip existing relations when merging without overwrite
|
||||
if (merge && !overwrite) {
|
||||
// Note: We'll rely on relateMany's internal duplicate handling
|
||||
return true
|
||||
}
|
||||
|
||||
// v4.11.1 CRITICAL BUG FIX: Skip relationships where source or target entity failed to restore
|
||||
// This prevents creating orphaned references that cause "Entity not found" errors
|
||||
const sourceExists = successfulEntityIds.has(relation.from)
|
||||
const targetExists = successfulEntityIds.has(relation.to)
|
||||
|
||||
if (!sourceExists || !targetExists) {
|
||||
// Track skipped relationship
|
||||
result.relationshipsSkipped++
|
||||
|
||||
// Optionally log for debugging (can be removed in production)
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
console.debug(
|
||||
`Skipping orphaned relationship: ${relation.from} -> ${relation.to} ` +
|
||||
`(${!sourceExists ? 'missing source' : 'missing target'})`
|
||||
)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
.map(relation => ({
|
||||
from: relation.from,
|
||||
to: relation.to,
|
||||
type: relation.type as VerbType,
|
||||
metadata: relation.metadata || {},
|
||||
weight: relation.weight || 1.0
|
||||
// Note: relation.id is ignored - brain.relate() generates new IDs
|
||||
// This is intentional to avoid ID conflicts
|
||||
}))
|
||||
|
||||
// Restore relationships in batches using storage-aware batching (v4.11.0)
|
||||
if (relationParams.length > 0) {
|
||||
try {
|
||||
const relateResult = await this.brain.relateMany({
|
||||
items: relationParams,
|
||||
continueOnError: true
|
||||
})
|
||||
|
||||
result.relationshipsRestored = relateResult.successful.length
|
||||
|
||||
// Track errors
|
||||
relateResult.failed.forEach((failure: any) => {
|
||||
result.errors.push({
|
||||
type: 'relation',
|
||||
id: failure.item?.from + '->' + failure.item?.to || 'unknown',
|
||||
error: failure.error || 'Unknown error'
|
||||
})
|
||||
})
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to restore relationships: ${(error as Error).message}`)
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Phase 3: Verify restoration succeeded
|
||||
// ============================================
|
||||
|
||||
// Sample verification: Check that first entity is actually retrievable
|
||||
if (backup.entities.length > 0 && result.entitiesRestored > 0) {
|
||||
const firstEntityId = backup.entities[0].id
|
||||
const verified = await this.brain.get(firstEntityId)
|
||||
|
||||
if (!verified) {
|
||||
console.warn(
|
||||
`⚠️ Restore completed but verification failed - entity ${firstEntityId} not retrievable. ` +
|
||||
`This may indicate a persistence issue with the storage adapter.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear data
|
||||
*/
|
||||
async clear(params: {
|
||||
entities?: boolean
|
||||
relations?: boolean
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue