feat: comprehensive metadata namespace architecture and cleanup system

BREAKING CHANGE: Remove hard delete option from deleteVerb() for consistent API

- Add complete metadata namespace architecture with O(1) soft delete performance
- Implement periodic cleanup system for old soft-deleted items
- Add restore methods for both nouns and verbs
- Require metadata contracts for all augmentations
- Eliminate namespace collisions with clean separation (_brainy, _augmentations, _audit)
- Optimize index performance using flattened dot-notation for O(1) lookups
- Add comprehensive augmentation safety system with type-safe access control
- Maintain full backward compatibility for existing data
- Add enterprise-grade cleanup with configurable age thresholds and batch processing

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-08-27 15:38:48 -07:00
parent 9220ca6748
commit 7e243b6f2b
35 changed files with 2700 additions and 125 deletions

View file

@ -0,0 +1,410 @@
/**
* Augmentation Metadata Contract System
*
* Prevents accidental metadata corruption while allowing intentional enrichment
* Each augmentation declares its metadata intentions upfront
*/
export interface AugmentationMetadataContract {
// Augmentation identity
name: string
version: string
// What fields this augmentation READS
reads?: {
userFields?: string[] // e.g., ['title', 'description']
internalFields?: string[] // e.g., ['_brainy.deleted']
augmentationFields?: string[] // e.g., ['_augmentations.otherAug.score']
}
// What fields this augmentation WRITES
writes?: {
// User metadata it enriches (MUST be declared!)
userFields?: Array<{
field: string
type: 'create' | 'update' | 'merge' | 'delete'
description: string
example?: any
}>
// Its own augmentation namespace
augmentationFields?: Array<{
field: string
description: string
}>
// Internal fields (requires special permission)
internalFields?: Array<{
field: string
permission: 'granted' | 'requested'
reason: string
}>
}
// Conflict resolution strategy
conflictResolution?: {
strategy: 'error' | 'warn' | 'merge' | 'skip' | 'override'
priority?: number // Higher priority wins in conflicts
}
// Safety guarantees
guarantees?: {
preservesExisting?: boolean // Won't delete existing fields
reversible?: boolean // Can undo changes
idempotent?: boolean // Safe to run multiple times
validatesTypes?: boolean // Checks field types before modifying
}
}
/**
* Runtime metadata safety enforcer
*/
export class MetadataSafetyEnforcer {
private contracts: Map<string, AugmentationMetadataContract> = new Map()
private modifications: Map<string, Set<string>> = new Map() // field -> augmentations that modify it
/**
* Register an augmentation's contract
*/
registerContract(contract: AugmentationMetadataContract): void {
this.contracts.set(contract.name, contract)
// Track which augmentations modify which fields
if (contract.writes?.userFields) {
for (const fieldDef of contract.writes.userFields) {
if (!this.modifications.has(fieldDef.field)) {
this.modifications.set(fieldDef.field, new Set())
}
this.modifications.get(fieldDef.field)!.add(contract.name)
}
}
}
/**
* Check if an augmentation can modify a field
*/
canModifyField(augName: string, field: string, value: any): {
allowed: boolean
reason?: string
warnings?: string[]
} {
const contract = this.contracts.get(augName)
if (!contract) {
return {
allowed: false,
reason: `Augmentation '${augName}' has no registered contract`
}
}
// Check if field is in user namespace
if (!field.startsWith('_brainy.') && !field.startsWith('_augmentations.')) {
// It's a user field
const declaredField = contract.writes?.userFields?.find(f => f.field === field)
if (!declaredField) {
return {
allowed: false,
reason: `Augmentation '${augName}' did not declare intent to modify '${field}'`
}
}
// Check for conflicts
const modifiers = this.modifications.get(field)
if (modifiers && modifiers.size > 1) {
const others = Array.from(modifiers).filter(a => a !== augName)
return {
allowed: true, // Still allowed but with warning
warnings: [`Field '${field}' is also modified by: ${others.join(', ')}`]
}
}
return { allowed: true }
}
// Check internal fields
if (field.startsWith('_brainy.')) {
const internalField = contract.writes?.internalFields?.find(f =>
field === `_brainy.${f.field}`
)
if (!internalField) {
return {
allowed: false,
reason: `Augmentation '${augName}' cannot modify internal field '${field}'`
}
}
if (internalField.permission !== 'granted') {
return {
allowed: false,
reason: `Permission not granted for internal field '${field}'`
}
}
return { allowed: true }
}
// Check augmentation namespace
if (field.startsWith('_augmentations.')) {
const parts = field.split('.')
const targetAug = parts[1]
// Can only modify own namespace
if (targetAug !== augName) {
return {
allowed: false,
reason: `Cannot modify another augmentation's namespace: ${targetAug}`
}
}
return { allowed: true }
}
return { allowed: true }
}
/**
* Create safe metadata proxy for an augmentation
*/
createSafeProxy(metadata: any, augName: string): any {
const self = this
return new Proxy(metadata, {
set(target, prop, value) {
const field = String(prop)
// Check permission
const permission = self.canModifyField(augName, field, value)
if (!permission.allowed) {
throw new Error(`[${augName}] ${permission.reason}`)
}
if (permission.warnings) {
console.warn(`[${augName}] Warning:`, ...permission.warnings)
}
// Track modification for audit
if (!target._audit) {
target._audit = []
}
target._audit.push({
augmentation: augName,
field,
oldValue: target[prop],
newValue: value,
timestamp: Date.now()
})
target[prop] = value
return true
},
deleteProperty(target, prop) {
const field = String(prop)
const permission = self.canModifyField(augName, field, undefined)
if (!permission.allowed) {
throw new Error(`[${augName}] Cannot delete field: ${permission.reason}`)
}
delete target[prop]
return true
}
})
}
}
/**
* Example augmentation contracts
*/
export const EXAMPLE_CONTRACTS: Record<string, AugmentationMetadataContract> = {
// Enrichment augmentation that adds categories
categoryEnricher: {
name: 'categoryEnricher',
version: '1.0.0',
reads: {
userFields: ['title', 'description', 'content']
},
writes: {
userFields: [
{
field: 'category',
type: 'create',
description: 'Auto-detected category',
example: 'technology'
},
{
field: 'subcategories',
type: 'create',
description: 'List of relevant subcategories',
example: ['web', 'framework']
}
],
augmentationFields: [
{
field: 'confidence',
description: 'Confidence score of categorization'
}
]
},
guarantees: {
preservesExisting: true,
idempotent: true
}
},
// Translation augmentation
translator: {
name: 'translator',
version: '1.0.0',
reads: {
userFields: ['title', 'description']
},
writes: {
userFields: [
{
field: 'translations',
type: 'merge',
description: 'Translations in multiple languages',
example: { es: 'Título', fr: 'Titre' }
},
{
field: 'detectedLanguage',
type: 'create',
description: 'Detected source language',
example: 'en'
}
]
},
conflictResolution: {
strategy: 'merge',
priority: 10
}
},
// Sentiment analyzer
sentimentAnalyzer: {
name: 'sentimentAnalyzer',
version: '1.0.0',
reads: {
userFields: ['content', 'description', 'reviews']
},
writes: {
userFields: [
{
field: 'sentiment',
type: 'update',
description: 'Overall sentiment score',
example: { score: 0.8, label: 'positive' }
}
],
augmentationFields: [
{
field: 'analysis',
description: 'Detailed sentiment breakdown'
}
]
},
guarantees: {
reversible: true,
validatesTypes: true
}
},
// System augmentation with internal access
garbageCollector: {
name: 'garbageCollector',
version: '1.0.0',
reads: {
internalFields: ['_brainy.createdAt', '_brainy.lastAccessed']
},
writes: {
internalFields: [
{
field: 'deleted',
permission: 'granted',
reason: 'Soft delete expired items'
},
{
field: 'archived',
permission: 'granted',
reason: 'Archive old items'
}
]
}
}
}
/**
* Augmentation base class with safety
*/
export abstract class SafeAugmentation {
protected enforcer: MetadataSafetyEnforcer
protected contract: AugmentationMetadataContract
constructor(contract: AugmentationMetadataContract) {
this.contract = contract
this.enforcer = new MetadataSafetyEnforcer()
this.enforcer.registerContract(contract)
}
/**
* Get safe metadata proxy
*/
protected getSafeMetadata(metadata: any): any {
return this.enforcer.createSafeProxy(metadata, this.contract.name)
}
/**
* Abstract method to implement augmentation logic
*/
abstract execute(metadata: any): Promise<any>
}
/**
* Example: Category enricher implementation
*/
export class CategoryEnricherAugmentation extends SafeAugmentation {
constructor() {
super(EXAMPLE_CONTRACTS.categoryEnricher)
}
async execute(metadata: any): Promise<any> {
const safe = this.getSafeMetadata(metadata)
// Read declared fields
const title = safe.title
const description = safe.description
// Analyze and categorize
const category = this.detectCategory(title, description)
const subcategories = this.detectSubcategories(title, description)
// Write to declared fields (will be checked by proxy)
safe.category = category // ✅ Allowed - declared in contract
safe.subcategories = subcategories // ✅ Allowed
// Try to write undeclared field
// safe.randomField = 'test' // ❌ Would throw error!
// Write to our augmentation namespace
if (!safe._augmentations) safe._augmentations = {}
if (!safe._augmentations.categoryEnricher) {
safe._augmentations.categoryEnricher = {}
}
safe._augmentations.categoryEnricher.confidence = 0.95 // ✅ Allowed
return safe
}
private detectCategory(title: string, description: string): string {
// Simplified logic
return 'technology'
}
private detectSubcategories(title: string, description: string): string[] {
return ['web', 'framework']
}
}

View file

@ -57,6 +57,7 @@ interface ConnectedClient {
export class APIServerAugmentation extends BaseAugmentation {
readonly name = 'api-server'
readonly timing = 'after' as const
readonly metadata = 'readonly' as const // API server reads metadata to serve data
readonly operations = ['all'] as ('all')[]
readonly priority = 5 // Low priority, runs after other augmentations

View file

@ -44,6 +44,7 @@ interface BatchMetrics {
}
export class BatchProcessingAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for batching decisions
name = 'BatchProcessing'
timing = 'around' as const
operations = ['add', 'addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'storage'] as ('add' | 'addNoun' | 'addVerb' | 'saveNoun' | 'saveVerb' | 'storage')[]

View file

@ -12,6 +12,15 @@
* - StreamingPipeline: Enables unlimited data processing
*/
/**
* Metadata access declaration for augmentations
*/
export interface MetadataAccess {
reads?: string[] | '*' // Fields to read, or '*' for all
writes?: string[] | '*' // Fields to write, or '*' for all
namespace?: string // Optional: custom namespace like '_myAug'
}
export interface BrainyAugmentation {
/**
* Unique identifier for the augmentation
@ -27,6 +36,14 @@ export interface BrainyAugmentation {
*/
timing: 'before' | 'after' | 'around' | 'replace'
/**
* Metadata access contract - REQUIRED
* - 'none': No metadata access at all
* - 'readonly': Can read any metadata but cannot write
* - MetadataAccess: Specific fields to read/write
*/
metadata: 'none' | 'readonly' | MetadataAccess
/**
* Which operations this augmentation applies to
* Granular operation matching for precise augmentation targeting
@ -125,6 +142,7 @@ export interface AugmentationContext {
export abstract class BaseAugmentation implements BrainyAugmentation {
abstract name: string
abstract timing: 'before' | 'after' | 'around' | 'replace'
abstract metadata: 'none' | 'readonly' | MetadataAccess
abstract operations: (
// Data Operations
| 'add' | 'addNoun' | 'addVerb'

View file

@ -31,6 +31,7 @@ export interface CacheConfig {
export class CacheAugmentation extends BaseAugmentation {
readonly name = 'cache'
readonly timing = 'around' as const
readonly metadata = 'none' as const // Cache doesn't access metadata
operations = ['search', 'add', 'delete', 'clear', 'all'] as ('search' | 'add' | 'delete' | 'clear' | 'all')[]
readonly priority = 50 // Mid-priority, runs after data operations

View file

@ -21,6 +21,7 @@ export interface WebSocketConnection {
*/
abstract class BaseConduitAugmentation extends BaseAugmentation {
readonly timing = 'after' as const // Conduits run after operations to sync
readonly metadata = 'readonly' as const // Conduits read metadata to pass to external systems
readonly operations = ['addNoun', 'delete', 'addVerb'] as ('addNoun' | 'delete' | 'addVerb')[]
readonly priority = 20 // Medium-low priority

View file

@ -41,6 +41,7 @@ interface QueuedRequest {
export class ConnectionPoolAugmentation extends BaseAugmentation {
name = 'ConnectionPool'
timing = 'around' as const
metadata = 'none' as const // Connection pooling doesn't access metadata
operations = ['storage'] as ('storage')[]
priority = 95 // Very high priority for storage operations

View file

@ -54,6 +54,7 @@ export interface EntityMapping {
* Optimized for streaming data scenarios like Bluesky firehose processing
*/
export class EntityRegistryAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata to register entities
readonly name = 'entity-registry'
readonly description = 'Fast external-ID to internal-UUID mapping for streaming data'
readonly timing: 'before' | 'after' | 'around' | 'replace' = 'before'
@ -469,6 +470,7 @@ export class EntityRegistryAugmentation extends BaseAugmentation {
// Hook into Brainy's add operations to automatically register entities
export class AutoRegisterEntitiesAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for auto-registration
readonly name = 'auto-register-entities'
readonly description = 'Automatically register entities in the registry when added'
readonly timing: 'before' | 'after' | 'around' | 'replace' = 'after'

View file

@ -31,6 +31,7 @@ export interface IndexConfig {
* - Zero-config with smart defaults
*/
export class IndexAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata to build indexes
readonly name = 'index'
readonly timing = 'after' as const
operations = ['add', 'updateMetadata', 'delete', 'clear', 'all'] as ('add' | 'updateMetadata' | 'delete' | 'clear' | 'all')[]

View file

@ -66,6 +66,10 @@ interface ScoringMetrics {
export class IntelligentVerbScoringAugmentation extends BaseAugmentation {
name = 'IntelligentVerbScoring'
timing = 'around' as const
readonly metadata = {
reads: ['type', 'verb', 'source', 'target'] as string[],
writes: ['weight', 'confidence', 'intelligentScoring'] as string[]
} // Adds scoring metadata to verbs
operations = ['addVerb', 'relate'] as ('addVerb' | 'relate')[]
priority = 10 // Enhancement feature - runs after core operations

View file

@ -0,0 +1,204 @@
/**
* Runtime enforcement of metadata contracts
* Ensures augmentations only access declared fields
*/
import { BrainyAugmentation, MetadataAccess } from './brainyAugmentation.js'
export class MetadataEnforcer {
/**
* Enforce metadata access based on augmentation contract
* Returns a wrapped metadata object that enforces the contract
*/
static enforce(
augmentation: BrainyAugmentation,
metadata: any,
operation: 'read' | 'write' = 'write'
): any {
// Handle simple contracts
if (augmentation.metadata === 'none') {
// No access at all
if (operation === 'read') return null
throw new Error(`Augmentation '${augmentation.name}' has metadata='none' - cannot access metadata`)
}
if (augmentation.metadata === 'readonly') {
if (operation === 'read') {
// Return frozen deep clone for read-only access
return deepFreeze(deepClone(metadata))
}
throw new Error(`Augmentation '${augmentation.name}' has metadata='readonly' - cannot write`)
}
// Handle specific field access
const access = augmentation.metadata as MetadataAccess
if (operation === 'read') {
// For reads, filter to allowed fields
if (access.reads === '*') {
return deepClone(metadata) // Can read everything
}
if (!access.reads) {
return {} // No read access
}
// Filter to specific fields
const filtered: any = {}
for (const field of access.reads) {
if (field.includes('.')) {
// Handle nested fields like '_brainy.deleted'
const parts = field.split('.')
let source = metadata
let target = filtered
for (let i = 0; i < parts.length - 1; i++) {
const part = parts[i]
if (!source[part]) break
if (!target[part]) target[part] = {}
source = source[part]
target = target[part]
}
const lastPart = parts[parts.length - 1]
if (source && lastPart in source) {
target[lastPart] = source[lastPart]
}
} else {
// Simple field
if (field in metadata) {
filtered[field] = metadata[field]
}
}
}
return filtered
}
// For writes, create a proxy that validates
return new Proxy(metadata, {
set(target, prop, value) {
const field = String(prop)
// Check if write is allowed
if (access.writes === '*') {
// Can write anything
target[prop] = value
return true
}
if (!access.writes || !access.writes.includes(field)) {
throw new Error(
`Augmentation '${augmentation.name}' cannot write to field '${field}'. ` +
`Allowed writes: ${access.writes?.join(', ') || 'none'}`
)
}
// Check namespace if specified
if (access.namespace && !field.startsWith(access.namespace)) {
console.warn(
`Augmentation '${augmentation.name}' writing outside its namespace. ` +
`Expected: ${access.namespace}.*, got: ${field}`
)
}
target[prop] = value
return true
},
deleteProperty(target, prop) {
const field = String(prop)
// Deletion counts as a write
if (access.writes === '*' || access.writes?.includes(field)) {
delete target[prop]
return true
}
throw new Error(
`Augmentation '${augmentation.name}' cannot delete field '${field}'`
)
}
})
}
/**
* Validate that an augmentation's actual behavior matches its contract
* Used in testing to verify contracts are accurate
*/
static async validateContract(
augmentation: BrainyAugmentation,
testMetadata: any = { test: 'data', _brainy: { deleted: false } }
): Promise<{ valid: boolean; violations: string[] }> {
const violations: string[] = []
// Test read access
try {
const readable = this.enforce(augmentation, testMetadata, 'read')
if (augmentation.metadata === 'none' && readable !== null) {
violations.push(`Contract says 'none' but got readable metadata`)
}
} catch (error) {
violations.push(`Read enforcement error: ${error}`)
}
// Test write access
try {
const writable = this.enforce(augmentation, testMetadata, 'write')
if (augmentation.metadata === 'none') {
violations.push(`Contract says 'none' but got writable metadata`)
}
if (augmentation.metadata === 'readonly') {
// Try to write - should fail
try {
writable.testWrite = 'value'
violations.push(`Contract says 'readonly' but write succeeded`)
} catch {
// Expected to fail
}
}
} catch (error) {
// Expected for 'none' and 'readonly' on write
if (augmentation.metadata !== 'none' && augmentation.metadata !== 'readonly') {
violations.push(`Write enforcement error: ${error}`)
}
}
return {
valid: violations.length === 0,
violations
}
}
}
// Helper functions
function deepClone(obj: any): any {
if (obj === null || typeof obj !== 'object') return obj
if (obj instanceof Date) return new Date(obj)
if (obj instanceof Array) return obj.map(item => deepClone(item))
const cloned: any = {}
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
cloned[key] = deepClone(obj[key])
}
}
return cloned
}
function deepFreeze(obj: any): any {
Object.freeze(obj)
Object.getOwnPropertyNames(obj).forEach(prop => {
if (obj[prop] !== null &&
(typeof obj[prop] === 'object' || typeof obj[prop] === 'function') &&
!Object.isFrozen(obj[prop])) {
deepFreeze(obj[prop])
}
})
return obj
}

View file

@ -32,6 +32,7 @@ export interface MetricsConfig {
* - Zero-config with smart defaults
*/
export class MetricsAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for metrics
readonly name = 'metrics'
readonly timing = 'after' as const
operations = ['add', 'search', 'delete', 'clear', 'all'] as ('add' | 'search' | 'delete' | 'clear' | 'all')[]

View file

@ -33,6 +33,7 @@ export interface MonitoringConfig {
* - Zero-config with smart defaults
*/
export class MonitoringAugmentation extends BaseAugmentation {
readonly metadata = 'readonly' as const // Reads metadata for monitoring
readonly name = 'monitoring'
readonly timing = 'after' as const
operations = ['search', 'add', 'delete', 'all'] as ('search' | 'add' | 'delete' | 'all')[]

View file

@ -65,6 +65,10 @@ export interface NeuralImportConfig {
export class NeuralImportAugmentation extends BaseAugmentation {
readonly name = 'neural-import'
readonly timing = 'before' as const // Process data before storage
readonly metadata = {
reads: '*' as '*', // Needs to read data for analysis
writes: ['_neuralProcessed', '_neuralConfidence', '_detectedEntities', '_detectedRelationships', '_neuralInsights', 'nounType', 'verbType'] as string[]
} // Enriches metadata with neural analysis
operations = ['add', 'addNoun', 'addVerb', 'all'] as ('add' | 'addNoun' | 'addVerb' | 'all')[] // Use 'all' to catch batch operations
readonly priority = 80 // High priority for data processing

View file

@ -22,6 +22,7 @@ interface DeduplicatorConfig {
export class RequestDeduplicatorAugmentation extends BaseAugmentation {
name = 'RequestDeduplicator'
timing = 'around' as const
metadata = 'none' as const // Doesn't access metadata
operations = ['search', 'searchText', 'searchByNounTypes', 'findSimilar', 'get'] as ('search' | 'searchText' | 'searchByNounTypes' | 'findSimilar' | 'get')[]
priority = 50 // Performance optimization

View file

@ -23,6 +23,7 @@ import { BrainyDataInterface } from '../types/brainyDataInterface.js'
export class ServerSearchConduitAugmentation extends BaseAugmentation {
readonly name = 'server-search-conduit'
readonly timing = 'after' as const
readonly metadata = 'readonly' as const // Reads metadata to sync with server
operations = ['addNoun', 'delete', 'addVerb'] as ('addNoun' | 'delete' | 'addVerb')[]
readonly priority = 20
private localDb: BrainyDataInterface | null = null
@ -377,6 +378,7 @@ export class ServerSearchConduitAugmentation extends BaseAugmentation {
export class ServerSearchActivationAugmentation extends BaseAugmentation {
readonly name = 'server-search-activation'
readonly timing = 'after' as const
readonly metadata = 'readonly' as const // Reads metadata for server activation
operations = ['search', 'addNoun'] as ('search' | 'addNoun')[]
readonly priority = 20

View file

@ -14,6 +14,7 @@ import { StorageAdapter } from '../coreTypes.js'
*/
export abstract class StorageAugmentation extends BaseAugmentation implements BrainyAugmentation {
readonly timing = 'replace' as const
readonly metadata = 'none' as const // Storage doesn't directly access metadata
operations = ['storage'] as ('storage')[] // Make mutable for TypeScript compatibility
readonly priority = 100 // High priority for storage

View file

@ -32,6 +32,10 @@ export abstract class SynapseAugmentation extends BaseAugmentation {
readonly timing = 'after' as const
readonly operations = ['all'] as ('all')[]
readonly priority = 10
readonly metadata = {
reads: '*' as '*', // Needs to read for syncing
writes: ['_synapse', '_syncedAt'] as string[]
} // Adds synapse tracking metadata
// Synapse-specific properties
abstract readonly synapseId: string

View file

@ -38,6 +38,7 @@ interface WALConfig {
export class WALAugmentation extends BaseAugmentation {
name = 'WAL'
timing = 'around' as const
metadata = 'readonly' as const // Reads metadata for logging/recovery
operations = ['addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'updateMetadata', 'delete', 'deleteVerb', 'clear'] as ('addNoun' | 'addVerb' | 'saveNoun' | 'saveVerb' | 'updateMetadata' | 'delete' | 'deleteVerb' | 'clear')[]
priority = 100 // Critical system operation - highest priority