fix: Knowledge Layer EntityManager integration

* Add EntityManager base class for standardized entity operations
* Update SemanticVersioning to extend EntityManager
* Update EventRecorder to extend EntityManager
* Update PersistentEntitySystem to extend EntityManager
* Update ConceptSystem to extend EntityManager
* Fix entity ID mismatch patterns across all Knowledge Layer components
* Add proper domain ID to Brainy ID mapping
* Replace direct brain.add/find calls with EntityManager methods
* Ensure all entity interfaces extend ManagedEntity
* Add proper metadata fields for querying (eventType, conceptType, entityType)

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-09-25 12:54:35 -07:00
parent 6e6da5011f
commit cc6fa00f30
8 changed files with 587 additions and 328 deletions

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{ {
"name": "@soulcraft/brainy", "name": "@soulcraft/brainy",
"version": "3.10.0", "version": "3.10.1",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@soulcraft/brainy", "name": "@soulcraft/brainy",
"version": "3.10.0", "version": "3.10.1",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.540.0", "@aws-sdk/client-s3": "^3.540.0",

View file

@ -1,6 +1,6 @@
{ {
"name": "@soulcraft/brainy", "name": "@soulcraft/brainy",
"version": "3.10.0", "version": "3.10.1",
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
"main": "dist/index.js", "main": "dist/index.js",
"module": "dist/index.js", "module": "dist/index.js",

View file

@ -10,11 +10,12 @@ import { Brainy } from '../brainy.js'
import { NounType, VerbType } from '../types/graphTypes.js' import { NounType, VerbType } from '../types/graphTypes.js'
import { cosineDistance } from '../utils/distance.js' import { cosineDistance } from '../utils/distance.js'
import { v4 as uuidv4 } from '../universal/uuid.js' import { v4 as uuidv4 } from '../universal/uuid.js'
import { EntityManager, ManagedEntity } from './EntityManager.js'
/** /**
* Universal concept that exists independently of files * Universal concept that exists independently of files
*/ */
export interface UniversalConcept { export interface UniversalConcept extends ManagedEntity {
id: string id: string
name: string name: string
description?: string description?: string
@ -28,6 +29,7 @@ export interface UniversalConcept {
lastUpdated: number lastUpdated: number
version: number version: number
metadata: Record<string, any> metadata: Record<string, any>
conceptType?: string // For EntityManager queries
} }
/** /**
@ -45,7 +47,7 @@ export interface ConceptLink {
/** /**
* A manifestation of a concept in a specific location * A manifestation of a concept in a specific location
*/ */
export interface ConceptManifestation { export interface ConceptManifestation extends ManagedEntity {
id: string id: string
conceptId: string conceptId: string
filePath: string filePath: string
@ -100,14 +102,15 @@ export interface ConceptGraph {
* - "Dependency Injection" pattern across multiple codebases * - "Dependency Injection" pattern across multiple codebases
* - "Sustainability" theme in various research papers * - "Sustainability" theme in various research papers
*/ */
export class ConceptSystem { export class ConceptSystem extends EntityManager {
private config: Required<ConceptSystemConfig> private config: Required<ConceptSystemConfig>
private conceptCache = new Map<string, UniversalConcept>() private conceptCache = new Map<string, UniversalConcept>()
constructor( constructor(
private brain: Brainy, brain: Brainy,
config?: ConceptSystemConfig config?: ConceptSystemConfig
) { ) {
super(brain, 'vfs-concept')
this.config = { this.config = {
autoLink: config?.autoLink ?? false, autoLink: config?.autoLink ?? false,
similarityThreshold: config?.similarityThreshold ?? 0.7, similarityThreshold: config?.similarityThreshold ?? 0.7,
@ -124,13 +127,20 @@ export class ConceptSystem {
const timestamp = Date.now() const timestamp = Date.now()
const universalConcept: UniversalConcept = { const universalConcept: UniversalConcept = {
...concept,
id: conceptId, id: conceptId,
name: concept.name,
description: concept.description,
domain: concept.domain,
category: concept.category,
keywords: concept.keywords,
strength: concept.strength,
metadata: concept.metadata || {},
created: timestamp, created: timestamp,
lastUpdated: timestamp, lastUpdated: timestamp,
version: 1, version: 1,
links: [], links: [],
manifestations: [] manifestations: [],
conceptType: 'universal' // Add conceptType for querying
} }
// Generate embedding for concept // Generate embedding for concept
@ -141,17 +151,13 @@ export class ConceptSystem {
console.warn('Failed to generate concept embedding:', error) console.warn('Failed to generate concept embedding:', error)
} }
// Store concept in Brainy // Store concept using EntityManager
const brainyEntity = await this.brain.add({ await this.storeEntity(
type: NounType.Concept, universalConcept,
data: Buffer.from(JSON.stringify(universalConcept)), NounType.Concept,
metadata: { embedding,
...universalConcept, Buffer.from(JSON.stringify(universalConcept))
conceptType: 'universal', )
system: 'vfs-concept'
},
vector: embedding
})
// Auto-link to similar concepts if enabled // Auto-link to similar concepts if enabled
if (this.config.autoLink) { if (this.config.autoLink) {
@ -161,7 +167,7 @@ export class ConceptSystem {
// Update cache // Update cache
this.conceptCache.set(conceptId, universalConcept) this.conceptCache.set(conceptId, universalConcept)
return brainyEntity return conceptId // Return domain ID, not Brainy ID
} }
/** /**
@ -193,17 +199,16 @@ export class ConceptSystem {
// File manifestation search // File manifestation search
if (query.manifestedIn) { if (query.manifestedIn) {
// Find concepts that have manifestations in this file // Find concepts that have manifestations in this file
const manifestationResults = await this.brain.find({ const manifestationResults = await this.findEntities<ConceptManifestation>(
where: { {
filePath: query.manifestedIn, filePath: query.manifestedIn,
eventType: 'concept-manifestation', eventType: 'concept-manifestation'
system: 'vfs-concept'
}, },
type: NounType.Event, NounType.Event,
limit: 1000 1000
}) )
const conceptIds = manifestationResults.map(r => r.entity.metadata.conceptId) const conceptIds = manifestationResults.map(m => m.conceptId)
if (conceptIds.length > 0) { if (conceptIds.length > 0) {
searchQuery.id = { $in: conceptIds } searchQuery.id = { $in: conceptIds }
} else { } else {
@ -211,12 +216,12 @@ export class ConceptSystem {
} }
} }
// Search in Brainy // Search using EntityManager
let results = await this.brain.find({ const results = await this.findEntities<UniversalConcept>(
where: searchQuery, searchQuery,
type: NounType.Concept, NounType.Concept,
limit: 1000 1000
}) )
// If searching for similar concepts, use vector similarity // If searching for similar concepts, use vector similarity
if (query.similar) { if (query.similar) {
@ -224,29 +229,42 @@ export class ConceptSystem {
const queryEmbedding = await this.generateTextEmbedding(query.similar) const queryEmbedding = await this.generateTextEmbedding(query.similar)
if (queryEmbedding) { if (queryEmbedding) {
// Get all concepts and rank by similarity // Get all concepts and rank by similarity
const allConcepts = await this.brain.find({ const allConcepts = await this.findEntities<UniversalConcept>(
where: { conceptType: 'universal', system: 'vfs-concept' }, { conceptType: 'universal' },
type: NounType.Concept, NounType.Concept,
limit: 10000 10000
}) )
const withSimilarity = allConcepts // For similarity search, we need to get the actual vector data from Brainy
.filter(c => c.entity.vector && c.entity.vector.length > 0) const conceptsWithVectors = []
for (const concept of allConcepts) {
if (concept.brainyId) {
const brainyEntity = await this.brain.get(concept.brainyId)
if (brainyEntity?.vector && brainyEntity.vector.length > 0) {
conceptsWithVectors.push({
concept,
vector: brainyEntity.vector
})
}
}
}
const withSimilarity = conceptsWithVectors
.map(c => ({ .map(c => ({
concept: c, concept: c.concept,
similarity: 1 - cosineDistance(queryEmbedding, c.entity.vector!) similarity: 1 - cosineDistance(queryEmbedding, c.vector)
})) }))
.filter(s => s.similarity > this.config.similarityThreshold) .filter(s => s.similarity > this.config.similarityThreshold)
.sort((a, b) => b.similarity - a.similarity) .sort((a, b) => b.similarity - a.similarity)
results = withSimilarity.map(s => s.concept) return withSimilarity.map(s => s.concept)
} }
} catch (error) { } catch (error) {
console.warn('Failed to perform concept similarity search:', error) console.warn('Failed to perform concept similarity search:', error)
} }
} }
return results.map(r => r.entity.metadata as UniversalConcept) return results
} }
/** /**
@ -302,17 +320,17 @@ export class ConceptSystem {
await this.updateConcept(toConcept) await this.updateConcept(toConcept)
} }
// Create Brainy relationship // Create relationship using EntityManager
await this.brain.relate({ await this.createRelationship(
from: fromConceptId, fromConceptId,
to: toConceptId, toConceptId,
type: this.getVerbType(relationship), this.getVerbType(relationship),
metadata: { {
strength: link.strength, strength: link.strength,
context: link.context, context: link.context,
bidirectional: link.bidirectional bidirectional: link.bidirectional
} }
}) )
return linkId return linkId
} }
@ -351,23 +369,25 @@ export class ConceptSystem {
extractedBy: options?.extractedBy ?? 'manual' extractedBy: options?.extractedBy ?? 'manual'
} }
// Store manifestation as Brainy event // Store manifestation as managed entity (with eventType for manifestations)
await this.brain.add({ const manifestationWithEventType = {
type: NounType.Event,
data: Buffer.from(context),
metadata: {
...manifestation, ...manifestation,
eventType: 'concept-manifestation', eventType: 'concept-manifestation'
system: 'vfs-concept'
} }
})
// Create relationship to concept await this.storeEntity(
await this.brain.relate({ manifestationWithEventType,
from: manifestationId, NounType.Event,
to: conceptId, undefined,
type: VerbType.Implements Buffer.from(context)
}) )
// Create relationship to concept using EntityManager
await this.createRelationship(
manifestationId,
conceptId,
VerbType.Implements
)
// Update concept with new manifestation // Update concept with new manifestation
concept.manifestations.push(manifestation) concept.manifestations.push(manifestation)
@ -476,13 +496,11 @@ export class ConceptSystem {
query.strength = { $gte: options.minStrength } query.strength = { $gte: options.minStrength }
} }
const results = await this.brain.find({ const concepts = await this.findEntities<UniversalConcept>(
where: query, query,
type: NounType.Concept, NounType.Concept,
limit: options?.maxConcepts || 1000 options?.maxConcepts || 1000
}) )
const concepts = results.map(r => r.entity.metadata as UniversalConcept)
// Build graph structure // Build graph structure
const graphConcepts = concepts.map(c => ({ const graphConcepts = concepts.map(c => ({
@ -541,15 +559,13 @@ export class ConceptSystem {
query.confidence = { $gte: options.minConfidence } query.confidence = { $gte: options.minConfidence }
} }
const results = await this.brain.find({ const manifestations = await this.findEntities<ConceptManifestation>(
where: query, query,
type: NounType.Event, NounType.Event,
limit: options?.limit || 1000 options?.limit || 1000
}) )
return results return manifestations.sort((a, b) => b.timestamp - a.timestamp)
.map(r => r.entity.metadata as ConceptManifestation)
.sort((a, b) => b.timestamp - a.timestamp)
} }
/** /**
@ -590,23 +606,11 @@ export class ConceptSystem {
return this.conceptCache.get(conceptId)! return this.conceptCache.get(conceptId)!
} }
// Query from Brainy // Query using EntityManager
const results = await this.brain.find({ const concept = await this.getEntity<UniversalConcept>(conceptId)
where: { if (concept) {
id: conceptId,
conceptType: 'universal',
system: 'vfs-concept'
},
type: NounType.Concept,
limit: 1
})
if (results.length === 0) {
return null
}
const concept = results[0].entity.metadata as UniversalConcept
this.conceptCache.set(conceptId, concept) this.conceptCache.set(conceptId, concept)
}
return concept return concept
} }
@ -614,28 +618,11 @@ export class ConceptSystem {
* Update stored concept * Update stored concept
*/ */
private async updateConcept(concept: UniversalConcept): Promise<void> { private async updateConcept(concept: UniversalConcept): Promise<void> {
// Find the Brainy entity // Add conceptType metadata before updating
const results = await this.brain.find({ concept.conceptType = 'universal'
where: {
id: concept.id,
conceptType: 'universal',
system: 'vfs-concept'
},
type: NounType.Concept,
limit: 1
})
if (results.length > 0) { // Update using EntityManager
await this.brain.update({ await this.updateEntity(concept)
id: results[0].entity.id,
data: Buffer.from(JSON.stringify(concept)),
metadata: {
...concept,
conceptType: 'universal',
system: 'vfs-concept'
}
})
}
// Update cache // Update cache
this.conceptCache.set(concept.id, concept) this.conceptCache.set(concept.id, concept)

291
src/vfs/EntityManager.ts Normal file
View file

@ -0,0 +1,291 @@
/**
* EntityManager Base Class
*
* Provides standardized entity ID management for all Knowledge Layer components
* Solves the root cause of ID mismatch issues by establishing clear patterns
*/
import { Brainy } from '../brainy.js'
import { NounType } from '../types/graphTypes.js'
import { v4 as uuidv4 } from '../universal/uuid.js'
/**
* Standard entity structure used by all Knowledge Layer components
*/
export interface ManagedEntity {
/** Domain-specific ID (for external references) */
id: string
/** The actual Brainy entity ID (for internal operations) */
brainyId?: string
/** Entity metadata */
[key: string]: any
}
/**
* ID mapping to track domain IDs to Brainy entity IDs
*/
interface EntityIdMapping {
domainId: string
brainyId: string
system: string
type: string
}
/**
* EntityManager Base Class
*
* All Knowledge Layer components should extend this to get standardized:
* - Entity storage and retrieval
* - ID management and mapping
* - Query patterns
* - Relationship creation
*/
export abstract class EntityManager {
private idMappings = new Map<string, string>() // domainId -> brainyId
private brainyToMappings = new Map<string, string>() // brainyId -> domainId
constructor(
protected brain: Brainy,
protected systemName: string
) {}
/**
* Store an entity with proper ID management
*/
protected async storeEntity<T extends ManagedEntity>(
entity: T,
nounType: NounType,
embedding?: number[],
data?: any
): Promise<string> {
// Generate domain ID if not provided
if (!entity.id) {
entity.id = uuidv4()
}
// Store in Brainy and get the Brainy entity ID
const brainyId = await this.brain.add({
type: nounType,
data: data || Buffer.from(JSON.stringify(entity)),
metadata: {
...entity,
system: this.systemName,
domainId: entity.id, // Store the domain ID for reference
storedAt: Date.now()
},
vector: embedding
})
// Store ID mapping
this.idMappings.set(entity.id, brainyId)
this.brainyToMappings.set(brainyId, entity.id)
// Update entity with Brainy ID
entity.brainyId = brainyId
return entity.id // Return domain ID for external use
}
/**
* Update an existing entity
*/
protected async updateEntity<T extends ManagedEntity>(
entity: T,
embedding?: number[]
): Promise<void> {
const brainyId = this.idMappings.get(entity.id)
if (!brainyId) {
throw new Error(`Entity ${entity.id} not found in mappings`)
}
await this.brain.update({
id: brainyId,
data: Buffer.from(JSON.stringify(entity)),
metadata: {
...entity,
system: this.systemName,
domainId: entity.id,
updatedAt: Date.now()
},
vector: embedding
})
}
/**
* Retrieve entity by domain ID
*/
protected async getEntity<T extends ManagedEntity>(domainId: string): Promise<T | null> {
// First try from cache
let brainyId = this.idMappings.get(domainId)
// If not in cache, search by domain ID in metadata
if (!brainyId) {
const results = await this.brain.find({
where: {
domainId,
system: this.systemName
},
limit: 1
})
if (results.length === 0) {
return null
}
brainyId = results[0].entity.id
// Cache the mapping
this.idMappings.set(domainId, brainyId)
this.brainyToMappings.set(brainyId, domainId)
}
// Get entity using Brainy ID
const brainyEntity = await this.brain.get(brainyId)
if (!brainyEntity) {
return null
}
// Parse entity from metadata
const entity = brainyEntity.metadata as T
entity.brainyId = brainyId
return entity
}
/**
* Find entities by metadata criteria
*/
protected async findEntities<T extends ManagedEntity>(
criteria: Record<string, any>,
nounType?: NounType,
limit = 100
): Promise<T[]> {
const query: any = {
...criteria,
system: this.systemName
}
const results = await this.brain.find({
where: query,
type: nounType,
limit
})
const entities: T[] = []
for (const result of results) {
const entity = result.entity.metadata as T
entity.brainyId = result.entity.id
// Update mappings cache
this.idMappings.set(entity.id, result.entity.id)
this.brainyToMappings.set(result.entity.id, entity.id)
entities.push(entity)
}
return entities
}
/**
* Create relationship between entities using proper Brainy IDs
*/
protected async createRelationship(
fromDomainId: string,
toDomainId: string,
relationshipType: any,
metadata?: Record<string, any>
): Promise<void> {
// Get Brainy IDs for both entities
const fromBrainyId = await this.getBrainyId(fromDomainId)
const toBrainyId = await this.getBrainyId(toDomainId)
if (!fromBrainyId || !toBrainyId) {
throw new Error(`Cannot find Brainy IDs for relationship: ${fromDomainId} -> ${toDomainId}`)
}
// Create relationship using Brainy IDs
await this.brain.relate({
from: fromBrainyId,
to: toBrainyId,
type: relationshipType,
metadata
})
}
/**
* Get Brainy ID for a domain ID
*/
protected async getBrainyId(domainId: string): Promise<string | null> {
// Check cache first
let brainyId = this.idMappings.get(domainId)
if (!brainyId) {
// Search in Brainy
const results = await this.brain.find({
where: {
domainId,
system: this.systemName
},
limit: 1
})
if (results.length > 0) {
brainyId = results[0].entity.id
// Cache the mapping
this.idMappings.set(domainId, brainyId)
this.brainyToMappings.set(brainyId, domainId)
}
}
return brainyId || null
}
/**
* Get domain ID for a Brainy ID
*/
protected getDomainId(brainyId: string): string | null {
return this.brainyToMappings.get(brainyId) || null
}
/**
* Delete entity by domain ID
*/
protected async deleteEntity(domainId: string): Promise<void> {
const brainyId = await this.getBrainyId(domainId)
if (brainyId) {
await this.brain.delete(brainyId)
// Clean up mappings
this.idMappings.delete(domainId)
this.brainyToMappings.delete(brainyId)
}
}
/**
* Clear ID mapping cache (useful for tests)
*/
protected clearMappingCache(): void {
this.idMappings.clear()
this.brainyToMappings.clear()
}
/**
* Batch load mappings for performance
*/
protected async loadMappings(domainIds: string[]): Promise<void> {
const unknownIds = domainIds.filter(id => !this.idMappings.has(id))
if (unknownIds.length === 0) return
const results = await this.brain.find({
where: {
domainId: { $in: unknownIds },
system: this.systemName
},
limit: unknownIds.length
})
for (const result of results) {
const domainId = result.entity.metadata.domainId
this.idMappings.set(domainId, result.entity.id)
this.brainyToMappings.set(result.entity.id, domainId)
}
}
}

View file

@ -9,11 +9,12 @@ import { Brainy } from '../brainy.js'
import { NounType, VerbType } from '../types/graphTypes.js' import { NounType, VerbType } from '../types/graphTypes.js'
import { v4 as uuidv4 } from '../universal/uuid.js' import { v4 as uuidv4 } from '../universal/uuid.js'
import { createHash } from 'crypto' import { createHash } from 'crypto'
import { EntityManager, ManagedEntity } from './EntityManager.js'
/** /**
* File operation event * File operation event
*/ */
export interface FileEvent { export interface FileEvent extends ManagedEntity {
id: string id: string
type: 'create' | 'write' | 'append' | 'delete' | 'move' | 'rename' | 'mkdir' | 'rmdir' type: 'create' | 'write' | 'append' | 'delete' | 'move' | 'rename' | 'mkdir' | 'rmdir'
path: string path: string
@ -25,13 +26,16 @@ export interface FileEvent {
metadata?: Record<string, any> metadata?: Record<string, any>
previousHash?: string // For tracking changes previousHash?: string // For tracking changes
oldPath?: string // For move/rename operations oldPath?: string // For move/rename operations
eventType?: string // For EntityManager queries
} }
/** /**
* Event Recorder - Stores all file operations as searchable events * Event Recorder - Stores all file operations as searchable events
*/ */
export class EventRecorder { export class EventRecorder extends EntityManager {
constructor(private brain: Brainy) {} constructor(brain: Brainy) {
super(brain, 'vfs')
}
/** /**
* Record a file operation event * Record a file operation event
@ -45,27 +49,44 @@ export class EventRecorder {
? createHash('sha256').update(event.content).digest('hex') ? createHash('sha256').update(event.content).digest('hex')
: undefined : undefined
// Store event as Brainy entity // Create file event
const entity = await this.brain.add({ const fileEvent: FileEvent = {
type: NounType.Event,
data: event.content || Buffer.from(JSON.stringify(event)),
metadata: {
...event,
id: eventId, id: eventId,
type: event.type,
path: event.path,
timestamp, timestamp,
content: event.content,
size: event.size,
hash, hash,
eventType: 'file-operation', author: event.author,
system: 'vfs' metadata: event.metadata,
}, previousHash: event.previousHash,
oldPath: event.oldPath
}
// Generate embedding for content-based events // Generate embedding for content-based events
vector: event.content && event.content.length < 100000 const embedding = event.content && event.content.length < 100000
? await this.generateEventEmbedding(event) ? await this.generateEventEmbedding(event)
: undefined : undefined
})
return entity // Add eventType for event classification
const eventWithType = {
...fileEvent,
eventType: 'file-operation'
} }
// Store event using EntityManager
await this.storeEntity(
eventWithType,
NounType.Event,
embedding,
event.content || Buffer.from(JSON.stringify(event))
)
return eventId
}
/** /**
* Get all events matching criteria * Get all events matching criteria
*/ */
@ -95,19 +116,8 @@ export class EventRecorder {
query.type = { $in: options.types } query.type = { $in: options.types }
} }
// Query events from Brainy // Query using EntityManager
const results = await this.brain.find({ const events = await this.findEntities<FileEvent>(query, NounType.Event, limit)
where: query,
type: NounType.Event,
limit
})
const events: FileEvent[] = []
for (const result of results) {
if (result.entity?.metadata) {
events.push(result.entity.metadata as FileEvent)
}
}
return events.sort((a, b) => b.timestamp - a.timestamp) return events.sort((a, b) => b.timestamp - a.timestamp)
} }
@ -138,28 +148,8 @@ export class EventRecorder {
query.type = { $in: options.types } query.type = { $in: options.types }
} }
// Query events from Brainy // Query using EntityManager
const results = await this.brain.find({ const events = await this.findEntities<FileEvent>(query, NounType.Event, options?.limit || 100)
where: query,
type: NounType.Event,
limit: options?.limit || 100,
// Sort by timestamp descending (newest first)
// Note: Sorting would need to be implemented in Brainy
})
// Convert results to FileEvent format
const events = results.map(r => ({
id: r.entity.metadata.id,
type: r.entity.metadata.type,
path: r.entity.metadata.path,
timestamp: r.entity.metadata.timestamp,
content: r.entity.metadata.content,
size: r.entity.metadata.size,
hash: r.entity.metadata.hash,
author: r.entity.metadata.author,
metadata: r.entity.metadata.metadata,
previousHash: r.entity.metadata.previousHash
} as FileEvent))
// Sort by timestamp (newest first) // Sort by timestamp (newest first)
return events.sort((a, b) => b.timestamp - a.timestamp) return events.sort((a, b) => b.timestamp - a.timestamp)
@ -219,20 +209,17 @@ export class EventRecorder {
query.timestamp.$lte = until query.timestamp.$lte = until
} }
const results = await this.brain.find({ // Query using EntityManager
where: query, const events = await this.findEntities<FileEvent>(query, NounType.Event, 1000)
type: NounType.Event,
limit: 1000
})
return results.map(r => ({ return events.map(event => ({
id: r.entity.metadata.id, id: event.id,
type: r.entity.metadata.type, type: event.type,
path: r.entity.metadata.path, path: event.path,
timestamp: r.entity.metadata.timestamp, timestamp: event.timestamp,
size: r.entity.metadata.size, size: event.size,
hash: r.entity.metadata.hash, hash: event.hash,
author: r.entity.metadata.author author: event.author
} as FileEvent)) } as FileEvent))
} }
@ -357,7 +344,7 @@ export class EventRecorder {
for (let i = 0; i < events.length; i++) { for (let i = 0; i < events.length; i++) {
// Keep every Nth event for history sampling // Keep every Nth event for history sampling
if (i % keepEvery !== 0) { if (i % keepEvery !== 0) {
await this.brain.delete(events[i].id) await this.deleteEntity(events[i].id)
pruned++ pruned++
} }
} }

View file

@ -10,11 +10,12 @@ import { Brainy } from '../brainy.js'
import { NounType, VerbType } from '../types/graphTypes.js' import { NounType, VerbType } from '../types/graphTypes.js'
import { cosineDistance } from '../utils/distance.js' import { cosineDistance } from '../utils/distance.js'
import { v4 as uuidv4 } from '../universal/uuid.js' import { v4 as uuidv4 } from '../universal/uuid.js'
import { EntityManager, ManagedEntity } from './EntityManager.js'
/** /**
* Persistent entity that exists across files and evolves over time * Persistent entity that exists across files and evolves over time
*/ */
export interface PersistentEntity { export interface PersistentEntity extends ManagedEntity {
id: string id: string
name: string name: string
type: string // 'character', 'api', 'service', 'concept', 'customer', etc. type: string // 'character', 'api', 'service', 'concept', 'customer', etc.
@ -25,12 +26,13 @@ export interface PersistentEntity {
created: number created: number
lastUpdated: number lastUpdated: number
version: number version: number
entityType?: string // For EntityManager queries
} }
/** /**
* An appearance of an entity in a specific file/location * An appearance of an entity in a specific file/location
*/ */
export interface EntityAppearance { export interface EntityAppearance extends ManagedEntity {
id: string id: string
entityId: string entityId: string
filePath: string filePath: string
@ -44,6 +46,7 @@ export interface EntityAppearance {
version: number version: number
changes?: EntityChange[] changes?: EntityChange[]
confidence: number // How confident we are this is the entity (0-1) confidence: number // How confident we are this is the entity (0-1)
eventType?: string // For EntityManager queries
} }
/** /**
@ -78,14 +81,15 @@ export interface PersistentEntityConfig {
* - Business entities that appear in multiple reports * - Business entities that appear in multiple reports
* - Code classes/functions that span multiple files * - Code classes/functions that span multiple files
*/ */
export class PersistentEntitySystem { export class PersistentEntitySystem extends EntityManager {
private config: Required<PersistentEntityConfig> private config: Required<PersistentEntityConfig>
private entityCache = new Map<string, PersistentEntity>() private entityCache = new Map<string, PersistentEntity>()
constructor( constructor(
private brain: Brainy, brain: Brainy,
config?: PersistentEntityConfig config?: PersistentEntityConfig
) { ) {
super(brain, 'vfs-entity')
this.config = { this.config = {
autoExtract: config?.autoExtract ?? false, autoExtract: config?.autoExtract ?? false,
similarityThreshold: config?.similarityThreshold ?? 0.8, similarityThreshold: config?.similarityThreshold ?? 0.8,
@ -102,12 +106,17 @@ export class PersistentEntitySystem {
const timestamp = Date.now() const timestamp = Date.now()
const persistentEntity: PersistentEntity = { const persistentEntity: PersistentEntity = {
...entity,
id: entityId, id: entityId,
name: entity.name,
type: entity.type,
description: entity.description,
aliases: entity.aliases,
attributes: entity.attributes,
created: timestamp, created: timestamp,
lastUpdated: timestamp, lastUpdated: timestamp,
version: 1, version: 1,
appearances: [] appearances: [],
entityType: 'persistent' // Add entityType for querying
} }
// Generate embedding for entity description // Generate embedding for entity description
@ -120,22 +129,18 @@ export class PersistentEntitySystem {
console.warn('Failed to generate entity embedding:', error) console.warn('Failed to generate entity embedding:', error)
} }
// Store entity in Brainy // Store entity using EntityManager
const brainyEntity = await this.brain.add({ await this.storeEntity(
type: NounType.Concept, persistentEntity,
data: Buffer.from(JSON.stringify(persistentEntity)), NounType.Concept,
metadata: { embedding,
...persistentEntity, Buffer.from(JSON.stringify(persistentEntity))
entityType: 'persistent', )
system: 'vfs-entity'
},
vector: embedding
})
// Update cache // Update cache
this.entityCache.set(entityId, persistentEntity) this.entityCache.set(entityId, persistentEntity)
return brainyEntity return entityId // Return domain ID, not Brainy ID
} }
/** /**
@ -147,10 +152,7 @@ export class PersistentEntitySystem {
attributes?: Record<string, any> attributes?: Record<string, any>
similar?: string // Find similar entities to this description similar?: string // Find similar entities to this description
}): Promise<PersistentEntity[]> { }): Promise<PersistentEntity[]> {
const searchQuery: any = { const searchQuery: any = {}
entityType: 'persistent',
system: 'vfs-entity'
}
if (query.name) { if (query.name) {
// Search by exact name or aliases // Search by exact name or aliases
@ -170,42 +172,45 @@ export class PersistentEntitySystem {
} }
} }
// Search in Brainy // Add system metadata for EntityManager
let results = await this.brain.find({ searchQuery.entityType = 'persistent'
where: searchQuery,
type: NounType.Concept, // Search using EntityManager
limit: 100 let results = await this.findEntities<PersistentEntity>(searchQuery, NounType.Concept, 100)
})
// If searching for similar entities, use vector similarity // If searching for similar entities, use vector similarity
if (query.similar) { if (query.similar) {
try { try {
const queryEmbedding = await this.generateTextEmbedding(query.similar) const queryEmbedding = await this.generateTextEmbedding(query.similar)
if (queryEmbedding) { if (queryEmbedding) {
// Get all entities and rank by similarity // Get all entities and rank by similarity using EntityManager
const allEntities = await this.brain.find({ const allEntities = await this.findEntities<PersistentEntity>({}, NounType.Concept, 1000)
where: { entityType: 'persistent', system: 'vfs-entity' },
type: NounType.Concept,
limit: 1000
})
const withSimilarity = allEntities const withSimilarity = allEntities
.filter(e => e.entity.vector && e.entity.vector.length > 0) .filter(e => e.brainyId) // Only entities with brainyId have vectors
.map(e => ({ .map(async e => {
const brainyEntity = await this.brain.get(e.brainyId!)
if (brainyEntity?.vector) {
return {
entity: e, entity: e,
similarity: 1 - cosineDistance(queryEmbedding, e.entity.vector!) similarity: 1 - cosineDistance(queryEmbedding, brainyEntity.vector)
})) }
.filter(s => s.similarity > this.config.similarityThreshold) }
.sort((a, b) => b.similarity - a.similarity) return null
})
results = withSimilarity.map(s => s.entity) const resolved = (await Promise.all(withSimilarity))
.filter(s => s !== null && s.similarity > this.config.similarityThreshold)
.sort((a, b) => b!.similarity - a!.similarity)
results = resolved.map(s => s!.entity)
} }
} catch (error) { } catch (error) {
console.warn('Failed to perform similarity search:', error) console.warn('Failed to perform similarity search:', error)
} }
} }
return results.map(r => r.entity.metadata as PersistentEntity) return results
} }
/** /**
@ -221,7 +226,7 @@ export class PersistentEntitySystem {
extractChanges?: boolean extractChanges?: boolean
} }
): Promise<string> { ): Promise<string> {
const entity = await this.getEntity(entityId) const entity = await this.getPersistentEntity(entityId)
if (!entity) { if (!entity) {
throw new Error(`Entity ${entityId} not found`) throw new Error(`Entity ${entityId} not found`)
} }
@ -247,23 +252,25 @@ export class PersistentEntitySystem {
confidence: options?.confidence ?? 1.0 confidence: options?.confidence ?? 1.0
} }
// Store appearance as Brainy entity // Store appearance as managed entity (with eventType for appearances)
await this.brain.add({ const appearanceWithEventType = {
type: NounType.Event,
data: Buffer.from(context),
metadata: {
...appearance, ...appearance,
eventType: 'entity-appearance', eventType: 'entity-appearance'
system: 'vfs-entity'
} }
})
// Create relationship to entity await this.storeEntity(
await this.brain.relate({ appearanceWithEventType,
from: appearanceId, NounType.Event,
to: entityId, undefined,
type: VerbType.References Buffer.from(context)
}) )
// Create relationship to entity using EntityManager
await this.createRelationship(
appearanceId,
entityId,
VerbType.References
)
// Update entity with new appearance // Update entity with new appearance
entity.appearances.push(appearance) entity.appearances.push(appearance)
@ -287,7 +294,7 @@ export class PersistentEntitySystem {
} }
// Update stored entity // Update stored entity
await this.updateEntity(entity) await this.updatePersistentEntity(entity)
return appearanceId return appearanceId
} }
@ -304,7 +311,7 @@ export class PersistentEntitySystem {
appearance?: EntityAppearance appearance?: EntityAppearance
}> }>
}> { }> {
const entity = await this.getEntity(entityId) const entity = await this.getPersistentEntity(entityId)
if (!entity) { if (!entity) {
throw new Error(`Entity ${entityId} not found`) throw new Error(`Entity ${entityId} not found`)
} }
@ -377,7 +384,7 @@ export class PersistentEntitySystem {
source: string, source: string,
reason?: string reason?: string
): Promise<void> { ): Promise<void> {
const entity = await this.getEntity(entityId) const entity = await this.getPersistentEntity(entityId)
if (!entity) { if (!entity) {
throw new Error(`Entity ${entityId} not found`) throw new Error(`Entity ${entityId} not found`)
} }
@ -405,7 +412,7 @@ export class PersistentEntitySystem {
entity.version++ entity.version++
// Update stored entity // Update stored entity
await this.updateEntity(entity) await this.updatePersistentEntity(entity)
// Record evolution event // Record evolution event
if (changes.length > 0) { if (changes.length > 0) {
@ -443,7 +450,7 @@ export class PersistentEntitySystem {
// Character names (capitalized words) // Character names (capitalized words)
/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b/g, /\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b/g,
// API endpoints // API endpoints
/\/api\/[a-zA-Z0-9\/\-_]+/g, /\/api\/[a-zA-Z0-9/\-_]+/g,
// Class names // Class names
/class\s+([A-Z][a-zA-Z0-9_]*)/g, /class\s+([A-Z][a-zA-Z0-9_]*)/g,
// Function names // Function names
@ -471,7 +478,7 @@ export class PersistentEntitySystem {
} }
// Record appearance for existing or new entity // Record appearance for existing or new entity
const entity = existing[0] || await this.getEntity(entities[entities.length - 1]) const entity = existing[0] || await this.getPersistentEntity(entities[entities.length - 1])
if (entity) { if (entity) {
await this.recordAppearance( await this.recordAppearance(
entity.id, entity.id,
@ -524,38 +531,29 @@ export class PersistentEntitySystem {
} }
/** /**
* Get entity by ID * Get persistent entity by ID
*/ */
private async getEntity(entityId: string): Promise<PersistentEntity | null> { async getPersistentEntity(entityId: string): Promise<PersistentEntity | null> {
// Check cache first // Check cache first
if (this.entityCache.has(entityId)) { if (this.entityCache.has(entityId)) {
return this.entityCache.get(entityId)! return this.entityCache.get(entityId)!
} }
// Query from Brainy // Use parent getEntity method
const results = await this.brain.find({ const entity = await super.getEntity<PersistentEntity>(entityId)
where: {
id: entityId,
entityType: 'persistent',
system: 'vfs-entity'
},
type: NounType.Concept,
limit: 1
})
if (results.length === 0) { if (entity) {
return null // Cache and return
this.entityCache.set(entityId, entity)
} }
const entity = results[0].entity.metadata as PersistentEntity
this.entityCache.set(entityId, entity)
return entity return entity
} }
/** /**
* Update stored entity * Update stored entity (rename to avoid parent method conflict)
*/ */
private async updateEntity(entity: PersistentEntity): Promise<void> { private async updatePersistentEntity(entity: PersistentEntity): Promise<void> {
// Find the Brainy entity // Find the Brainy entity
const results = await this.brain.find({ const results = await this.brain.find({
where: { where: {

View file

@ -10,11 +10,12 @@ import { NounType, VerbType } from '../types/graphTypes.js'
import { cosineDistance } from '../utils/distance.js' import { cosineDistance } from '../utils/distance.js'
import { createHash } from 'crypto' import { createHash } from 'crypto'
import { v4 as uuidv4 } from '../universal/uuid.js' import { v4 as uuidv4 } from '../universal/uuid.js'
import { EntityManager, ManagedEntity } from './EntityManager.js'
/** /**
* Version metadata * Version metadata
*/ */
export interface Version { export interface Version extends ManagedEntity {
id: string id: string
path: string path: string
version: number version: number
@ -43,14 +44,15 @@ export interface SemanticVersioningConfig {
* Creates versions only when content meaning changes significantly * Creates versions only when content meaning changes significantly
* Uses vector embeddings to detect semantic changes * Uses vector embeddings to detect semantic changes
*/ */
export class SemanticVersioning { export class SemanticVersioning extends EntityManager {
private config: Required<SemanticVersioningConfig> private config: Required<SemanticVersioningConfig>
private versionCache = new Map<string, Version[]>() private versionCache = new Map<string, Version[]>()
constructor( constructor(
private brain: Brainy, brain: Brainy,
config?: SemanticVersioningConfig config?: SemanticVersioningConfig
) { ) {
super(brain, 'vfs-version')
this.config = { this.config = {
threshold: config?.threshold ?? 0.3, threshold: config?.threshold ?? 0.3,
maxVersions: config?.maxVersions ?? 10, maxVersions: config?.maxVersions ?? 10,
@ -128,11 +130,8 @@ export class SemanticVersioning {
console.warn('Failed to generate embedding for version:', error) console.warn('Failed to generate embedding for version:', error)
} }
// Store version as Brainy entity // Create version entity
const entity = await this.brain.add({ const version: Version = {
type: NounType.State,
data: content, // Store actual content
metadata: {
id: versionId, id: versionId,
path, path,
version: versionNumber, version: versionNumber,
@ -142,19 +141,20 @@ export class SemanticVersioning {
size: content.length, size: content.length,
author: metadata?.author, author: metadata?.author,
message: metadata?.message, message: metadata?.message,
parentVersion, parentVersion
system: 'vfs-version' }
} as Version,
vector: embedding // Store version using EntityManager (with actual content as data)
}) await this.storeEntity(version, NounType.State, embedding, content)
// Create relationship to parent version if exists // Create relationship to parent version if exists
if (parentVersion) { if (parentVersion) {
await this.brain.relate({ try {
from: entity, await this.createRelationship(versionId, parentVersion, VerbType.Succeeds)
to: parentVersion, } catch (error) {
type: VerbType.Succeeds console.warn(`Failed to create parent relationship for version ${versionId}:`, error)
}) // Continue without relationship - non-critical for version functionality
}
} }
// Update cache // Update cache
@ -189,19 +189,15 @@ export class SemanticVersioning {
return this.versionCache.get(path)! return this.versionCache.get(path)!
} }
// Query from Brainy // Query using EntityManager
const results = await this.brain.find({ const versions = await this.findEntities<Version>(
where: { { path },
path, NounType.State,
system: 'vfs-version' this.config.maxVersions * 2 // Get extra in case some are pruned
}, )
type: NounType.State,
limit: this.config.maxVersions * 2 // Get extra in case some are pruned
})
const versions = results // Sort by timestamp (newest first)
.map(r => r.entity.metadata as Version) versions.sort((a, b) => b.timestamp - a.timestamp)
.sort((a, b) => b.timestamp - a.timestamp) // Newest first
// Update cache // Update cache
this.versionCache.set(path, versions) this.versionCache.set(path, versions)
@ -213,21 +209,20 @@ export class SemanticVersioning {
* Get a specific version's content * Get a specific version's content
*/ */
async getVersion(path: string, versionId: string): Promise<Buffer | null> { async getVersion(path: string, versionId: string): Promise<Buffer | null> {
const results = await this.brain.find({ // Get the version entity
where: { const version = await this.getEntity<Version>(versionId)
id: versionId, if (!version || version.path !== path) {
path,
system: 'vfs-version'
},
type: NounType.State,
limit: 1
})
if (results.length === 0) {
return null return null
} }
return results[0].entity.data as Buffer // Get the content from Brainy using the Brainy ID
const brainyId = await this.getBrainyId(versionId)
if (!brainyId) {
return null
}
const entity = await this.brain.get(brainyId)
return entity?.data as Buffer | null
} }
/** /**
@ -319,7 +314,7 @@ export class SemanticVersioning {
// Delete excess versions // Delete excess versions
for (const id of toDelete.slice(0, versions.length - this.config.maxVersions)) { for (const id of toDelete.slice(0, versions.length - this.config.maxVersions)) {
await this.brain.delete(id) await this.deleteEntity(id)
} }
// Update cache // Update cache

View file

@ -2057,7 +2057,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
case 'mkdir': case 'mkdir':
await this.mkdir(op.path, op.options) await this.mkdir(op.path, op.options)
break break
case 'update': case 'update': {
// Update only metadata without changing content // Update only metadata without changing content
const entityId = await this.pathResolver.resolve(op.path) const entityId = await this.pathResolver.resolve(op.path)
await this.brain.update({ await this.brain.update({
@ -2066,6 +2066,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
}) })
break break
} }
}
result.successful++ result.successful++
} catch (error: any) { } catch (error: any) {
result.failed.push({ result.failed.push({