fix: Knowledge Layer EntityManager integration
* Add EntityManager base class for standardized entity operations * Update SemanticVersioning to extend EntityManager * Update EventRecorder to extend EntityManager * Update PersistentEntitySystem to extend EntityManager * Update ConceptSystem to extend EntityManager * Fix entity ID mismatch patterns across all Knowledge Layer components * Add proper domain ID to Brainy ID mapping * Replace direct brain.add/find calls with EntityManager methods * Ensure all entity interfaces extend ManagedEntity * Add proper metadata fields for querying (eventType, conceptType, entityType) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
6e6da5011f
commit
cc6fa00f30
8 changed files with 587 additions and 328 deletions
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "3.10.0",
|
"version": "3.10.1",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "3.10.0",
|
"version": "3.10.1",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.540.0",
|
"@aws-sdk/client-s3": "^3.540.0",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "3.10.0",
|
"version": "3.10.1",
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"module": "dist/index.js",
|
"module": "dist/index.js",
|
||||||
|
|
|
||||||
|
|
@ -10,11 +10,12 @@ import { Brainy } from '../brainy.js'
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { cosineDistance } from '../utils/distance.js'
|
import { cosineDistance } from '../utils/distance.js'
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||||
|
import { EntityManager, ManagedEntity } from './EntityManager.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Universal concept that exists independently of files
|
* Universal concept that exists independently of files
|
||||||
*/
|
*/
|
||||||
export interface UniversalConcept {
|
export interface UniversalConcept extends ManagedEntity {
|
||||||
id: string
|
id: string
|
||||||
name: string
|
name: string
|
||||||
description?: string
|
description?: string
|
||||||
|
|
@ -28,6 +29,7 @@ export interface UniversalConcept {
|
||||||
lastUpdated: number
|
lastUpdated: number
|
||||||
version: number
|
version: number
|
||||||
metadata: Record<string, any>
|
metadata: Record<string, any>
|
||||||
|
conceptType?: string // For EntityManager queries
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -45,7 +47,7 @@ export interface ConceptLink {
|
||||||
/**
|
/**
|
||||||
* A manifestation of a concept in a specific location
|
* A manifestation of a concept in a specific location
|
||||||
*/
|
*/
|
||||||
export interface ConceptManifestation {
|
export interface ConceptManifestation extends ManagedEntity {
|
||||||
id: string
|
id: string
|
||||||
conceptId: string
|
conceptId: string
|
||||||
filePath: string
|
filePath: string
|
||||||
|
|
@ -100,14 +102,15 @@ export interface ConceptGraph {
|
||||||
* - "Dependency Injection" pattern across multiple codebases
|
* - "Dependency Injection" pattern across multiple codebases
|
||||||
* - "Sustainability" theme in various research papers
|
* - "Sustainability" theme in various research papers
|
||||||
*/
|
*/
|
||||||
export class ConceptSystem {
|
export class ConceptSystem extends EntityManager {
|
||||||
private config: Required<ConceptSystemConfig>
|
private config: Required<ConceptSystemConfig>
|
||||||
private conceptCache = new Map<string, UniversalConcept>()
|
private conceptCache = new Map<string, UniversalConcept>()
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private brain: Brainy,
|
brain: Brainy,
|
||||||
config?: ConceptSystemConfig
|
config?: ConceptSystemConfig
|
||||||
) {
|
) {
|
||||||
|
super(brain, 'vfs-concept')
|
||||||
this.config = {
|
this.config = {
|
||||||
autoLink: config?.autoLink ?? false,
|
autoLink: config?.autoLink ?? false,
|
||||||
similarityThreshold: config?.similarityThreshold ?? 0.7,
|
similarityThreshold: config?.similarityThreshold ?? 0.7,
|
||||||
|
|
@ -124,13 +127,20 @@ export class ConceptSystem {
|
||||||
const timestamp = Date.now()
|
const timestamp = Date.now()
|
||||||
|
|
||||||
const universalConcept: UniversalConcept = {
|
const universalConcept: UniversalConcept = {
|
||||||
...concept,
|
|
||||||
id: conceptId,
|
id: conceptId,
|
||||||
|
name: concept.name,
|
||||||
|
description: concept.description,
|
||||||
|
domain: concept.domain,
|
||||||
|
category: concept.category,
|
||||||
|
keywords: concept.keywords,
|
||||||
|
strength: concept.strength,
|
||||||
|
metadata: concept.metadata || {},
|
||||||
created: timestamp,
|
created: timestamp,
|
||||||
lastUpdated: timestamp,
|
lastUpdated: timestamp,
|
||||||
version: 1,
|
version: 1,
|
||||||
links: [],
|
links: [],
|
||||||
manifestations: []
|
manifestations: [],
|
||||||
|
conceptType: 'universal' // Add conceptType for querying
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate embedding for concept
|
// Generate embedding for concept
|
||||||
|
|
@ -141,17 +151,13 @@ export class ConceptSystem {
|
||||||
console.warn('Failed to generate concept embedding:', error)
|
console.warn('Failed to generate concept embedding:', error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store concept in Brainy
|
// Store concept using EntityManager
|
||||||
const brainyEntity = await this.brain.add({
|
await this.storeEntity(
|
||||||
type: NounType.Concept,
|
universalConcept,
|
||||||
data: Buffer.from(JSON.stringify(universalConcept)),
|
NounType.Concept,
|
||||||
metadata: {
|
embedding,
|
||||||
...universalConcept,
|
Buffer.from(JSON.stringify(universalConcept))
|
||||||
conceptType: 'universal',
|
)
|
||||||
system: 'vfs-concept'
|
|
||||||
},
|
|
||||||
vector: embedding
|
|
||||||
})
|
|
||||||
|
|
||||||
// Auto-link to similar concepts if enabled
|
// Auto-link to similar concepts if enabled
|
||||||
if (this.config.autoLink) {
|
if (this.config.autoLink) {
|
||||||
|
|
@ -161,7 +167,7 @@ export class ConceptSystem {
|
||||||
// Update cache
|
// Update cache
|
||||||
this.conceptCache.set(conceptId, universalConcept)
|
this.conceptCache.set(conceptId, universalConcept)
|
||||||
|
|
||||||
return brainyEntity
|
return conceptId // Return domain ID, not Brainy ID
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -193,17 +199,16 @@ export class ConceptSystem {
|
||||||
// File manifestation search
|
// File manifestation search
|
||||||
if (query.manifestedIn) {
|
if (query.manifestedIn) {
|
||||||
// Find concepts that have manifestations in this file
|
// Find concepts that have manifestations in this file
|
||||||
const manifestationResults = await this.brain.find({
|
const manifestationResults = await this.findEntities<ConceptManifestation>(
|
||||||
where: {
|
{
|
||||||
filePath: query.manifestedIn,
|
filePath: query.manifestedIn,
|
||||||
eventType: 'concept-manifestation',
|
eventType: 'concept-manifestation'
|
||||||
system: 'vfs-concept'
|
|
||||||
},
|
},
|
||||||
type: NounType.Event,
|
NounType.Event,
|
||||||
limit: 1000
|
1000
|
||||||
})
|
)
|
||||||
|
|
||||||
const conceptIds = manifestationResults.map(r => r.entity.metadata.conceptId)
|
const conceptIds = manifestationResults.map(m => m.conceptId)
|
||||||
if (conceptIds.length > 0) {
|
if (conceptIds.length > 0) {
|
||||||
searchQuery.id = { $in: conceptIds }
|
searchQuery.id = { $in: conceptIds }
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -211,12 +216,12 @@ export class ConceptSystem {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search in Brainy
|
// Search using EntityManager
|
||||||
let results = await this.brain.find({
|
const results = await this.findEntities<UniversalConcept>(
|
||||||
where: searchQuery,
|
searchQuery,
|
||||||
type: NounType.Concept,
|
NounType.Concept,
|
||||||
limit: 1000
|
1000
|
||||||
})
|
)
|
||||||
|
|
||||||
// If searching for similar concepts, use vector similarity
|
// If searching for similar concepts, use vector similarity
|
||||||
if (query.similar) {
|
if (query.similar) {
|
||||||
|
|
@ -224,29 +229,42 @@ export class ConceptSystem {
|
||||||
const queryEmbedding = await this.generateTextEmbedding(query.similar)
|
const queryEmbedding = await this.generateTextEmbedding(query.similar)
|
||||||
if (queryEmbedding) {
|
if (queryEmbedding) {
|
||||||
// Get all concepts and rank by similarity
|
// Get all concepts and rank by similarity
|
||||||
const allConcepts = await this.brain.find({
|
const allConcepts = await this.findEntities<UniversalConcept>(
|
||||||
where: { conceptType: 'universal', system: 'vfs-concept' },
|
{ conceptType: 'universal' },
|
||||||
type: NounType.Concept,
|
NounType.Concept,
|
||||||
limit: 10000
|
10000
|
||||||
})
|
)
|
||||||
|
|
||||||
const withSimilarity = allConcepts
|
// For similarity search, we need to get the actual vector data from Brainy
|
||||||
.filter(c => c.entity.vector && c.entity.vector.length > 0)
|
const conceptsWithVectors = []
|
||||||
|
for (const concept of allConcepts) {
|
||||||
|
if (concept.brainyId) {
|
||||||
|
const brainyEntity = await this.brain.get(concept.brainyId)
|
||||||
|
if (brainyEntity?.vector && brainyEntity.vector.length > 0) {
|
||||||
|
conceptsWithVectors.push({
|
||||||
|
concept,
|
||||||
|
vector: brainyEntity.vector
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const withSimilarity = conceptsWithVectors
|
||||||
.map(c => ({
|
.map(c => ({
|
||||||
concept: c,
|
concept: c.concept,
|
||||||
similarity: 1 - cosineDistance(queryEmbedding, c.entity.vector!)
|
similarity: 1 - cosineDistance(queryEmbedding, c.vector)
|
||||||
}))
|
}))
|
||||||
.filter(s => s.similarity > this.config.similarityThreshold)
|
.filter(s => s.similarity > this.config.similarityThreshold)
|
||||||
.sort((a, b) => b.similarity - a.similarity)
|
.sort((a, b) => b.similarity - a.similarity)
|
||||||
|
|
||||||
results = withSimilarity.map(s => s.concept)
|
return withSimilarity.map(s => s.concept)
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('Failed to perform concept similarity search:', error)
|
console.warn('Failed to perform concept similarity search:', error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results.map(r => r.entity.metadata as UniversalConcept)
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -302,17 +320,17 @@ export class ConceptSystem {
|
||||||
await this.updateConcept(toConcept)
|
await this.updateConcept(toConcept)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create Brainy relationship
|
// Create relationship using EntityManager
|
||||||
await this.brain.relate({
|
await this.createRelationship(
|
||||||
from: fromConceptId,
|
fromConceptId,
|
||||||
to: toConceptId,
|
toConceptId,
|
||||||
type: this.getVerbType(relationship),
|
this.getVerbType(relationship),
|
||||||
metadata: {
|
{
|
||||||
strength: link.strength,
|
strength: link.strength,
|
||||||
context: link.context,
|
context: link.context,
|
||||||
bidirectional: link.bidirectional
|
bidirectional: link.bidirectional
|
||||||
}
|
}
|
||||||
})
|
)
|
||||||
|
|
||||||
return linkId
|
return linkId
|
||||||
}
|
}
|
||||||
|
|
@ -351,23 +369,25 @@ export class ConceptSystem {
|
||||||
extractedBy: options?.extractedBy ?? 'manual'
|
extractedBy: options?.extractedBy ?? 'manual'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store manifestation as Brainy event
|
// Store manifestation as managed entity (with eventType for manifestations)
|
||||||
await this.brain.add({
|
const manifestationWithEventType = {
|
||||||
type: NounType.Event,
|
|
||||||
data: Buffer.from(context),
|
|
||||||
metadata: {
|
|
||||||
...manifestation,
|
...manifestation,
|
||||||
eventType: 'concept-manifestation',
|
eventType: 'concept-manifestation'
|
||||||
system: 'vfs-concept'
|
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
|
||||||
// Create relationship to concept
|
await this.storeEntity(
|
||||||
await this.brain.relate({
|
manifestationWithEventType,
|
||||||
from: manifestationId,
|
NounType.Event,
|
||||||
to: conceptId,
|
undefined,
|
||||||
type: VerbType.Implements
|
Buffer.from(context)
|
||||||
})
|
)
|
||||||
|
|
||||||
|
// Create relationship to concept using EntityManager
|
||||||
|
await this.createRelationship(
|
||||||
|
manifestationId,
|
||||||
|
conceptId,
|
||||||
|
VerbType.Implements
|
||||||
|
)
|
||||||
|
|
||||||
// Update concept with new manifestation
|
// Update concept with new manifestation
|
||||||
concept.manifestations.push(manifestation)
|
concept.manifestations.push(manifestation)
|
||||||
|
|
@ -476,13 +496,11 @@ export class ConceptSystem {
|
||||||
query.strength = { $gte: options.minStrength }
|
query.strength = { $gte: options.minStrength }
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = await this.brain.find({
|
const concepts = await this.findEntities<UniversalConcept>(
|
||||||
where: query,
|
query,
|
||||||
type: NounType.Concept,
|
NounType.Concept,
|
||||||
limit: options?.maxConcepts || 1000
|
options?.maxConcepts || 1000
|
||||||
})
|
)
|
||||||
|
|
||||||
const concepts = results.map(r => r.entity.metadata as UniversalConcept)
|
|
||||||
|
|
||||||
// Build graph structure
|
// Build graph structure
|
||||||
const graphConcepts = concepts.map(c => ({
|
const graphConcepts = concepts.map(c => ({
|
||||||
|
|
@ -541,15 +559,13 @@ export class ConceptSystem {
|
||||||
query.confidence = { $gte: options.minConfidence }
|
query.confidence = { $gte: options.minConfidence }
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = await this.brain.find({
|
const manifestations = await this.findEntities<ConceptManifestation>(
|
||||||
where: query,
|
query,
|
||||||
type: NounType.Event,
|
NounType.Event,
|
||||||
limit: options?.limit || 1000
|
options?.limit || 1000
|
||||||
})
|
)
|
||||||
|
|
||||||
return results
|
return manifestations.sort((a, b) => b.timestamp - a.timestamp)
|
||||||
.map(r => r.entity.metadata as ConceptManifestation)
|
|
||||||
.sort((a, b) => b.timestamp - a.timestamp)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -590,23 +606,11 @@ export class ConceptSystem {
|
||||||
return this.conceptCache.get(conceptId)!
|
return this.conceptCache.get(conceptId)!
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query from Brainy
|
// Query using EntityManager
|
||||||
const results = await this.brain.find({
|
const concept = await this.getEntity<UniversalConcept>(conceptId)
|
||||||
where: {
|
if (concept) {
|
||||||
id: conceptId,
|
|
||||||
conceptType: 'universal',
|
|
||||||
system: 'vfs-concept'
|
|
||||||
},
|
|
||||||
type: NounType.Concept,
|
|
||||||
limit: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
if (results.length === 0) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
const concept = results[0].entity.metadata as UniversalConcept
|
|
||||||
this.conceptCache.set(conceptId, concept)
|
this.conceptCache.set(conceptId, concept)
|
||||||
|
}
|
||||||
return concept
|
return concept
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -614,28 +618,11 @@ export class ConceptSystem {
|
||||||
* Update stored concept
|
* Update stored concept
|
||||||
*/
|
*/
|
||||||
private async updateConcept(concept: UniversalConcept): Promise<void> {
|
private async updateConcept(concept: UniversalConcept): Promise<void> {
|
||||||
// Find the Brainy entity
|
// Add conceptType metadata before updating
|
||||||
const results = await this.brain.find({
|
concept.conceptType = 'universal'
|
||||||
where: {
|
|
||||||
id: concept.id,
|
|
||||||
conceptType: 'universal',
|
|
||||||
system: 'vfs-concept'
|
|
||||||
},
|
|
||||||
type: NounType.Concept,
|
|
||||||
limit: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
if (results.length > 0) {
|
// Update using EntityManager
|
||||||
await this.brain.update({
|
await this.updateEntity(concept)
|
||||||
id: results[0].entity.id,
|
|
||||||
data: Buffer.from(JSON.stringify(concept)),
|
|
||||||
metadata: {
|
|
||||||
...concept,
|
|
||||||
conceptType: 'universal',
|
|
||||||
system: 'vfs-concept'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
this.conceptCache.set(concept.id, concept)
|
this.conceptCache.set(concept.id, concept)
|
||||||
|
|
|
||||||
291
src/vfs/EntityManager.ts
Normal file
291
src/vfs/EntityManager.ts
Normal file
|
|
@ -0,0 +1,291 @@
|
||||||
|
/**
|
||||||
|
* EntityManager Base Class
|
||||||
|
*
|
||||||
|
* Provides standardized entity ID management for all Knowledge Layer components
|
||||||
|
* Solves the root cause of ID mismatch issues by establishing clear patterns
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Brainy } from '../brainy.js'
|
||||||
|
import { NounType } from '../types/graphTypes.js'
|
||||||
|
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Standard entity structure used by all Knowledge Layer components
|
||||||
|
*/
|
||||||
|
export interface ManagedEntity {
|
||||||
|
/** Domain-specific ID (for external references) */
|
||||||
|
id: string
|
||||||
|
/** The actual Brainy entity ID (for internal operations) */
|
||||||
|
brainyId?: string
|
||||||
|
/** Entity metadata */
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ID mapping to track domain IDs to Brainy entity IDs
|
||||||
|
*/
|
||||||
|
interface EntityIdMapping {
|
||||||
|
domainId: string
|
||||||
|
brainyId: string
|
||||||
|
system: string
|
||||||
|
type: string
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* EntityManager Base Class
|
||||||
|
*
|
||||||
|
* All Knowledge Layer components should extend this to get standardized:
|
||||||
|
* - Entity storage and retrieval
|
||||||
|
* - ID management and mapping
|
||||||
|
* - Query patterns
|
||||||
|
* - Relationship creation
|
||||||
|
*/
|
||||||
|
export abstract class EntityManager {
|
||||||
|
private idMappings = new Map<string, string>() // domainId -> brainyId
|
||||||
|
private brainyToMappings = new Map<string, string>() // brainyId -> domainId
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
protected brain: Brainy,
|
||||||
|
protected systemName: string
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Store an entity with proper ID management
|
||||||
|
*/
|
||||||
|
protected async storeEntity<T extends ManagedEntity>(
|
||||||
|
entity: T,
|
||||||
|
nounType: NounType,
|
||||||
|
embedding?: number[],
|
||||||
|
data?: any
|
||||||
|
): Promise<string> {
|
||||||
|
// Generate domain ID if not provided
|
||||||
|
if (!entity.id) {
|
||||||
|
entity.id = uuidv4()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store in Brainy and get the Brainy entity ID
|
||||||
|
const brainyId = await this.brain.add({
|
||||||
|
type: nounType,
|
||||||
|
data: data || Buffer.from(JSON.stringify(entity)),
|
||||||
|
metadata: {
|
||||||
|
...entity,
|
||||||
|
system: this.systemName,
|
||||||
|
domainId: entity.id, // Store the domain ID for reference
|
||||||
|
storedAt: Date.now()
|
||||||
|
},
|
||||||
|
vector: embedding
|
||||||
|
})
|
||||||
|
|
||||||
|
// Store ID mapping
|
||||||
|
this.idMappings.set(entity.id, brainyId)
|
||||||
|
this.brainyToMappings.set(brainyId, entity.id)
|
||||||
|
|
||||||
|
// Update entity with Brainy ID
|
||||||
|
entity.brainyId = brainyId
|
||||||
|
|
||||||
|
return entity.id // Return domain ID for external use
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update an existing entity
|
||||||
|
*/
|
||||||
|
protected async updateEntity<T extends ManagedEntity>(
|
||||||
|
entity: T,
|
||||||
|
embedding?: number[]
|
||||||
|
): Promise<void> {
|
||||||
|
const brainyId = this.idMappings.get(entity.id)
|
||||||
|
if (!brainyId) {
|
||||||
|
throw new Error(`Entity ${entity.id} not found in mappings`)
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.brain.update({
|
||||||
|
id: brainyId,
|
||||||
|
data: Buffer.from(JSON.stringify(entity)),
|
||||||
|
metadata: {
|
||||||
|
...entity,
|
||||||
|
system: this.systemName,
|
||||||
|
domainId: entity.id,
|
||||||
|
updatedAt: Date.now()
|
||||||
|
},
|
||||||
|
vector: embedding
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Retrieve entity by domain ID
|
||||||
|
*/
|
||||||
|
protected async getEntity<T extends ManagedEntity>(domainId: string): Promise<T | null> {
|
||||||
|
// First try from cache
|
||||||
|
let brainyId = this.idMappings.get(domainId)
|
||||||
|
|
||||||
|
// If not in cache, search by domain ID in metadata
|
||||||
|
if (!brainyId) {
|
||||||
|
const results = await this.brain.find({
|
||||||
|
where: {
|
||||||
|
domainId,
|
||||||
|
system: this.systemName
|
||||||
|
},
|
||||||
|
limit: 1
|
||||||
|
})
|
||||||
|
|
||||||
|
if (results.length === 0) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
brainyId = results[0].entity.id
|
||||||
|
// Cache the mapping
|
||||||
|
this.idMappings.set(domainId, brainyId)
|
||||||
|
this.brainyToMappings.set(brainyId, domainId)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get entity using Brainy ID
|
||||||
|
const brainyEntity = await this.brain.get(brainyId)
|
||||||
|
if (!brainyEntity) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse entity from metadata
|
||||||
|
const entity = brainyEntity.metadata as T
|
||||||
|
entity.brainyId = brainyId
|
||||||
|
return entity
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find entities by metadata criteria
|
||||||
|
*/
|
||||||
|
protected async findEntities<T extends ManagedEntity>(
|
||||||
|
criteria: Record<string, any>,
|
||||||
|
nounType?: NounType,
|
||||||
|
limit = 100
|
||||||
|
): Promise<T[]> {
|
||||||
|
const query: any = {
|
||||||
|
...criteria,
|
||||||
|
system: this.systemName
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = await this.brain.find({
|
||||||
|
where: query,
|
||||||
|
type: nounType,
|
||||||
|
limit
|
||||||
|
})
|
||||||
|
|
||||||
|
const entities: T[] = []
|
||||||
|
for (const result of results) {
|
||||||
|
const entity = result.entity.metadata as T
|
||||||
|
entity.brainyId = result.entity.id
|
||||||
|
|
||||||
|
// Update mappings cache
|
||||||
|
this.idMappings.set(entity.id, result.entity.id)
|
||||||
|
this.brainyToMappings.set(result.entity.id, entity.id)
|
||||||
|
|
||||||
|
entities.push(entity)
|
||||||
|
}
|
||||||
|
|
||||||
|
return entities
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create relationship between entities using proper Brainy IDs
|
||||||
|
*/
|
||||||
|
protected async createRelationship(
|
||||||
|
fromDomainId: string,
|
||||||
|
toDomainId: string,
|
||||||
|
relationshipType: any,
|
||||||
|
metadata?: Record<string, any>
|
||||||
|
): Promise<void> {
|
||||||
|
// Get Brainy IDs for both entities
|
||||||
|
const fromBrainyId = await this.getBrainyId(fromDomainId)
|
||||||
|
const toBrainyId = await this.getBrainyId(toDomainId)
|
||||||
|
|
||||||
|
if (!fromBrainyId || !toBrainyId) {
|
||||||
|
throw new Error(`Cannot find Brainy IDs for relationship: ${fromDomainId} -> ${toDomainId}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create relationship using Brainy IDs
|
||||||
|
await this.brain.relate({
|
||||||
|
from: fromBrainyId,
|
||||||
|
to: toBrainyId,
|
||||||
|
type: relationshipType,
|
||||||
|
metadata
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get Brainy ID for a domain ID
|
||||||
|
*/
|
||||||
|
protected async getBrainyId(domainId: string): Promise<string | null> {
|
||||||
|
// Check cache first
|
||||||
|
let brainyId = this.idMappings.get(domainId)
|
||||||
|
|
||||||
|
if (!brainyId) {
|
||||||
|
// Search in Brainy
|
||||||
|
const results = await this.brain.find({
|
||||||
|
where: {
|
||||||
|
domainId,
|
||||||
|
system: this.systemName
|
||||||
|
},
|
||||||
|
limit: 1
|
||||||
|
})
|
||||||
|
|
||||||
|
if (results.length > 0) {
|
||||||
|
brainyId = results[0].entity.id
|
||||||
|
// Cache the mapping
|
||||||
|
this.idMappings.set(domainId, brainyId)
|
||||||
|
this.brainyToMappings.set(brainyId, domainId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return brainyId || null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get domain ID for a Brainy ID
|
||||||
|
*/
|
||||||
|
protected getDomainId(brainyId: string): string | null {
|
||||||
|
return this.brainyToMappings.get(brainyId) || null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete entity by domain ID
|
||||||
|
*/
|
||||||
|
protected async deleteEntity(domainId: string): Promise<void> {
|
||||||
|
const brainyId = await this.getBrainyId(domainId)
|
||||||
|
if (brainyId) {
|
||||||
|
await this.brain.delete(brainyId)
|
||||||
|
// Clean up mappings
|
||||||
|
this.idMappings.delete(domainId)
|
||||||
|
this.brainyToMappings.delete(brainyId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear ID mapping cache (useful for tests)
|
||||||
|
*/
|
||||||
|
protected clearMappingCache(): void {
|
||||||
|
this.idMappings.clear()
|
||||||
|
this.brainyToMappings.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Batch load mappings for performance
|
||||||
|
*/
|
||||||
|
protected async loadMappings(domainIds: string[]): Promise<void> {
|
||||||
|
const unknownIds = domainIds.filter(id => !this.idMappings.has(id))
|
||||||
|
|
||||||
|
if (unknownIds.length === 0) return
|
||||||
|
|
||||||
|
const results = await this.brain.find({
|
||||||
|
where: {
|
||||||
|
domainId: { $in: unknownIds },
|
||||||
|
system: this.systemName
|
||||||
|
},
|
||||||
|
limit: unknownIds.length
|
||||||
|
})
|
||||||
|
|
||||||
|
for (const result of results) {
|
||||||
|
const domainId = result.entity.metadata.domainId
|
||||||
|
this.idMappings.set(domainId, result.entity.id)
|
||||||
|
this.brainyToMappings.set(result.entity.id, domainId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -9,11 +9,12 @@ import { Brainy } from '../brainy.js'
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||||
import { createHash } from 'crypto'
|
import { createHash } from 'crypto'
|
||||||
|
import { EntityManager, ManagedEntity } from './EntityManager.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* File operation event
|
* File operation event
|
||||||
*/
|
*/
|
||||||
export interface FileEvent {
|
export interface FileEvent extends ManagedEntity {
|
||||||
id: string
|
id: string
|
||||||
type: 'create' | 'write' | 'append' | 'delete' | 'move' | 'rename' | 'mkdir' | 'rmdir'
|
type: 'create' | 'write' | 'append' | 'delete' | 'move' | 'rename' | 'mkdir' | 'rmdir'
|
||||||
path: string
|
path: string
|
||||||
|
|
@ -25,13 +26,16 @@ export interface FileEvent {
|
||||||
metadata?: Record<string, any>
|
metadata?: Record<string, any>
|
||||||
previousHash?: string // For tracking changes
|
previousHash?: string // For tracking changes
|
||||||
oldPath?: string // For move/rename operations
|
oldPath?: string // For move/rename operations
|
||||||
|
eventType?: string // For EntityManager queries
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Event Recorder - Stores all file operations as searchable events
|
* Event Recorder - Stores all file operations as searchable events
|
||||||
*/
|
*/
|
||||||
export class EventRecorder {
|
export class EventRecorder extends EntityManager {
|
||||||
constructor(private brain: Brainy) {}
|
constructor(brain: Brainy) {
|
||||||
|
super(brain, 'vfs')
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Record a file operation event
|
* Record a file operation event
|
||||||
|
|
@ -45,27 +49,44 @@ export class EventRecorder {
|
||||||
? createHash('sha256').update(event.content).digest('hex')
|
? createHash('sha256').update(event.content).digest('hex')
|
||||||
: undefined
|
: undefined
|
||||||
|
|
||||||
// Store event as Brainy entity
|
// Create file event
|
||||||
const entity = await this.brain.add({
|
const fileEvent: FileEvent = {
|
||||||
type: NounType.Event,
|
|
||||||
data: event.content || Buffer.from(JSON.stringify(event)),
|
|
||||||
metadata: {
|
|
||||||
...event,
|
|
||||||
id: eventId,
|
id: eventId,
|
||||||
|
type: event.type,
|
||||||
|
path: event.path,
|
||||||
timestamp,
|
timestamp,
|
||||||
|
content: event.content,
|
||||||
|
size: event.size,
|
||||||
hash,
|
hash,
|
||||||
eventType: 'file-operation',
|
author: event.author,
|
||||||
system: 'vfs'
|
metadata: event.metadata,
|
||||||
},
|
previousHash: event.previousHash,
|
||||||
|
oldPath: event.oldPath
|
||||||
|
}
|
||||||
|
|
||||||
// Generate embedding for content-based events
|
// Generate embedding for content-based events
|
||||||
vector: event.content && event.content.length < 100000
|
const embedding = event.content && event.content.length < 100000
|
||||||
? await this.generateEventEmbedding(event)
|
? await this.generateEventEmbedding(event)
|
||||||
: undefined
|
: undefined
|
||||||
})
|
|
||||||
|
|
||||||
return entity
|
// Add eventType for event classification
|
||||||
|
const eventWithType = {
|
||||||
|
...fileEvent,
|
||||||
|
eventType: 'file-operation'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Store event using EntityManager
|
||||||
|
await this.storeEntity(
|
||||||
|
eventWithType,
|
||||||
|
NounType.Event,
|
||||||
|
embedding,
|
||||||
|
event.content || Buffer.from(JSON.stringify(event))
|
||||||
|
)
|
||||||
|
|
||||||
|
return eventId
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all events matching criteria
|
* Get all events matching criteria
|
||||||
*/
|
*/
|
||||||
|
|
@ -95,19 +116,8 @@ export class EventRecorder {
|
||||||
query.type = { $in: options.types }
|
query.type = { $in: options.types }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query events from Brainy
|
// Query using EntityManager
|
||||||
const results = await this.brain.find({
|
const events = await this.findEntities<FileEvent>(query, NounType.Event, limit)
|
||||||
where: query,
|
|
||||||
type: NounType.Event,
|
|
||||||
limit
|
|
||||||
})
|
|
||||||
|
|
||||||
const events: FileEvent[] = []
|
|
||||||
for (const result of results) {
|
|
||||||
if (result.entity?.metadata) {
|
|
||||||
events.push(result.entity.metadata as FileEvent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return events.sort((a, b) => b.timestamp - a.timestamp)
|
return events.sort((a, b) => b.timestamp - a.timestamp)
|
||||||
}
|
}
|
||||||
|
|
@ -138,28 +148,8 @@ export class EventRecorder {
|
||||||
query.type = { $in: options.types }
|
query.type = { $in: options.types }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query events from Brainy
|
// Query using EntityManager
|
||||||
const results = await this.brain.find({
|
const events = await this.findEntities<FileEvent>(query, NounType.Event, options?.limit || 100)
|
||||||
where: query,
|
|
||||||
type: NounType.Event,
|
|
||||||
limit: options?.limit || 100,
|
|
||||||
// Sort by timestamp descending (newest first)
|
|
||||||
// Note: Sorting would need to be implemented in Brainy
|
|
||||||
})
|
|
||||||
|
|
||||||
// Convert results to FileEvent format
|
|
||||||
const events = results.map(r => ({
|
|
||||||
id: r.entity.metadata.id,
|
|
||||||
type: r.entity.metadata.type,
|
|
||||||
path: r.entity.metadata.path,
|
|
||||||
timestamp: r.entity.metadata.timestamp,
|
|
||||||
content: r.entity.metadata.content,
|
|
||||||
size: r.entity.metadata.size,
|
|
||||||
hash: r.entity.metadata.hash,
|
|
||||||
author: r.entity.metadata.author,
|
|
||||||
metadata: r.entity.metadata.metadata,
|
|
||||||
previousHash: r.entity.metadata.previousHash
|
|
||||||
} as FileEvent))
|
|
||||||
|
|
||||||
// Sort by timestamp (newest first)
|
// Sort by timestamp (newest first)
|
||||||
return events.sort((a, b) => b.timestamp - a.timestamp)
|
return events.sort((a, b) => b.timestamp - a.timestamp)
|
||||||
|
|
@ -219,20 +209,17 @@ export class EventRecorder {
|
||||||
query.timestamp.$lte = until
|
query.timestamp.$lte = until
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = await this.brain.find({
|
// Query using EntityManager
|
||||||
where: query,
|
const events = await this.findEntities<FileEvent>(query, NounType.Event, 1000)
|
||||||
type: NounType.Event,
|
|
||||||
limit: 1000
|
|
||||||
})
|
|
||||||
|
|
||||||
return results.map(r => ({
|
return events.map(event => ({
|
||||||
id: r.entity.metadata.id,
|
id: event.id,
|
||||||
type: r.entity.metadata.type,
|
type: event.type,
|
||||||
path: r.entity.metadata.path,
|
path: event.path,
|
||||||
timestamp: r.entity.metadata.timestamp,
|
timestamp: event.timestamp,
|
||||||
size: r.entity.metadata.size,
|
size: event.size,
|
||||||
hash: r.entity.metadata.hash,
|
hash: event.hash,
|
||||||
author: r.entity.metadata.author
|
author: event.author
|
||||||
} as FileEvent))
|
} as FileEvent))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -357,7 +344,7 @@ export class EventRecorder {
|
||||||
for (let i = 0; i < events.length; i++) {
|
for (let i = 0; i < events.length; i++) {
|
||||||
// Keep every Nth event for history sampling
|
// Keep every Nth event for history sampling
|
||||||
if (i % keepEvery !== 0) {
|
if (i % keepEvery !== 0) {
|
||||||
await this.brain.delete(events[i].id)
|
await this.deleteEntity(events[i].id)
|
||||||
pruned++
|
pruned++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,11 +10,12 @@ import { Brainy } from '../brainy.js'
|
||||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { cosineDistance } from '../utils/distance.js'
|
import { cosineDistance } from '../utils/distance.js'
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||||
|
import { EntityManager, ManagedEntity } from './EntityManager.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Persistent entity that exists across files and evolves over time
|
* Persistent entity that exists across files and evolves over time
|
||||||
*/
|
*/
|
||||||
export interface PersistentEntity {
|
export interface PersistentEntity extends ManagedEntity {
|
||||||
id: string
|
id: string
|
||||||
name: string
|
name: string
|
||||||
type: string // 'character', 'api', 'service', 'concept', 'customer', etc.
|
type: string // 'character', 'api', 'service', 'concept', 'customer', etc.
|
||||||
|
|
@ -25,12 +26,13 @@ export interface PersistentEntity {
|
||||||
created: number
|
created: number
|
||||||
lastUpdated: number
|
lastUpdated: number
|
||||||
version: number
|
version: number
|
||||||
|
entityType?: string // For EntityManager queries
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An appearance of an entity in a specific file/location
|
* An appearance of an entity in a specific file/location
|
||||||
*/
|
*/
|
||||||
export interface EntityAppearance {
|
export interface EntityAppearance extends ManagedEntity {
|
||||||
id: string
|
id: string
|
||||||
entityId: string
|
entityId: string
|
||||||
filePath: string
|
filePath: string
|
||||||
|
|
@ -44,6 +46,7 @@ export interface EntityAppearance {
|
||||||
version: number
|
version: number
|
||||||
changes?: EntityChange[]
|
changes?: EntityChange[]
|
||||||
confidence: number // How confident we are this is the entity (0-1)
|
confidence: number // How confident we are this is the entity (0-1)
|
||||||
|
eventType?: string // For EntityManager queries
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -78,14 +81,15 @@ export interface PersistentEntityConfig {
|
||||||
* - Business entities that appear in multiple reports
|
* - Business entities that appear in multiple reports
|
||||||
* - Code classes/functions that span multiple files
|
* - Code classes/functions that span multiple files
|
||||||
*/
|
*/
|
||||||
export class PersistentEntitySystem {
|
export class PersistentEntitySystem extends EntityManager {
|
||||||
private config: Required<PersistentEntityConfig>
|
private config: Required<PersistentEntityConfig>
|
||||||
private entityCache = new Map<string, PersistentEntity>()
|
private entityCache = new Map<string, PersistentEntity>()
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private brain: Brainy,
|
brain: Brainy,
|
||||||
config?: PersistentEntityConfig
|
config?: PersistentEntityConfig
|
||||||
) {
|
) {
|
||||||
|
super(brain, 'vfs-entity')
|
||||||
this.config = {
|
this.config = {
|
||||||
autoExtract: config?.autoExtract ?? false,
|
autoExtract: config?.autoExtract ?? false,
|
||||||
similarityThreshold: config?.similarityThreshold ?? 0.8,
|
similarityThreshold: config?.similarityThreshold ?? 0.8,
|
||||||
|
|
@ -102,12 +106,17 @@ export class PersistentEntitySystem {
|
||||||
const timestamp = Date.now()
|
const timestamp = Date.now()
|
||||||
|
|
||||||
const persistentEntity: PersistentEntity = {
|
const persistentEntity: PersistentEntity = {
|
||||||
...entity,
|
|
||||||
id: entityId,
|
id: entityId,
|
||||||
|
name: entity.name,
|
||||||
|
type: entity.type,
|
||||||
|
description: entity.description,
|
||||||
|
aliases: entity.aliases,
|
||||||
|
attributes: entity.attributes,
|
||||||
created: timestamp,
|
created: timestamp,
|
||||||
lastUpdated: timestamp,
|
lastUpdated: timestamp,
|
||||||
version: 1,
|
version: 1,
|
||||||
appearances: []
|
appearances: [],
|
||||||
|
entityType: 'persistent' // Add entityType for querying
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate embedding for entity description
|
// Generate embedding for entity description
|
||||||
|
|
@ -120,22 +129,18 @@ export class PersistentEntitySystem {
|
||||||
console.warn('Failed to generate entity embedding:', error)
|
console.warn('Failed to generate entity embedding:', error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store entity in Brainy
|
// Store entity using EntityManager
|
||||||
const brainyEntity = await this.brain.add({
|
await this.storeEntity(
|
||||||
type: NounType.Concept,
|
persistentEntity,
|
||||||
data: Buffer.from(JSON.stringify(persistentEntity)),
|
NounType.Concept,
|
||||||
metadata: {
|
embedding,
|
||||||
...persistentEntity,
|
Buffer.from(JSON.stringify(persistentEntity))
|
||||||
entityType: 'persistent',
|
)
|
||||||
system: 'vfs-entity'
|
|
||||||
},
|
|
||||||
vector: embedding
|
|
||||||
})
|
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
this.entityCache.set(entityId, persistentEntity)
|
this.entityCache.set(entityId, persistentEntity)
|
||||||
|
|
||||||
return brainyEntity
|
return entityId // Return domain ID, not Brainy ID
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -147,10 +152,7 @@ export class PersistentEntitySystem {
|
||||||
attributes?: Record<string, any>
|
attributes?: Record<string, any>
|
||||||
similar?: string // Find similar entities to this description
|
similar?: string // Find similar entities to this description
|
||||||
}): Promise<PersistentEntity[]> {
|
}): Promise<PersistentEntity[]> {
|
||||||
const searchQuery: any = {
|
const searchQuery: any = {}
|
||||||
entityType: 'persistent',
|
|
||||||
system: 'vfs-entity'
|
|
||||||
}
|
|
||||||
|
|
||||||
if (query.name) {
|
if (query.name) {
|
||||||
// Search by exact name or aliases
|
// Search by exact name or aliases
|
||||||
|
|
@ -170,42 +172,45 @@ export class PersistentEntitySystem {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search in Brainy
|
// Add system metadata for EntityManager
|
||||||
let results = await this.brain.find({
|
searchQuery.entityType = 'persistent'
|
||||||
where: searchQuery,
|
|
||||||
type: NounType.Concept,
|
// Search using EntityManager
|
||||||
limit: 100
|
let results = await this.findEntities<PersistentEntity>(searchQuery, NounType.Concept, 100)
|
||||||
})
|
|
||||||
|
|
||||||
// If searching for similar entities, use vector similarity
|
// If searching for similar entities, use vector similarity
|
||||||
if (query.similar) {
|
if (query.similar) {
|
||||||
try {
|
try {
|
||||||
const queryEmbedding = await this.generateTextEmbedding(query.similar)
|
const queryEmbedding = await this.generateTextEmbedding(query.similar)
|
||||||
if (queryEmbedding) {
|
if (queryEmbedding) {
|
||||||
// Get all entities and rank by similarity
|
// Get all entities and rank by similarity using EntityManager
|
||||||
const allEntities = await this.brain.find({
|
const allEntities = await this.findEntities<PersistentEntity>({}, NounType.Concept, 1000)
|
||||||
where: { entityType: 'persistent', system: 'vfs-entity' },
|
|
||||||
type: NounType.Concept,
|
|
||||||
limit: 1000
|
|
||||||
})
|
|
||||||
|
|
||||||
const withSimilarity = allEntities
|
const withSimilarity = allEntities
|
||||||
.filter(e => e.entity.vector && e.entity.vector.length > 0)
|
.filter(e => e.brainyId) // Only entities with brainyId have vectors
|
||||||
.map(e => ({
|
.map(async e => {
|
||||||
|
const brainyEntity = await this.brain.get(e.brainyId!)
|
||||||
|
if (brainyEntity?.vector) {
|
||||||
|
return {
|
||||||
entity: e,
|
entity: e,
|
||||||
similarity: 1 - cosineDistance(queryEmbedding, e.entity.vector!)
|
similarity: 1 - cosineDistance(queryEmbedding, brainyEntity.vector)
|
||||||
}))
|
}
|
||||||
.filter(s => s.similarity > this.config.similarityThreshold)
|
}
|
||||||
.sort((a, b) => b.similarity - a.similarity)
|
return null
|
||||||
|
})
|
||||||
|
|
||||||
results = withSimilarity.map(s => s.entity)
|
const resolved = (await Promise.all(withSimilarity))
|
||||||
|
.filter(s => s !== null && s.similarity > this.config.similarityThreshold)
|
||||||
|
.sort((a, b) => b!.similarity - a!.similarity)
|
||||||
|
|
||||||
|
results = resolved.map(s => s!.entity)
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('Failed to perform similarity search:', error)
|
console.warn('Failed to perform similarity search:', error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results.map(r => r.entity.metadata as PersistentEntity)
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -221,7 +226,7 @@ export class PersistentEntitySystem {
|
||||||
extractChanges?: boolean
|
extractChanges?: boolean
|
||||||
}
|
}
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
const entity = await this.getEntity(entityId)
|
const entity = await this.getPersistentEntity(entityId)
|
||||||
if (!entity) {
|
if (!entity) {
|
||||||
throw new Error(`Entity ${entityId} not found`)
|
throw new Error(`Entity ${entityId} not found`)
|
||||||
}
|
}
|
||||||
|
|
@ -247,23 +252,25 @@ export class PersistentEntitySystem {
|
||||||
confidence: options?.confidence ?? 1.0
|
confidence: options?.confidence ?? 1.0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store appearance as Brainy entity
|
// Store appearance as managed entity (with eventType for appearances)
|
||||||
await this.brain.add({
|
const appearanceWithEventType = {
|
||||||
type: NounType.Event,
|
|
||||||
data: Buffer.from(context),
|
|
||||||
metadata: {
|
|
||||||
...appearance,
|
...appearance,
|
||||||
eventType: 'entity-appearance',
|
eventType: 'entity-appearance'
|
||||||
system: 'vfs-entity'
|
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
|
||||||
// Create relationship to entity
|
await this.storeEntity(
|
||||||
await this.brain.relate({
|
appearanceWithEventType,
|
||||||
from: appearanceId,
|
NounType.Event,
|
||||||
to: entityId,
|
undefined,
|
||||||
type: VerbType.References
|
Buffer.from(context)
|
||||||
})
|
)
|
||||||
|
|
||||||
|
// Create relationship to entity using EntityManager
|
||||||
|
await this.createRelationship(
|
||||||
|
appearanceId,
|
||||||
|
entityId,
|
||||||
|
VerbType.References
|
||||||
|
)
|
||||||
|
|
||||||
// Update entity with new appearance
|
// Update entity with new appearance
|
||||||
entity.appearances.push(appearance)
|
entity.appearances.push(appearance)
|
||||||
|
|
@ -287,7 +294,7 @@ export class PersistentEntitySystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update stored entity
|
// Update stored entity
|
||||||
await this.updateEntity(entity)
|
await this.updatePersistentEntity(entity)
|
||||||
|
|
||||||
return appearanceId
|
return appearanceId
|
||||||
}
|
}
|
||||||
|
|
@ -304,7 +311,7 @@ export class PersistentEntitySystem {
|
||||||
appearance?: EntityAppearance
|
appearance?: EntityAppearance
|
||||||
}>
|
}>
|
||||||
}> {
|
}> {
|
||||||
const entity = await this.getEntity(entityId)
|
const entity = await this.getPersistentEntity(entityId)
|
||||||
if (!entity) {
|
if (!entity) {
|
||||||
throw new Error(`Entity ${entityId} not found`)
|
throw new Error(`Entity ${entityId} not found`)
|
||||||
}
|
}
|
||||||
|
|
@ -377,7 +384,7 @@ export class PersistentEntitySystem {
|
||||||
source: string,
|
source: string,
|
||||||
reason?: string
|
reason?: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const entity = await this.getEntity(entityId)
|
const entity = await this.getPersistentEntity(entityId)
|
||||||
if (!entity) {
|
if (!entity) {
|
||||||
throw new Error(`Entity ${entityId} not found`)
|
throw new Error(`Entity ${entityId} not found`)
|
||||||
}
|
}
|
||||||
|
|
@ -405,7 +412,7 @@ export class PersistentEntitySystem {
|
||||||
entity.version++
|
entity.version++
|
||||||
|
|
||||||
// Update stored entity
|
// Update stored entity
|
||||||
await this.updateEntity(entity)
|
await this.updatePersistentEntity(entity)
|
||||||
|
|
||||||
// Record evolution event
|
// Record evolution event
|
||||||
if (changes.length > 0) {
|
if (changes.length > 0) {
|
||||||
|
|
@ -443,7 +450,7 @@ export class PersistentEntitySystem {
|
||||||
// Character names (capitalized words)
|
// Character names (capitalized words)
|
||||||
/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b/g,
|
/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b/g,
|
||||||
// API endpoints
|
// API endpoints
|
||||||
/\/api\/[a-zA-Z0-9\/\-_]+/g,
|
/\/api\/[a-zA-Z0-9/\-_]+/g,
|
||||||
// Class names
|
// Class names
|
||||||
/class\s+([A-Z][a-zA-Z0-9_]*)/g,
|
/class\s+([A-Z][a-zA-Z0-9_]*)/g,
|
||||||
// Function names
|
// Function names
|
||||||
|
|
@ -471,7 +478,7 @@ export class PersistentEntitySystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record appearance for existing or new entity
|
// Record appearance for existing or new entity
|
||||||
const entity = existing[0] || await this.getEntity(entities[entities.length - 1])
|
const entity = existing[0] || await this.getPersistentEntity(entities[entities.length - 1])
|
||||||
if (entity) {
|
if (entity) {
|
||||||
await this.recordAppearance(
|
await this.recordAppearance(
|
||||||
entity.id,
|
entity.id,
|
||||||
|
|
@ -524,38 +531,29 @@ export class PersistentEntitySystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get entity by ID
|
* Get persistent entity by ID
|
||||||
*/
|
*/
|
||||||
private async getEntity(entityId: string): Promise<PersistentEntity | null> {
|
async getPersistentEntity(entityId: string): Promise<PersistentEntity | null> {
|
||||||
// Check cache first
|
// Check cache first
|
||||||
if (this.entityCache.has(entityId)) {
|
if (this.entityCache.has(entityId)) {
|
||||||
return this.entityCache.get(entityId)!
|
return this.entityCache.get(entityId)!
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query from Brainy
|
// Use parent getEntity method
|
||||||
const results = await this.brain.find({
|
const entity = await super.getEntity<PersistentEntity>(entityId)
|
||||||
where: {
|
|
||||||
id: entityId,
|
|
||||||
entityType: 'persistent',
|
|
||||||
system: 'vfs-entity'
|
|
||||||
},
|
|
||||||
type: NounType.Concept,
|
|
||||||
limit: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
if (results.length === 0) {
|
if (entity) {
|
||||||
return null
|
// Cache and return
|
||||||
|
this.entityCache.set(entityId, entity)
|
||||||
}
|
}
|
||||||
|
|
||||||
const entity = results[0].entity.metadata as PersistentEntity
|
|
||||||
this.entityCache.set(entityId, entity)
|
|
||||||
return entity
|
return entity
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update stored entity
|
* Update stored entity (rename to avoid parent method conflict)
|
||||||
*/
|
*/
|
||||||
private async updateEntity(entity: PersistentEntity): Promise<void> {
|
private async updatePersistentEntity(entity: PersistentEntity): Promise<void> {
|
||||||
// Find the Brainy entity
|
// Find the Brainy entity
|
||||||
const results = await this.brain.find({
|
const results = await this.brain.find({
|
||||||
where: {
|
where: {
|
||||||
|
|
|
||||||
|
|
@ -10,11 +10,12 @@ import { NounType, VerbType } from '../types/graphTypes.js'
|
||||||
import { cosineDistance } from '../utils/distance.js'
|
import { cosineDistance } from '../utils/distance.js'
|
||||||
import { createHash } from 'crypto'
|
import { createHash } from 'crypto'
|
||||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||||
|
import { EntityManager, ManagedEntity } from './EntityManager.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Version metadata
|
* Version metadata
|
||||||
*/
|
*/
|
||||||
export interface Version {
|
export interface Version extends ManagedEntity {
|
||||||
id: string
|
id: string
|
||||||
path: string
|
path: string
|
||||||
version: number
|
version: number
|
||||||
|
|
@ -43,14 +44,15 @@ export interface SemanticVersioningConfig {
|
||||||
* Creates versions only when content meaning changes significantly
|
* Creates versions only when content meaning changes significantly
|
||||||
* Uses vector embeddings to detect semantic changes
|
* Uses vector embeddings to detect semantic changes
|
||||||
*/
|
*/
|
||||||
export class SemanticVersioning {
|
export class SemanticVersioning extends EntityManager {
|
||||||
private config: Required<SemanticVersioningConfig>
|
private config: Required<SemanticVersioningConfig>
|
||||||
private versionCache = new Map<string, Version[]>()
|
private versionCache = new Map<string, Version[]>()
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private brain: Brainy,
|
brain: Brainy,
|
||||||
config?: SemanticVersioningConfig
|
config?: SemanticVersioningConfig
|
||||||
) {
|
) {
|
||||||
|
super(brain, 'vfs-version')
|
||||||
this.config = {
|
this.config = {
|
||||||
threshold: config?.threshold ?? 0.3,
|
threshold: config?.threshold ?? 0.3,
|
||||||
maxVersions: config?.maxVersions ?? 10,
|
maxVersions: config?.maxVersions ?? 10,
|
||||||
|
|
@ -128,11 +130,8 @@ export class SemanticVersioning {
|
||||||
console.warn('Failed to generate embedding for version:', error)
|
console.warn('Failed to generate embedding for version:', error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store version as Brainy entity
|
// Create version entity
|
||||||
const entity = await this.brain.add({
|
const version: Version = {
|
||||||
type: NounType.State,
|
|
||||||
data: content, // Store actual content
|
|
||||||
metadata: {
|
|
||||||
id: versionId,
|
id: versionId,
|
||||||
path,
|
path,
|
||||||
version: versionNumber,
|
version: versionNumber,
|
||||||
|
|
@ -142,19 +141,20 @@ export class SemanticVersioning {
|
||||||
size: content.length,
|
size: content.length,
|
||||||
author: metadata?.author,
|
author: metadata?.author,
|
||||||
message: metadata?.message,
|
message: metadata?.message,
|
||||||
parentVersion,
|
parentVersion
|
||||||
system: 'vfs-version'
|
}
|
||||||
} as Version,
|
|
||||||
vector: embedding
|
// Store version using EntityManager (with actual content as data)
|
||||||
})
|
await this.storeEntity(version, NounType.State, embedding, content)
|
||||||
|
|
||||||
// Create relationship to parent version if exists
|
// Create relationship to parent version if exists
|
||||||
if (parentVersion) {
|
if (parentVersion) {
|
||||||
await this.brain.relate({
|
try {
|
||||||
from: entity,
|
await this.createRelationship(versionId, parentVersion, VerbType.Succeeds)
|
||||||
to: parentVersion,
|
} catch (error) {
|
||||||
type: VerbType.Succeeds
|
console.warn(`Failed to create parent relationship for version ${versionId}:`, error)
|
||||||
})
|
// Continue without relationship - non-critical for version functionality
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
|
|
@ -189,19 +189,15 @@ export class SemanticVersioning {
|
||||||
return this.versionCache.get(path)!
|
return this.versionCache.get(path)!
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query from Brainy
|
// Query using EntityManager
|
||||||
const results = await this.brain.find({
|
const versions = await this.findEntities<Version>(
|
||||||
where: {
|
{ path },
|
||||||
path,
|
NounType.State,
|
||||||
system: 'vfs-version'
|
this.config.maxVersions * 2 // Get extra in case some are pruned
|
||||||
},
|
)
|
||||||
type: NounType.State,
|
|
||||||
limit: this.config.maxVersions * 2 // Get extra in case some are pruned
|
|
||||||
})
|
|
||||||
|
|
||||||
const versions = results
|
// Sort by timestamp (newest first)
|
||||||
.map(r => r.entity.metadata as Version)
|
versions.sort((a, b) => b.timestamp - a.timestamp)
|
||||||
.sort((a, b) => b.timestamp - a.timestamp) // Newest first
|
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
this.versionCache.set(path, versions)
|
this.versionCache.set(path, versions)
|
||||||
|
|
@ -213,21 +209,20 @@ export class SemanticVersioning {
|
||||||
* Get a specific version's content
|
* Get a specific version's content
|
||||||
*/
|
*/
|
||||||
async getVersion(path: string, versionId: string): Promise<Buffer | null> {
|
async getVersion(path: string, versionId: string): Promise<Buffer | null> {
|
||||||
const results = await this.brain.find({
|
// Get the version entity
|
||||||
where: {
|
const version = await this.getEntity<Version>(versionId)
|
||||||
id: versionId,
|
if (!version || version.path !== path) {
|
||||||
path,
|
|
||||||
system: 'vfs-version'
|
|
||||||
},
|
|
||||||
type: NounType.State,
|
|
||||||
limit: 1
|
|
||||||
})
|
|
||||||
|
|
||||||
if (results.length === 0) {
|
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
return results[0].entity.data as Buffer
|
// Get the content from Brainy using the Brainy ID
|
||||||
|
const brainyId = await this.getBrainyId(versionId)
|
||||||
|
if (!brainyId) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const entity = await this.brain.get(brainyId)
|
||||||
|
return entity?.data as Buffer | null
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -319,7 +314,7 @@ export class SemanticVersioning {
|
||||||
|
|
||||||
// Delete excess versions
|
// Delete excess versions
|
||||||
for (const id of toDelete.slice(0, versions.length - this.config.maxVersions)) {
|
for (const id of toDelete.slice(0, versions.length - this.config.maxVersions)) {
|
||||||
await this.brain.delete(id)
|
await this.deleteEntity(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
|
|
|
||||||
|
|
@ -2057,7 +2057,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
case 'mkdir':
|
case 'mkdir':
|
||||||
await this.mkdir(op.path, op.options)
|
await this.mkdir(op.path, op.options)
|
||||||
break
|
break
|
||||||
case 'update':
|
case 'update': {
|
||||||
// Update only metadata without changing content
|
// Update only metadata without changing content
|
||||||
const entityId = await this.pathResolver.resolve(op.path)
|
const entityId = await this.pathResolver.resolve(op.path)
|
||||||
await this.brain.update({
|
await this.brain.update({
|
||||||
|
|
@ -2066,6 +2066,7 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
})
|
})
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
}
|
||||||
result.successful++
|
result.successful++
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
result.failed.push({
|
result.failed.push({
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue