soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: complete VFS with Knowledge Layer integration

Browse source 
- Add importFile() method for single file imports
- Implement entity helper methods (linkEntities, findEntityOccurrences)
- Fix critical embedding tokenizer bug (char.charCodeAt error)
- Fix removeRelationship to actually remove using brain.unrelate()
- Add setMetadata/getMetadata methods
- Fix GitBridge to query real relationships and events
- Enable background Knowledge Layer processing
- Rewrite README to emphasize knowledge over files
- Add comprehensive VFS documentation (core, knowledge layer, examples)
- Add complete test suite covering all VFS methods

This completes the VFS implementation with full Knowledge Layer support,
enabling files as living knowledge that understand themselves, evolve
over time, and connect to everything related.
This commit is contained in:
David Snelling 2025-09-25 10:47:44 -07:00
parent b3c4f348ab
commit 581f9906fd
17 changed files with 2441 additions and 352 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/vfs/ConceptSystem.ts
View file

@ -680,7 +680,7 @@ export class ConceptSystem {
concept.description || '',
concept.domain,
concept.category,
...concept.keywords
...(Array.isArray(concept.keywords) ? concept.keywords : [])
].join(' ')
return await this.generateTextEmbedding(text)

28
src/vfs/EventRecorder.ts
View file

@ -24,6 +24,7 @@ export interface FileEvent {
author?: string
metadata?: Record<string, any>
previousHash?: string // For tracking changes
oldPath?: string // For move/rename operations
}
/**
@ -272,13 +273,28 @@ export class EventRecorder {
*/
private async generateEventEmbedding(event: Omit<FileEvent, 'id' | 'timestamp'>): Promise<number[] | undefined> {
try {
// For content events, generate embedding from content
if (event.content && event.content.length < 100000) {
// Use Brainy's embedding function if available
// This would need to be passed in or configured
return undefined // Placeholder - would use actual embedding
// Generate embedding based on event type and content
let textToEmbed: string
if (event.type === 'write' || event.type === 'create') {
// For write/create events with content, embed the content
if (event.content && event.content.length < 100000) {
// Convert Buffer to string for text files
textToEmbed = Buffer.isBuffer(event.content)
? event.content.toString('utf8', 0, Math.min(10240, event.content.length))
: String(event.content).slice(0, 10240)
} else {
// For large files or no content, create descriptive text
textToEmbed = `File ${event.type} event at ${event.path}, size: ${event.size || 0} bytes`
}
} else {
// For other events (read, delete, rename, etc), create descriptive text
textToEmbed = `File ${event.type} event at ${event.path}${event.oldPath ? ` from ${event.oldPath}` : ''}`
}
return undefined
// Use Brainy's embed function
const vector = await this.brain.embed(textToEmbed)
return vector
} catch (error) {
console.error('Failed to generate event embedding:', error)
return undefined

49
src/vfs/GitBridge.ts
View file

@ -387,9 +387,42 @@ export class GitBridge {
options?: ExportOptions
): Promise<void> {
// Get all relationships for files in the VFS path
// This would query Brainy for relationships
const relationships: any[] = []
try {
// Get relationships for the specified path and its children
const related = await this.vfs.getRelated(vfsPath)
if (related && related.length > 0) {
relationships.push({
path: vfsPath,
relationships: related
})
}
// If it's a directory, get relationships for all files within
const stats = await this.vfs.stat(vfsPath)
if (stats.isDirectory()) {
const files = await this.vfs.readdir(vfsPath, { recursive: true })
for (const file of files) {
const fileName = typeof file === 'string' ? file : file.name
const fullPath = path.join(vfsPath, fileName)
try {
const fileRelated = await this.vfs.getRelated(fullPath)
if (fileRelated && fileRelated.length > 0) {
relationships.push({
path: fullPath,
relationships: fileRelated
})
}
} catch (err) {
// Skip files without relationships
}
}
}
} catch (err) {
// Path might not have relationships
}
// Write relationships file
const relationshipsPath = path.join(gitRepoPath, '.vfs-relationships.json')
await fs.writeFile(relationshipsPath, JSON.stringify(relationships, null, 2))
@ -407,7 +440,19 @@ export class GitBridge {
const history = {
exportedAt: Date.now(),
vfsPath,
events: [] // Would contain VFS events
events: [] as any[]
}
// Get history if Knowledge Layer is enabled
if ('getHistory' in this.vfs && typeof (this.vfs as any).getHistory === 'function') {
try {
const events = await (this.vfs as any).getHistory(vfsPath)
if (events) {
history.events = events
}
} catch (err) {
// Knowledge Layer might not be enabled
}
}
// Write history file

211
src/vfs/KnowledgeLayer.ts
View file

@ -52,50 +52,48 @@ export class KnowledgeLayer {
// Call original VFS method first
const result = await originalWriteFile(path, data, options)
// TEMPORARY: Disable background processing for debugging
if (process.env.ENABLE_KNOWLEDGE_PROCESSING === 'true') {
setImmediate(async () => {
// Process in background (non-blocking)
setImmediate(async () => {
try {
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
// 1. Record the event
await this.eventRecorder.recordEvent({
type: 'write',
path,
content: buffer,
size: buffer.length,
author: options?.author || 'system'
})
// 2. Check for semantic versioning
try {
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
// 1. Record the event
await this.eventRecorder.recordEvent({
type: 'write',
path,
content: buffer,
size: buffer.length,
author: options?.author || 'system'
})
// 2. Check for semantic versioning
try {
const existingContent = await this.vfs.readFile(path).catch(() => null)
if (existingContent) {
const shouldVersion = await this.semanticVersioning.shouldVersion(existingContent, buffer)
if (shouldVersion) {
await this.semanticVersioning.createVersion(path, buffer, {
message: options?.message || 'Automatic semantic version'
})
}
const existingContent = await this.vfs.readFile(path).catch(() => null)
if (existingContent) {
const shouldVersion = await this.semanticVersioning.shouldVersion(existingContent, buffer)
if (shouldVersion) {
await this.semanticVersioning.createVersion(path, buffer, {
message: options?.message || 'Automatic semantic version'
})
}
} catch (err) {
console.debug('Versioning check failed:', err)
}
// 3. Extract entities
if (options?.extractEntities !== false) {
await this.entitySystem.extractEntities(path, buffer)
}
// 4. Extract concepts
if (options?.extractConcepts !== false) {
await this.conceptSystem.extractAndLinkConcepts(path, buffer)
}
} catch (error) {
console.debug('Knowledge Layer processing error:', error)
} catch (err) {
console.debug('Versioning check failed:', err)
}
})
}
// 3. Extract entities
if (options?.extractEntities !== false) {
await this.entitySystem.extractEntities(path, buffer)
}
// 4. Extract concepts
if (options?.extractConcepts !== false) {
await this.conceptSystem.extractAndLinkConcepts(path, buffer)
}
} catch (error) {
console.debug('Knowledge Layer processing error:', error)
}
})
return result
}
@ -235,6 +233,139 @@ export class KnowledgeLayer {
(this.vfs as any).findTemporalCoupling = async (path: string, windowMs?: number) => {
return await this.eventRecorder.findTemporalCoupling(path, windowMs)
}
// Entity convenience methods that wrap Brainy's core API
(this.vfs as any).linkEntities = async (fromEntity: string | any, toEntity: string | any, relationship: string) => {
// Handle both entity IDs and entity objects
const fromId = typeof fromEntity === 'string' ? fromEntity : fromEntity.id
const toId = typeof toEntity === 'string' ? toEntity : toEntity.id
// Use brain.relate to create the relationship
return await this.brain.relate({
from: fromId,
to: toId,
type: relationship as any // VerbType or string
})
}
// Find where an entity appears across files
(this.vfs as any).findEntityOccurrences = async (entityId: string) => {
const occurrences: Array<{ path: string, context?: string }> = []
// Search for files that contain references to this entity
// First, get all relationships where this entity is involved
const relations = await this.brain.getRelations({ from: entityId })
const toRelations = await this.brain.getRelations({ to: entityId })
// Find file entities that relate to this entity
for (const rel of [...relations, ...toRelations]) {
try {
// Check if the related entity is a file
const relatedId = rel.from === entityId ? rel.to : rel.from
const entity = await this.brain.get(relatedId)
if (entity?.metadata?.vfsType === 'file' && entity?.metadata?.path) {
occurrences.push({
path: entity.metadata.path as string,
context: entity.data ? entity.data.toString().substring(0, 200) : undefined
})
}
} catch (error) {
// Entity might not exist, continue
}
}
// Also search for files that mention the entity name in their content
const entityData = await this.brain.get(entityId)
if (entityData?.metadata?.name) {
const searchResults = await this.brain.find({
query: entityData.metadata.name as string,
where: { vfsType: 'file' },
limit: 20
})
for (const result of searchResults) {
if (result.entity?.metadata?.path && !occurrences.some(o => o.path === result.entity.metadata.path)) {
occurrences.push({
path: result.entity.metadata.path as string,
context: result.entity.data ? result.entity.data.toString().substring(0, 200) : undefined
})
}
}
}
return occurrences
}
// Update an entity (convenience wrapper)
(this.vfs as any).updateEntity = async (entityId: string, updates: any) => {
// Get current entity from brain
const currentEntity = await this.brain.get(entityId)
if (!currentEntity) {
throw new Error(`Entity ${entityId} not found`)
}
// Merge updates
const updatedMetadata = {
...currentEntity.metadata,
...updates,
lastUpdated: Date.now(),
version: ((currentEntity.metadata?.version as number) || 0) + 1
}
// Update via brain
await this.brain.update({
id: entityId,
data: JSON.stringify(updatedMetadata),
metadata: updatedMetadata
})
return entityId
}
// Get entity graph (convenience wrapper)
(this.vfs as any).getEntityGraph = async (entityId: string, options?: { depth?: number }) => {
const depth = options?.depth || 2
const graph = { nodes: new Map(), edges: [] as any[] }
const visited = new Set<string>()
const traverse = async (id: string, currentDepth: number) => {
if (visited.has(id) || currentDepth > depth) return
visited.add(id)
// Add node
const entity = await this.brain.get(id)
if (entity) {
graph.nodes.set(id, entity)
}
// Get relationships
const relations = await this.brain.getRelations({ from: id })
const toRelations = await this.brain.getRelations({ to: id })
for (const rel of [...relations, ...toRelations]) {
graph.edges.push(rel)
// Traverse connected nodes
if (currentDepth < depth) {
const nextId = rel.from === id ? rel.to : rel.from
await traverse(nextId, currentDepth + 1)
}
}
}
await traverse(entityId, 0)
return {
nodes: Array.from(graph.nodes.values()),
edges: graph.edges
}
}
// List all entities of a specific type
(this.vfs as any).listEntities = async (query?: { type?: string }) => {
return await this.entitySystem.findEntity(query || {})
}
}
/**

108
src/vfs/VirtualFileSystem.ts
View file

@ -846,28 +846,28 @@ export class VirtualFileSystem implements IVirtualFileSystem {
return !compressedSignatures.some(sig => firstBytes.startsWith(sig))
}
// Stub methods for external storage (implement based on your storage backend)
// External storage methods - leverages Brainy's storage adapters (memory, file, S3, R2)
private async readExternalContent(key: string): Promise<Buffer> {
// Store in Brainy entity with special type for large content
// Read from Brainy - Brainy's storage adapter handles retrieval
const entity = await this.brain.get(key)
if (!entity || !entity.metadata?.externalContent) {
if (!entity) {
throw new Error(`External content not found: ${key}`)
}
// Content stored as base64 in metadata for now
// In production, this would use S3/R2
return Buffer.from(entity.metadata.externalContent, 'base64')
// Content is stored in the data field
// Brainy handles storage/retrieval through its adapters (memory, file, S3, R2)
return Buffer.isBuffer(entity.data) ? entity.data : Buffer.from(entity.data)
}
private async storeExternalContent(buffer: Buffer): Promise<string> {
// Store as separate Brainy entity for large content
// Store as Brainy entity - let Brainy's storage adapter handle it
// Brainy automatically handles large data through its storage adapters (memory, file, S3, R2)
const entityId = await this.brain.add({
data: `Large file content: ${buffer.length} bytes`,
data: buffer, // Store actual buffer - Brainy will handle it efficiently
type: NounType.File,
metadata: {
vfsType: 'external-storage',
size: buffer.length,
externalContent: buffer.toString('base64'),
created: Date.now()
}
})
@ -890,10 +890,12 @@ export class VirtualFileSystem implements IVirtualFileSystem {
for (const chunkId of chunks) {
const entity = await this.brain.get(chunkId)
if (!entity || !entity.metadata?.chunkData) {
if (!entity) {
throw new Error(`Chunk not found: ${chunkId}`)
}
buffers.push(Buffer.from(entity.metadata.chunkData, 'base64'))
// Read actual data from entity - Brainy handles storage
const chunkBuffer = Buffer.isBuffer(entity.data) ? entity.data : Buffer.from(entity.data)
buffers.push(chunkBuffer)
}
return Buffer.concat(buffers)
@ -907,13 +909,13 @@ export class VirtualFileSystem implements IVirtualFileSystem {
const chunk = buffer.slice(i, Math.min(i + chunkSize, buffer.length))
// Store each chunk as a separate entity
// Let Brainy handle the chunk data efficiently
const chunkId = await this.brain.add({
data: `Chunk ${chunks.length + 1} of file, size: ${chunk.length}`,
data: chunk, // Store actual chunk - Brainy handles it
type: NounType.File,
metadata: {
vfsType: 'chunk',
chunkIndex: chunks.length,
chunkData: chunk.toString('base64'),
size: chunk.length,
created: Date.now()
}
@ -1731,13 +1733,14 @@ export class VirtualFileSystem implements IVirtualFileSystem {
const fromEntityId = await this.pathResolver.resolve(from)
const toEntityId = await this.pathResolver.resolve(to)
// Find the relationship
// Find and delete the relationship
const relations = await this.brain.getRelations({ from: fromEntityId })
for (const relation of relations) {
if (relation.to === toEntityId && (!type || relation.type === type)) {
// Delete the relationship (note: unrelate API might need adjustment)
// For now, we'll mark it in metadata
console.log(`Would remove relationship from ${from} to ${to} of type ${type || 'any'}`)
// Delete the relationship using brain.unrelate
if (relation.id) {
await this.brain.unrelate(relation.id)
}
}
}
@ -1807,6 +1810,42 @@ export class VirtualFileSystem implements IVirtualFileSystem {
this.invalidateCaches(path)
}
/**
* Get metadata for a file or directory
*/
async getMetadata(path: string): Promise<VFSMetadata | undefined> {
await this.ensureInitialized()
const entityId = await this.pathResolver.resolve(path)
const entity = await this.getEntityById(entityId)
return entity.metadata
}
/**
* Set custom metadata for a file or directory
* Merges with existing metadata
*/
async setMetadata(path: string, metadata: Partial<VFSMetadata>): Promise<void> {
await this.ensureInitialized()
const entityId = await this.pathResolver.resolve(path)
const entity = await this.getEntityById(entityId)
// Merge with existing metadata
await this.brain.update({
id: entityId,
metadata: {
...entity.metadata,
...metadata,
modified: Date.now()
}
})
// Invalidate caches
this.invalidateCaches(path)
}
// ============= Knowledge Layer =============
// Knowledge Layer methods are added by KnowledgeLayer.enable()
// This keeps VFS pure and fast while allowing optional intelligence
@ -1861,7 +1900,40 @@ export class VirtualFileSystem implements IVirtualFileSystem {
// ============= Import/Export Operations =============
/**
* Import a directory or file from the real filesystem into VFS
* Import a single file from the real filesystem into VFS
*/
async importFile(sourcePath: string, targetPath: string): Promise<void> {
const fs = await import('fs/promises')
const pathModule = await import('path')
// Read file from local filesystem
const content = await fs.readFile(sourcePath)
const stats = await fs.stat(sourcePath)
// Ensure parent directory exists in VFS
const parentPath = pathModule.dirname(targetPath)
if (parentPath !== '/' && parentPath !== '.') {
try {
await this.mkdir(parentPath, { recursive: true })
} catch (error: any) {
if (error.code !== 'EEXIST') throw error
}
}
// Write to VFS with metadata from source
await this.writeFile(targetPath, content, {
metadata: {
imported: true,
importedFrom: sourcePath,
sourceSize: stats.size,
sourceMtime: stats.mtime.getTime(),
sourceMode: stats.mode
}
})
}
/**
* Import a directory from the real filesystem into VFS
*/
async importDirectory(sourcePath: string, options?: any): Promise<any> {
const { DirectoryImporter } = await import('./importers/DirectoryImporter.js')

2
src/vfs/types.ts
View file

@ -426,6 +426,8 @@ export interface IVirtualFileSystem {
getTodos(path: string): Promise<VFSTodo[] | undefined>
setTodos(path: string, todos: VFSTodo[]): Promise<void>
addTodo(path: string, todo: VFSTodo): Promise<void>
getMetadata(path: string): Promise<VFSMetadata | undefined>
setMetadata(path: string, metadata: Partial<VFSMetadata>): Promise<void>
// Streaming (returns Node.js compatible streams)
createReadStream(path: string, options?: ReadStreamOptions): NodeJS.ReadableStream