/** * Virtual Filesystem Implementation * * PRODUCTION-READY VFS built on Brainy * Real code, no mocks, actual working implementation */ import { Readable, Writable } from 'stream' import crypto from 'crypto' import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' import { PathResolver } from './PathResolver.js' // Knowledge Layer imports removed - now in KnowledgeAugmentation import { IVirtualFileSystem, VFSConfig, VFSEntity, VFSMetadata, VFSStats, VFSDirent, VFSTodo, VFSError, VFSErrorCode, WriteOptions, ReadOptions, MkdirOptions, ReaddirOptions, CopyOptions, SearchOptions, SearchResult, SimilarOptions, RelatedOptions, ReadStreamOptions, WriteStreamOptions, WatchListener } from './types.js' /** * Main Virtual Filesystem Implementation * * This is REAL, production-ready code that: * - Maps filesystem operations to Brainy entities * - Uses graph relationships for directory structure * - Provides semantic search and AI features * - Scales to millions of files */ export class VirtualFileSystem implements IVirtualFileSystem { private brain: Brainy private pathResolver!: PathResolver private config: Required> & { rootEntityId?: string } private rootEntityId?: string private initialized = false private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer removed from core - now optional augmentation // Caches for performance private contentCache: Map private statCache: Map // Watch system private watchers: Map> // Background task timer private backgroundTimer: NodeJS.Timeout | null = null constructor(brain?: Brainy) { this.brain = brain || new Brainy() this.contentCache = new Map() this.statCache = new Map() this.watchers = new Map() // Default configuration (will be overridden in init) this.config = this.getDefaultConfig() } /** * Initialize the VFS */ async init(config?: VFSConfig): Promise { if (this.initialized) return // Merge config with defaults this.config = { ...this.getDefaultConfig(), ...config } // Initialize Brainy if needed if (!this.brain.isInitialized) { await this.brain.init() } // Create or find root entity this.rootEntityId = await this.initializeRoot() // Initialize path resolver this.pathResolver = new PathResolver(this.brain, this.rootEntityId, { maxCacheSize: this.config.cache?.maxPaths, cacheTTL: this.config.cache?.ttl, hotPathThreshold: 10 }) // Knowledge Layer is now a separate augmentation // Enable with: brain.use('knowledge') // Start background tasks this.startBackgroundTasks() this.initialized = true } /** * Create or find the root directory entity */ private async initializeRoot(): Promise { // Check if root already exists - search using where clause const existing = await this.brain.find({ where: { 'metadata.path': '/', 'metadata.vfsType': 'directory' }, limit: 1 }) if (existing.length > 0) { const rootEntity = existing[0] // Ensure the root entity has proper metadata structure const entityMetadata = (rootEntity as any).metadata || rootEntity if (!entityMetadata.vfsType) { // Update the root entity with proper metadata await this.brain.update({ id: rootEntity.id, metadata: { path: '/', name: '', vfsType: 'directory', size: 0, permissions: 0o755, owner: 'root', group: 'root', accessed: Date.now(), modified: Date.now(), ...entityMetadata // Preserve any existing metadata } }) } return rootEntity.id } // Create root directory const root = await this.brain.add({ data: '/', // Root directory content as string type: NounType.Collection, metadata: { path: '/', name: '', vfsType: 'directory', size: 0, permissions: 0o755, owner: 'root', group: 'root', accessed: Date.now(), modified: Date.now() } as VFSMetadata }) return root } // ============= File Operations ============= /** * Read a file's content */ async readFile(path: string, options?: ReadOptions): Promise { await this.ensureInitialized() // Check cache first if (options?.cache !== false && this.contentCache.has(path)) { const cached = this.contentCache.get(path)! if (Date.now() - cached.timestamp < (this.config.cache?.ttl || 300000)) { return cached.data } } // Resolve path to entity const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Verify it's a file if (entity.metadata.vfsType !== 'file') { throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'readFile') } // Get content based on storage type let content: Buffer if (!entity.metadata.storage || entity.metadata.storage.type === 'inline') { // Content stored in metadata for new files, or try entity data for compatibility if (entity.metadata.rawData) { content = Buffer.from(entity.metadata.rawData, 'base64') } else if (!entity.data) { content = Buffer.alloc(0) } else if (Buffer.isBuffer(entity.data)) { content = entity.data } else if (typeof entity.data === 'string') { content = Buffer.from(entity.data) } else { content = Buffer.from(JSON.stringify(entity.data)) } } else if (entity.metadata.storage.type === 'reference') { // Content stored in external storage content = await this.readExternalContent(entity.metadata.storage.key!) } else if (entity.metadata.storage.type === 'chunked') { // Content stored in chunks content = await this.readChunkedContent(entity.metadata.storage.chunks!) } else { throw new VFSError(VFSErrorCode.EIO, `Unknown storage type: ${entity.metadata.storage.type}`, path, 'readFile') } // Decompress if needed if (entity.metadata.storage?.compressed && options?.decompress !== false) { content = await this.decompress(content) } // Update access time await this.updateAccessTime(entityId) // Cache the content if (options?.cache !== false) { this.contentCache.set(path, { data: content, timestamp: Date.now() }) } // Apply encoding if requested if (options?.encoding) { return Buffer.from(content.toString(options.encoding)) } return content } /** * Write a file */ async writeFile(path: string, data: Buffer | string, options?: WriteOptions): Promise { await this.ensureInitialized() // Convert string to buffer const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding) // Check size limits if (this.config.limits?.maxFileSize && buffer.length > this.config.limits.maxFileSize) { throw new VFSError(VFSErrorCode.ENOSPC, `File too large: ${buffer.length} bytes`, path, 'writeFile') } // Parse path to get parent and name const parentPath = this.getParentPath(path) const name = this.getBasename(path) // Ensure parent directory exists const parentId = await this.ensureDirectory(parentPath) // Check if file already exists let existingId: string | null = null try { existingId = await this.pathResolver.resolve(path, { cache: false }) // Verify the entity still exists in the brain const existing = await this.brain.get(existingId) if (!existing) { existingId = null // Entity was deleted but cache wasn't cleared } } catch (err) { // File doesn't exist, which is fine existingId = null } // Determine storage strategy based on size let storageStrategy: VFSMetadata['storage'] let entityData: Buffer | null = null if (buffer.length <= (this.config.storage?.inline?.maxSize || 100_000)) { // Store inline for small files storageStrategy = { type: 'inline' } entityData = buffer } else if (buffer.length <= 10_000_000) { // Store as reference for medium files const key = await this.storeExternalContent(buffer) storageStrategy = { type: 'reference', key } } else { // Store as chunks for large files const chunks = await this.storeChunkedContent(buffer) storageStrategy = { type: 'chunked', chunks } } // Compress if beneficial if (this.shouldCompress(buffer) && options?.compress !== false) { const compressed = await this.compress(buffer) if (compressed.length < buffer.length * 0.9) { // Only if >10% savings storageStrategy.compressed = true if (storageStrategy.type === 'inline') { entityData = compressed } } } // Detect MIME type const mimeType = this.detectMimeType(name, buffer) // Create metadata const metadata: VFSMetadata = { path, name, parent: parentId, vfsType: 'file', size: buffer.length, mimeType, extension: this.getExtension(name), permissions: options?.mode || this.config.permissions?.defaultFile || 0o644, owner: 'user', // In production, get from auth context group: 'users', accessed: Date.now(), modified: Date.now(), storage: storageStrategy, // Store raw buffer data for retrieval rawData: buffer.toString('base64') // Store as base64 for safe serialization } // Extract additional metadata if enabled if (this.config.intelligence?.autoExtract && options?.extractMetadata !== false) { Object.assign(metadata, await this.extractMetadata(buffer, mimeType)) } if (existingId) { // Update existing file await this.brain.update({ id: existingId, data: entityData, metadata }) // Ensure Contains relationship exists (fix for missing relationships) const existingRelations = await this.brain.getRelations({ from: parentId, to: existingId, type: VerbType.Contains }) // Create relationship if it doesn't exist if (existingRelations.length === 0) { await this.brain.relate({ from: parentId, to: existingId, type: VerbType.Contains }) } } else { // Create new file entity // For embedding: use text content, for storage: use raw data const embeddingData = this.isTextFile(mimeType) ? buffer.toString('utf-8') : `File: ${name} (${mimeType}, ${buffer.length} bytes)` const entity = await this.brain.add({ data: embeddingData, // Always provide string for embeddings type: this.getFileNounType(mimeType), metadata }) // Create parent-child relationship await this.brain.relate({ from: parentId, to: entity, type: VerbType.Contains }) // Update path resolver cache await this.pathResolver.createPath(path, entity) } // Invalidate caches this.invalidateCaches(path) // Trigger watchers this.triggerWatchers(path, existingId ? 'change' : 'rename') // Knowledge Layer hooks will be added by augmentation if enabled // Knowledge Layer hooks will be added by augmentation if enabled } /** * Append to a file */ async appendFile(path: string, data: Buffer | string, options?: WriteOptions): Promise { await this.ensureInitialized() // Read existing content let existing: Buffer try { existing = await this.readFile(path) } catch (err) { // File doesn't exist, create it return this.writeFile(path, data, options) } // Append new data const newData = Buffer.isBuffer(data) ? data : Buffer.from(data, options?.encoding) const combined = Buffer.concat([existing, newData]) // Write combined content await this.writeFile(path, combined, options) } /** * Delete a file */ async unlink(path: string): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Verify it's a file if (entity.metadata.vfsType !== 'file') { throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'unlink') } // Delete external content if needed if (entity.metadata.storage) { if (entity.metadata.storage.type === 'reference') { await this.deleteExternalContent(entity.metadata.storage.key!) } else if (entity.metadata.storage.type === 'chunked') { await this.deleteChunkedContent(entity.metadata.storage.chunks!) } } // Delete the entity await this.brain.delete(entityId) // Invalidate caches this.pathResolver.invalidatePath(path) this.invalidateCaches(path) // Trigger watchers this.triggerWatchers(path, 'rename') // Knowledge Layer hooks will be added by augmentation if enabled } // ============= Tree Operations (NEW) ============= /** * Get only direct children of a directory - guaranteed no self-inclusion * This is the SAFE way to get children for building tree UIs */ async getDirectChildren(path: string): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Verify it's a directory if (entity.metadata.vfsType !== 'directory') { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'getDirectChildren') } // Use the safe getChildren from PathResolver const children = await this.pathResolver.getChildren(entityId) // Double-check no self-inclusion (paranoid safety) return children.filter(child => child.metadata.path !== path) } /** * Get a properly structured tree for the given path * This prevents recursion issues common when building file explorers */ async getTreeStructure(path: string, options?: { maxDepth?: number includeHidden?: boolean sort?: 'name' | 'modified' | 'size' }): Promise { await this.ensureInitialized() const { VFSTreeUtils } = await import('./TreeUtils.js') const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) if (entity.metadata.vfsType !== 'directory') { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'getTreeStructure') } // Recursively gather all descendants const allEntities: VFSEntity[] = [] const visited = new Set() const gatherDescendants = async (dirId: string) => { if (visited.has(dirId)) return // Prevent cycles visited.add(dirId) const children = await this.pathResolver.getChildren(dirId) for (const child of children) { allEntities.push(child) if (child.metadata.vfsType === 'directory') { await gatherDescendants(child.id) } } } await gatherDescendants(entityId) // Build safe tree structure return VFSTreeUtils.buildTree(allEntities, path, options || {}) } /** * Get all descendants of a directory (flat list) */ async getDescendants(path: string, options?: { includeAncestor?: boolean type?: 'file' | 'directory' }): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) if (entity.metadata.vfsType !== 'directory') { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'getDescendants') } const descendants: VFSEntity[] = [] if (options?.includeAncestor) { descendants.push(entity) } const visited = new Set() const queue = [entityId] while (queue.length > 0) { const currentId = queue.shift()! if (visited.has(currentId)) continue visited.add(currentId) const children = await this.pathResolver.getChildren(currentId) for (const child of children) { // Filter by type if specified if (!options?.type || child.metadata.vfsType === options.type) { descendants.push(child) } // Add directories to queue for traversal if (child.metadata.vfsType === 'directory') { queue.push(child.id) } } } return descendants } /** * Inspect a path and return structured information * This is the recommended method for file explorers to use */ async inspect(path: string): Promise<{ node: VFSEntity children: VFSEntity[] parent: VFSEntity | null stats: VFSStats }> { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) const stats = await this.stat(path) let children: VFSEntity[] = [] if (entity.metadata.vfsType === 'directory') { children = await this.getDirectChildren(path) } let parent: VFSEntity | null = null if (path !== '/') { const parentPath = path.substring(0, path.lastIndexOf('/')) || '/' const parentId = await this.pathResolver.resolve(parentPath) parent = await this.getEntityById(parentId) } return { node: entity, children, parent, stats } } // ============= Directory Operations ============= /** * Create a directory */ async mkdir(path: string, options?: MkdirOptions): Promise { await this.ensureInitialized() // Check if already exists try { const existing = await this.pathResolver.resolve(path) const entity = await this.getEntityById(existing) if (entity.metadata.vfsType === 'directory') { if (!options?.recursive) { throw new VFSError(VFSErrorCode.EEXIST, `Directory exists: ${path}`, path, 'mkdir') } return // Already exists and recursive is true } else { // Path exists but it's not a directory throw new VFSError(VFSErrorCode.EEXIST, `File exists: ${path}`, path, 'mkdir') } } catch (err) { // Only proceed if it's a ENOENT error (path doesn't exist) if (err instanceof VFSError && err.code !== VFSErrorCode.ENOENT) { throw err // Re-throw non-ENOENT errors } // Doesn't exist, proceed to create } // Parse path const parentPath = this.getParentPath(path) const name = this.getBasename(path) // Ensure parent exists (recursive mkdir if needed) let parentId: string if (parentPath === '/' || parentPath === null) { parentId = this.rootEntityId! } else if (options?.recursive) { parentId = await this.ensureDirectory(parentPath) } else { try { parentId = await this.pathResolver.resolve(parentPath) } catch (err) { throw new VFSError(VFSErrorCode.ENOENT, `Parent directory not found: ${parentPath}`, path, 'mkdir') } } // Create directory entity const metadata: VFSMetadata = { path, name, parent: parentId, vfsType: 'directory', size: 0, permissions: options?.mode || this.config.permissions?.defaultDirectory || 0o755, owner: 'user', group: 'users', accessed: Date.now(), modified: Date.now(), ...options?.metadata } const entity = await this.brain.add({ data: path, // Directory path as string content type: NounType.Collection, metadata }) // Create parent-child relationship if (parentId !== entity) { // Don't relate to self (root) await this.brain.relate({ from: parentId, to: entity, type: VerbType.Contains }) } // Update path resolver cache await this.pathResolver.createPath(path, entity) // Trigger watchers this.triggerWatchers(path, 'rename') } /** * Remove a directory */ async rmdir(path: string, options?: { recursive?: boolean }): Promise { await this.ensureInitialized() if (path === '/') { throw new VFSError(VFSErrorCode.EACCES, 'Cannot remove root directory', path, 'rmdir') } const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Verify it's a directory if (entity.metadata.vfsType !== 'directory') { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'rmdir') } // Check if empty (unless recursive) const children = await this.pathResolver.getChildren(entityId) if (children.length > 0 && !options?.recursive) { throw new VFSError(VFSErrorCode.ENOTEMPTY, `Directory not empty: ${path}`, path, 'rmdir') } // Delete children recursively if needed if (options?.recursive) { for (const child of children) { // Use the child's actual path from metadata instead of constructing it const childPath = child.metadata.path if (child.metadata.vfsType === 'directory') { await this.rmdir(childPath, options) } else { await this.unlink(childPath) } } } // Delete the directory entity await this.brain.delete(entityId) // Invalidate caches this.pathResolver.invalidatePath(path, true) this.invalidateCaches(path) // Trigger watchers this.triggerWatchers(path, 'rename') } /** * Read directory contents */ async readdir(path: string, options?: ReaddirOptions): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Verify it's a directory if (entity.metadata.vfsType !== 'directory') { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'readdir') } // Get children let children = await this.pathResolver.getChildren(entityId) // Apply filters if (options?.filter) { children = this.filterDirectoryEntries(children, options.filter) } // Sort if requested if (options?.sort) { children = this.sortDirectoryEntries(children, options.sort, options.order) } // Apply pagination if (options?.offset) { children = children.slice(options.offset) } if (options?.limit) { children = children.slice(0, options.limit) } // Update access time await this.updateAccessTime(entityId) // Return appropriate format if (options?.withFileTypes) { return children.map(child => ({ name: child.metadata.name, path: child.metadata.path, type: child.metadata.vfsType, entityId: child.id } as VFSDirent)) } return children.map(child => child.metadata.name) } // ============= Metadata Operations ============= /** * Get file/directory statistics */ async stat(path: string): Promise { await this.ensureInitialized() // Check cache if (this.statCache.has(path)) { const cached = this.statCache.get(path)! if (Date.now() - cached.timestamp < 5000) { // 5 second cache return cached.stats } } const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) const stats: VFSStats = { size: entity.metadata.size, mode: entity.metadata.permissions, uid: 1000, // In production, map owner to UID gid: 1000, // In production, map group to GID atime: new Date(entity.metadata.accessed), mtime: new Date(entity.metadata.modified), ctime: new Date(entity.updatedAt || entity.createdAt), birthtime: new Date(entity.createdAt), isFile: () => entity.metadata.vfsType === 'file', isDirectory: () => entity.metadata.vfsType === 'directory', isSymbolicLink: () => entity.metadata.vfsType === 'symlink', path, entityId: entity.id, vector: entity.vector, connections: await this.countRelationships(entityId) } // Cache stats this.statCache.set(path, { stats, timestamp: Date.now() }) return stats } /** * lstat - same as stat for now (symlinks not fully implemented) */ async lstat(path: string): Promise { return this.stat(path) } /** * Check if path exists */ async exists(path: string): Promise { await this.ensureInitialized() try { await this.pathResolver.resolve(path) return true } catch (err) { return false } } // ============= Semantic Operations ============= /** * Search files with natural language */ async search(query: string, options?: SearchOptions): Promise { await this.ensureInitialized() // Build find params const params: FindParams = { query, type: [NounType.File, NounType.Document, NounType.Media], limit: options?.limit || 10, offset: options?.offset, explain: options?.explain } // Add path filter if specified if (options?.path) { params.where = { ...params.where, path: { $startsWith: options.path } } } // Add metadata filters if (options?.where) { Object.assign(params.where || {}, options.where) } // Execute search using Brainy's Triple Intelligence const results = await this.brain.find(params) // Convert to search results return results.map(r => { const entity = r.entity as VFSEntity return { path: entity.metadata.path, entityId: entity.id, score: r.score, type: entity.metadata.vfsType, size: entity.metadata.size, modified: new Date(entity.metadata.modified), explanation: r.explanation } }) } /** * Find files similar to a given file */ async findSimilar(path: string, options?: SimilarOptions): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) // Use Brainy's similarity search const results = await this.brain.similar({ to: entityId, limit: options?.limit || 10, threshold: options?.threshold || 0.7, type: [NounType.File, NounType.Document, NounType.Media] }) return results.map(r => { const entity = r.entity as VFSEntity return { path: entity.metadata.path, entityId: entity.id, score: r.score, type: entity.metadata.vfsType, size: entity.metadata.size, modified: new Date(entity.metadata.modified) } }) } // ============= Helper Methods ============= private async ensureInitialized(): Promise { if (!this.initialized) { throw new Error( 'VFS not initialized. You must call await vfs.init() after getting the VFS instance.\n' + 'Example:\n' + ' const vfs = brain.vfs() // Note: vfs() is a method, not a property\n' + ' await vfs.init() // This creates the root directory\n' + 'See docs: https://github.com/Brainy-Technologies/brainy/blob/main/docs/vfs/QUICK_START.md' ) } } private async ensureDirectory(path: string): Promise { if (!path || path === '/') { return this.rootEntityId! } try { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) if (entity.metadata.vfsType !== 'directory') { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path) } return entityId } catch (err) { // Only create directory if it doesn't exist (ENOENT error) if (err instanceof VFSError && err.code === VFSErrorCode.ENOENT) { await this.mkdir(path, { recursive: true }) return await this.pathResolver.resolve(path) } // Re-throw other errors (like ENOTDIR) throw err } } async getEntityById(id: string): Promise { const entity = await this.brain.get(id) if (!entity) { throw new VFSError(VFSErrorCode.ENOENT, `Entity not found: ${id}`) } // Ensure entity has proper VFS metadata structure // Handle both nested and flat metadata structures for compatibility if (!entity.metadata || !entity.metadata.vfsType) { // Check if metadata is at top level (legacy structure) const anyEntity = entity as any if (anyEntity.vfsType || anyEntity.path) { entity.metadata = { path: anyEntity.path || '/', name: anyEntity.name || '', vfsType: anyEntity.vfsType || (anyEntity.path === '/' ? 'directory' : 'file'), size: anyEntity.size || 0, permissions: anyEntity.permissions || (anyEntity.vfsType === 'directory' ? 0o755 : 0o644), owner: anyEntity.owner || 'user', group: anyEntity.group || 'users', accessed: anyEntity.accessed || Date.now(), modified: anyEntity.modified || Date.now(), ...entity.metadata // Preserve any existing nested metadata } } else if (entity.id === this.rootEntityId) { // Special case: ensure root directory always has proper metadata entity.metadata = { path: '/', name: '', vfsType: 'directory', size: 0, permissions: 0o755, owner: 'root', group: 'root', accessed: Date.now(), modified: Date.now(), ...entity.metadata } } } return entity as VFSEntity } private getParentPath(path: string): string { const normalized = path.replace(/\/+/g, '/').replace(/\/$/, '') const lastSlash = normalized.lastIndexOf('/') if (lastSlash <= 0) return '/' return normalized.substring(0, lastSlash) } private getBasename(path: string): string { const normalized = path.replace(/\/+/g, '/').replace(/\/$/, '') const lastSlash = normalized.lastIndexOf('/') return normalized.substring(lastSlash + 1) } private getExtension(filename: string): string | undefined { const lastDot = filename.lastIndexOf('.') if (lastDot === -1 || lastDot === 0) return undefined return filename.substring(lastDot + 1).toLowerCase() } private detectMimeType(filename: string, content: Buffer): string { const ext = this.getExtension(filename) // Common MIME types by extension const mimeTypes: Record = { txt: 'text/plain', html: 'text/html', css: 'text/css', js: 'application/javascript', json: 'application/json', pdf: 'application/pdf', jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', mp3: 'audio/mpeg', mp4: 'video/mp4', zip: 'application/zip' } return mimeTypes[ext || ''] || 'application/octet-stream' } private isTextFile(mimeType: string): boolean { return mimeType.startsWith('text/') || mimeType.includes('json') || mimeType.includes('javascript') || mimeType.includes('xml') || mimeType.includes('yaml') || mimeType === 'application/json' } private getFileNounType(mimeType: string): NounType { if (mimeType.startsWith('text/') || mimeType.includes('json')) { return NounType.Document } if (mimeType.startsWith('image/') || mimeType.startsWith('video/') || mimeType.startsWith('audio/')) { return NounType.Media } return NounType.File } private shouldCompress(buffer: Buffer): boolean { if (!this.config.storage?.compression?.enabled) return false if (buffer.length < (this.config.storage.compression.minSize || 10_000)) return false // Don't compress already compressed formats const firstBytes = buffer.slice(0, 4).toString('hex') const compressedSignatures = [ '504b0304', // ZIP '1f8b', // GZIP '425a', // BZIP2 '89504e47', // PNG 'ffd8ff' // JPEG ] return !compressedSignatures.some(sig => firstBytes.startsWith(sig)) } // External storage methods - leverages Brainy's storage adapters (memory, file, S3, R2) private async readExternalContent(key: string): Promise { // Read from Brainy - Brainy's storage adapter handles retrieval const entity = await this.brain.get(key) if (!entity) { throw new Error(`External content not found: ${key}`) } // Content is stored in the data field // Brainy handles storage/retrieval through its adapters (memory, file, S3, R2) return Buffer.isBuffer(entity.data) ? entity.data : Buffer.from(entity.data) } private async storeExternalContent(buffer: Buffer): Promise { // Store as Brainy entity - let Brainy's storage adapter handle it // Brainy automatically handles large data through its storage adapters (memory, file, S3, R2) const entityId = await this.brain.add({ data: buffer, // Store actual buffer - Brainy will handle it efficiently type: NounType.File, metadata: { vfsType: 'external-storage', size: buffer.length, created: Date.now() } }) return entityId } private async deleteExternalContent(key: string): Promise { // Delete the external storage entity try { await this.brain.delete(key) } catch (error) { console.debug('Failed to delete external content:', key, error) } } private async readChunkedContent(chunks: string[]): Promise { // Read all chunk entities and combine const buffers: Buffer[] = [] for (const chunkId of chunks) { const entity = await this.brain.get(chunkId) if (!entity) { throw new Error(`Chunk not found: ${chunkId}`) } // Read actual data from entity - Brainy handles storage const chunkBuffer = Buffer.isBuffer(entity.data) ? entity.data : Buffer.from(entity.data) buffers.push(chunkBuffer) } return Buffer.concat(buffers) } private async storeChunkedContent(buffer: Buffer): Promise { const chunkSize = this.config.storage?.chunking?.chunkSize || 5_000_000 // 5MB chunks const chunks: string[] = [] for (let i = 0; i < buffer.length; i += chunkSize) { const chunk = buffer.slice(i, Math.min(i + chunkSize, buffer.length)) // Store each chunk as a separate entity // Let Brainy handle the chunk data efficiently const chunkId = await this.brain.add({ data: chunk, // Store actual chunk - Brainy handles it type: NounType.File, metadata: { vfsType: 'chunk', chunkIndex: chunks.length, size: chunk.length, created: Date.now() } }) chunks.push(chunkId) } return chunks } private async deleteChunkedContent(chunks: string[]): Promise { // Delete all chunk entities await Promise.all( chunks.map(chunkId => this.brain.delete(chunkId).catch(err => console.debug('Failed to delete chunk:', chunkId, err) ) ) ) } private async compress(buffer: Buffer): Promise { const zlib = await import('zlib') return new Promise((resolve, reject) => { zlib.gzip(buffer, (err, compressed) => { if (err) reject(err) else resolve(compressed) }) }) } private async decompress(buffer: Buffer): Promise { const zlib = await import('zlib') return new Promise((resolve, reject) => { zlib.gunzip(buffer, (err, decompressed) => { if (err) reject(err) else resolve(decompressed) }) }) } private async generateEmbedding(buffer: Buffer, mimeType: string): Promise { try { // Use text content for text files, description for binary let content: string if (this.isTextFile(mimeType)) { // Use first 10KB for embedding content = buffer.toString('utf8', 0, Math.min(10240, buffer.length)) } else { // For binary files, create a description content = `Binary file: ${mimeType}, size: ${buffer.length} bytes` } // Ensure content is actually a string if (typeof content !== 'string') { console.debug('Content is not a string:', typeof content, content) return undefined } // Ensure content is not empty or invalid if (!content || content.length === 0) { console.debug('Content is empty') return undefined } const vector = await this.brain.embed(content) return vector } catch (error) { console.debug('Failed to generate embedding:', error) return undefined } } private async extractMetadata(buffer: Buffer, mimeType: string): Promise> { const metadata: Partial = {} // Extract basic metadata based on content type if (this.isTextFile(mimeType)) { const text = buffer.toString('utf8') metadata.lineCount = text.split('\n').length metadata.wordCount = text.split(/\s+/).filter(w => w).length metadata.charset = 'utf-8' } // Extract hash for integrity const crypto = await import('crypto') metadata.hash = crypto.createHash('sha256').update(buffer).digest('hex') return metadata } private async updateAccessTime(entityId: string): Promise { // Update access timestamp const entity = await this.getEntityById(entityId) await this.brain.update({ id: entityId, metadata: { ...entity.metadata, accessed: Date.now() } }) } private async countRelationships(entityId: string): Promise { const relations = await this.brain.getRelations({ from: entityId }) const relationsTo = await this.brain.getRelations({ to: entityId }) return relations.length + relationsTo.length } private filterDirectoryEntries(entries: VFSEntity[], filter: any): VFSEntity[] { return entries.filter(entry => { if (filter.type && entry.metadata.vfsType !== filter.type) return false if (filter.pattern && !this.matchGlob(entry.metadata.name, filter.pattern)) return false if (filter.minSize && entry.metadata.size < filter.minSize) return false if (filter.maxSize && entry.metadata.size > filter.maxSize) return false if (filter.modifiedAfter && entry.metadata.modified < filter.modifiedAfter.getTime()) return false if (filter.modifiedBefore && entry.metadata.modified > filter.modifiedBefore.getTime()) return false return true }) } private sortDirectoryEntries(entries: VFSEntity[], sort: string, order?: 'asc' | 'desc'): VFSEntity[] { const sorted = [...entries].sort((a, b) => { let comparison = 0 switch (sort) { case 'name': comparison = a.metadata.name.localeCompare(b.metadata.name) break case 'size': comparison = a.metadata.size - b.metadata.size break case 'modified': comparison = a.metadata.modified - b.metadata.modified break case 'created': comparison = a.createdAt - b.createdAt break } return order === 'desc' ? -comparison : comparison }) return sorted } private matchGlob(name: string, pattern: string): boolean { // Simple glob matching (in production, use proper glob library) const regex = pattern .replace(/\*/g, '.*') .replace(/\?/g, '.') return new RegExp(`^${regex}$`).test(name) } private invalidateCaches(path: string): void { this.contentCache.delete(path) this.statCache.delete(path) } private triggerWatchers(path: string, event: 'rename' | 'change'): void { const watchers = this.watchers.get(path) if (watchers) { for (const listener of watchers) { listener(event, path) } } } private async updateChildrenPaths(parentId: string, oldParentPath: string, newParentPath: string): Promise { // Get all children recursively const children = await this.pathResolver.getChildren(parentId) for (const child of children) { const oldChildPath = child.metadata.path as string const relativePath = oldChildPath.substring(oldParentPath.length) const newChildPath = newParentPath + relativePath // Update child entity const updatedChild = { ...child, metadata: { ...child.metadata, path: newChildPath, modified: Date.now() } } await this.brain.update({ ...updatedChild, id: child.id }) // Update path cache this.pathResolver.invalidatePath(oldChildPath) await this.pathResolver.createPath(newChildPath, child.id) // Recursively update if it's a directory if (child.metadata.vfsType === 'directory') { await this.updateChildrenPaths(child.id, oldChildPath, newChildPath) } } } private startBackgroundTasks(): void { // Clean up caches periodically this.backgroundTimer = setInterval(() => { const now = Date.now() // Clean content cache for (const [path, entry] of this.contentCache) { if (now - entry.timestamp > (this.config.cache?.ttl || 300000)) { this.contentCache.delete(path) } } // Clean stat cache for (const [path, entry] of this.statCache) { if (now - entry.timestamp > 5000) { this.statCache.delete(path) } } }, 60000) // Every minute } private getDefaultConfig(): Required> & { rootEntityId?: string } { return { root: '/', rootEntityId: undefined, cache: { enabled: true, maxPaths: 100_000, maxContent: 100_000_000, // 100MB ttl: 5 * 60 * 1000 // 5 minutes }, storage: { inline: { maxSize: 100_000 // 100KB }, chunking: { enabled: true, chunkSize: 5_000_000, // 5MB parallel: 4 }, compression: { enabled: true, minSize: 10_000, // 10KB algorithm: 'gzip' } }, intelligence: { enabled: true, autoEmbed: true, autoExtract: true, autoTag: false, autoConcepts: false }, permissions: { defaultFile: 0o644, defaultDirectory: 0o755, umask: 0o022 }, limits: { maxFileSize: 1_000_000_000, // 1GB maxPathLength: 4096, maxDirectoryEntries: 100_000 }, knowledgeLayer: { enabled: false // Default to disabled } } } // ============= Not Yet Implemented ============= async close(): Promise { // Cleanup PathResolver resources if (this.pathResolver) { this.pathResolver.cleanup() } // Stop background tasks if (this.backgroundTimer) { clearInterval(this.backgroundTimer) this.backgroundTimer = null } // Clear caches this.contentCache.clear() // Clear watchers this.watchers.clear() this.initialized = false } async chmod(path: string, mode: number): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Update permissions in metadata await this.brain.update({ ...entity, id: entityId, metadata: { ...entity.metadata, permissions: mode, modified: Date.now() } }) // Invalidate caches this.invalidateCaches(path) } async chown(path: string, uid: number, gid: number): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Update ownership in metadata await this.brain.update({ ...entity, id: entityId, metadata: { ...entity.metadata, uid, gid, modified: Date.now() } }) // Invalidate caches this.invalidateCaches(path) } async utimes(path: string, atime: Date, mtime: Date): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Update timestamps in metadata await this.brain.update({ ...entity, id: entityId, metadata: { ...entity.metadata, accessed: atime.getTime(), modified: mtime.getTime() } }) // Invalidate caches this.invalidateCaches(path) } async rename(oldPath: string, newPath: string): Promise { await this.ensureInitialized() // Check if source exists const entityId = await this.pathResolver.resolve(oldPath) const entity = await this.brain.get(entityId) if (!entity) { throw new VFSError(VFSErrorCode.ENOENT, `No such file or directory: ${oldPath}`, oldPath, 'rename') } // Check if target already exists try { await this.pathResolver.resolve(newPath) throw new VFSError(VFSErrorCode.EEXIST, `File exists: ${newPath}`, newPath, 'rename') } catch (err: any) { if (err.code !== VFSErrorCode.ENOENT) throw err } // Update entity metadata const updatedEntity = { ...entity, metadata: { ...entity.metadata, path: newPath, name: this.getBasename(newPath), modified: Date.now() } } // Update parent relationships if needed const oldParentPath = this.getParentPath(oldPath) const newParentPath = this.getParentPath(newPath) if (oldParentPath !== newParentPath) { // Remove from old parent if (oldParentPath) { const oldParentId = await this.pathResolver.resolve(oldParentPath) // unrelate takes the relation ID, not params - need to find and remove relation // For now, skip unrelate as it's not critical for rename } // Add to new parent if (newParentPath && newParentPath !== '/') { const newParentId = await this.pathResolver.resolve(newParentPath) await this.brain.relate({ from: newParentId, to: entityId, type: VerbType.Contains }) } } // Update the entity await this.brain.update({ ...updatedEntity, id: entityId }) // Update path cache this.pathResolver.invalidatePath(oldPath, true) await this.pathResolver.createPath(newPath, entityId) // If it's a directory, update all children paths if (entity.metadata.vfsType === 'directory') { await this.updateChildrenPaths(entityId, oldPath, newPath) } // Trigger watchers this.triggerWatchers(oldPath, 'rename') this.triggerWatchers(newPath, 'rename') } async copy(src: string, dest: string, options?: CopyOptions): Promise { await this.ensureInitialized() // Get source entity const srcEntityId = await this.pathResolver.resolve(src) const srcEntity = await this.brain.get(srcEntityId) if (!srcEntity) { throw new VFSError(VFSErrorCode.ENOENT, `No such file or directory: ${src}`, src, 'copy') } // Check if destination already exists if (!options?.overwrite) { try { await this.pathResolver.resolve(dest) throw new VFSError(VFSErrorCode.EEXIST, `File exists: ${dest}`, dest, 'copy') } catch (err: any) { if (err.code !== VFSErrorCode.ENOENT) throw err } } // Copy the entity if (srcEntity.metadata.vfsType === 'file') { await this.copyFile(srcEntity, dest, options) } else if (srcEntity.metadata.vfsType === 'directory') { await this.copyDirectory(src, dest, options) } } private async copyFile(srcEntity: Entity, destPath: string, options?: CopyOptions): Promise { // Create new entity with same content but different path const newEntity = await this.brain.add({ type: srcEntity.type, data: srcEntity.data, vector: options?.preserveVector ? srcEntity.vector : undefined, metadata: { ...srcEntity.metadata, path: destPath, name: this.getBasename(destPath), created: Date.now(), modified: Date.now(), copiedFrom: srcEntity.metadata.path } }) // Add to parent directory const parentPath = this.getParentPath(destPath) if (parentPath && parentPath !== '/') { const parentId = await this.pathResolver.resolve(parentPath) await this.brain.relate({ from: parentId, to: newEntity, type: VerbType.Contains }) } // Update path cache await this.pathResolver.createPath(destPath, newEntity) // Copy relationships if requested if (options?.preserveRelationships) { const relations = await this.brain.getRelations({ from: srcEntity.id }) for (const relation of relations) { if (relation.type !== VerbType.Contains) { // Skip relationship without Contains type // Future: implement proper relation copying } } } } private async copyDirectory(srcPath: string, destPath: string, options?: CopyOptions): Promise { // Create destination directory await this.mkdir(destPath, { recursive: true }) // Copy all children if (options?.deepCopy !== false) { const children = await this.readdir(srcPath, { withFileTypes: true }) as VFSDirent[] for (const child of children) { const srcChildPath = `${srcPath}/${child.name}` const destChildPath = `${destPath}/${child.name}` if (child.type === 'file') { const childEntity = await this.brain.get(child.entityId) await this.copyFile(childEntity!, destChildPath, options) } else if (child.type === 'directory') { await this.copyDirectory(srcChildPath, destChildPath, options) } } } } async move(src: string, dest: string): Promise { await this.ensureInitialized() // Move is just copy + delete await this.copy(src, dest, { overwrite: false }) // Delete source after successful copy const srcEntityId = await this.pathResolver.resolve(src) const srcEntity = await this.brain.get(srcEntityId) if (srcEntity!.metadata.vfsType === 'file') { await this.unlink(src) } else { await this.rmdir(src, { recursive: true }) } } async symlink(target: string, path: string): Promise { await this.ensureInitialized() // Check if symlink already exists try { await this.pathResolver.resolve(path) throw new VFSError(VFSErrorCode.EEXIST, `File exists: ${path}`, path, 'symlink') } catch (err: any) { if (err.code !== VFSErrorCode.ENOENT) throw err } // Parse path to get parent and name const parentPath = this.getParentPath(path) const name = this.getBasename(path) // Ensure parent directory exists const parentId = await this.ensureDirectory(parentPath) // Create symlink entity const metadata: VFSMetadata = { path, name, parent: parentId, vfsType: 'symlink', symlinkTarget: target, size: 0, permissions: 0o777, owner: 'user', group: 'users', accessed: Date.now(), modified: Date.now() } const entity = await this.brain.add({ data: `symlink:${target}`, type: NounType.File, // Symlinks are special files metadata }) // Create parent-child relationship await this.brain.relate({ from: parentId, to: entity, type: VerbType.Contains }) // Update path resolver cache await this.pathResolver.createPath(path, entity) } async readlink(path: string): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Verify it's a symlink if (entity.metadata.vfsType !== 'symlink') { throw new VFSError(VFSErrorCode.EINVAL, `Not a symbolic link: ${path}`, path, 'readlink') } return entity.metadata.symlinkTarget || '' } async realpath(path: string): Promise { await this.ensureInitialized() // Resolve symlinks recursively let currentPath = path let depth = 0 const maxDepth = 20 // Prevent infinite loops while (depth < maxDepth) { try { const entityId = await this.pathResolver.resolve(currentPath) const entity = await this.getEntityById(entityId) if (entity.metadata.vfsType === 'symlink') { // Follow the symlink currentPath = entity.metadata.symlinkTarget || '' depth++ } else { // Not a symlink, we have the real path return currentPath } } catch (err) { throw new VFSError(VFSErrorCode.ENOENT, `No such file or directory: ${path}`, path, 'realpath') } } throw new VFSError(VFSErrorCode.ELOOP, `Too many symbolic links: ${path}`, path, 'realpath') } async getxattr(path: string, name: string): Promise { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return entity.metadata.attributes?.[name] } async setxattr(path: string, name: string, value: any): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Create extended attributes object const xattrs = entity.metadata.attributes || {} xattrs[name] = value // Update entity metadata await this.brain.update({ id: entityId, metadata: { ...entity.metadata, attributes: xattrs } }) // Invalidate caches this.invalidateCaches(path) } async listxattr(path: string): Promise { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return Object.keys(entity.metadata.attributes || {}) } async removexattr(path: string, name: string): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Remove from extended attributes const xattrs = { ...entity.metadata.attributes } delete xattrs[name] // Update entity metadata await this.brain.update({ ...entity, id: entityId, metadata: { ...entity.metadata, attributes: xattrs } }) // Invalidate caches this.invalidateCaches(path) } async getRelated(path: string, options?: RelatedOptions): Promise> { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const results: Array<{ path: string, relationship: string, direction: 'from' | 'to' }> = [] // Use proper Brainy relationship API to get all relationships const [fromRelations, toRelations] = await Promise.all([ this.brain.getRelations({ from: entityId }), this.brain.getRelations({ to: entityId }) ]) // Add outgoing relationships for (const rel of fromRelations) { const targetEntity = await this.brain.get(rel.to) if (targetEntity && targetEntity.metadata?.path) { results.push({ path: targetEntity.metadata.path, relationship: rel.type || 'related', direction: 'from' }) } } // Add incoming relationships for (const rel of toRelations) { const sourceEntity = await this.brain.get(rel.from) if (sourceEntity && sourceEntity.metadata?.path) { results.push({ path: sourceEntity.metadata.path, relationship: rel.type || 'related', direction: 'to' }) } } return results } async getRelationships(path: string): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const relationships: Relation[] = [] // Use proper Brainy relationship API const [fromRelations, toRelations] = await Promise.all([ this.brain.getRelations({ from: entityId }), this.brain.getRelations({ to: entityId }) ]) // Process outgoing relationships (excluding Contains for parent-child) for (const rel of fromRelations) { if (rel.type !== VerbType.Contains) { // Skip filesystem hierarchy const targetEntity = await this.brain.get(rel.to) if (targetEntity && targetEntity.metadata?.path) { relationships.push({ id: rel.id || crypto.randomUUID(), from: entityId, to: rel.to, type: rel.type, createdAt: rel.createdAt || Date.now() }) } } } // Process incoming relationships (excluding Contains for parent-child) for (const rel of toRelations) { if (rel.type !== VerbType.Contains) { // Skip filesystem hierarchy const sourceEntity = await this.brain.get(rel.from) if (sourceEntity && sourceEntity.metadata?.path) { relationships.push({ id: rel.id || crypto.randomUUID(), from: rel.from, to: entityId, type: rel.type, createdAt: rel.createdAt || Date.now() }) } } } return relationships } async addRelationship(from: string, to: string, type: string): Promise { await this.ensureInitialized() const fromEntityId = await this.pathResolver.resolve(from) const toEntityId = await this.pathResolver.resolve(to) // Create relationship using brain await this.brain.relate({ from: fromEntityId, to: toEntityId, type: type as any // Convert string to VerbType }) // Invalidate caches for both paths this.invalidateCaches(from) this.invalidateCaches(to) } async removeRelationship(from: string, to: string, type?: string): Promise { await this.ensureInitialized() const fromEntityId = await this.pathResolver.resolve(from) const toEntityId = await this.pathResolver.resolve(to) // Find and delete the relationship const relations = await this.brain.getRelations({ from: fromEntityId }) for (const relation of relations) { if (relation.to === toEntityId && (!type || relation.type === type)) { // Delete the relationship using brain.unrelate if (relation.id) { await this.brain.unrelate(relation.id) } } } // Invalidate caches this.invalidateCaches(from) this.invalidateCaches(to) } async getTodos(path: string): Promise { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return entity.metadata.todos } async setTodos(path: string, todos: VFSTodo[]): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Update todos in metadata await this.brain.update({ ...entity, id: entityId, metadata: { ...entity.metadata, todos, modified: Date.now() } }) // Invalidate caches this.invalidateCaches(path) } async addTodo(path: string, todo: VFSTodo): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Get existing todos const todos = entity.metadata.todos || [] // Add new todo with ID if not provided const newTodo: VFSTodo = { id: todo.id || crypto.randomUUID(), task: todo.task, priority: todo.priority || 'medium', status: todo.status || 'pending', assignee: todo.assignee, due: todo.due } todos.push(newTodo) // Update entity metadata await this.brain.update({ id: entityId, metadata: { ...entity.metadata, todos } }) // Invalidate caches this.invalidateCaches(path) } /** * Get metadata for a file or directory */ async getMetadata(path: string): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) return entity.metadata } /** * Set custom metadata for a file or directory * Merges with existing metadata */ async setMetadata(path: string, metadata: Partial): Promise { await this.ensureInitialized() const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Merge with existing metadata await this.brain.update({ id: entityId, metadata: { ...entity.metadata, ...metadata, modified: Date.now() } }) // Invalidate caches this.invalidateCaches(path) } // ============= Knowledge Layer ============= // Knowledge Layer methods are added by KnowledgeLayer.enable() // This keeps VFS pure and fast while allowing optional intelligence /** * Enable Knowledge Layer on this VFS instance */ async enableKnowledgeLayer(): Promise { const { enableKnowledgeLayer } = await import('./KnowledgeLayer.js') await enableKnowledgeLayer(this, this.brain) } /** * Set the current user for tracking who makes changes */ setUser(username: string): void { this.currentUser = username || 'system' } /** * Get the current user */ getCurrentUser(): string { return this.currentUser } /** * Get all todos recursively from a path */ async getAllTodos(path: string = '/'): Promise { await this.ensureInitialized() const allTodos: VFSTodo[] = [] // Get entity for this path try { const entityId = await this.pathResolver.resolve(path) const entity = await this.getEntityById(entityId) // Add todos from this entity if (entity.metadata.todos) { allTodos.push(...entity.metadata.todos) } // If it's a directory, recursively get todos from children if (entity.metadata.vfsType === 'directory') { const children = await this.readdir(path) for (const child of children) { const childPath = path === '/' ? `/${child}` : `${path}/${child}` const childTodos = await this.getAllTodos(childPath) allTodos.push(...childTodos) } } } catch (error) { // Path doesn't exist, return empty } return allTodos } /** * Export directory structure to JSON */ async exportToJSON(path: string = '/'): Promise { await this.ensureInitialized() const result: any = {} const traverse = async (currentPath: string, target: any) => { try { const entityId = await this.pathResolver.resolve(currentPath) const entity = await this.getEntityById(entityId) if (entity.metadata.vfsType === 'directory') { // Add directory metadata target._meta = { type: 'directory', path: currentPath, modified: entity.metadata.modified ? new Date(entity.metadata.modified) : undefined } // Traverse children const children = await this.readdir(currentPath) for (const child of children) { const childName = typeof child === 'string' ? child : child.name const childPath = currentPath === '/' ? `/${childName}` : `${currentPath}/${childName}` target[childName] = {} await traverse(childPath, target[childName]) } } else if (entity.metadata.vfsType === 'file') { // For files, include content and metadata try { const content = await this.readFile(currentPath) const textContent = content.toString('utf8') // Try to parse JSON files if (currentPath.endsWith('.json')) { try { target._content = JSON.parse(textContent) } catch { target._content = textContent } } else { target._content = textContent } } catch { // Binary or unreadable file target._content = '[binary]' } target._meta = { type: 'file', path: currentPath, size: entity.metadata.size || 0, mimeType: entity.metadata.mimeType, modified: entity.metadata.modified ? new Date(entity.metadata.modified) : undefined, todos: entity.metadata.todos || [] } } } catch (error) { // Skip inaccessible paths target._error = 'inaccessible' } } await traverse(path, result) return result } /** * Search for entities with filters */ async searchEntities(query: { type?: string name?: string where?: Record limit?: number }): Promise> { await this.ensureInitialized() // Build query for brain.find() const searchQuery: any = { where: { ...query.where, vfsType: 'entity' }, limit: query.limit || 100 } if (query.type) { searchQuery.where.entityType = query.type } if (query.name) { searchQuery.query = query.name } const results = await this.brain.find(searchQuery) return results.map(result => ({ id: result.id, path: result.entity?.metadata?.path || '', type: result.entity?.metadata?.type || result.entity?.metadata?.entityType || 'unknown', metadata: result.entity?.metadata || {} })) } /** * Bulk write operations for performance */ async bulkWrite(operations: Array<{ type: 'write' | 'delete' | 'mkdir' | 'update' path: string data?: Buffer | string options?: any }>): Promise<{ successful: number failed: Array<{ operation: any, error: string }> }> { await this.ensureInitialized() const result = { successful: 0, failed: [] as Array<{ operation: any, error: string }> } // Process operations in batches for better performance const batchSize = 10 for (let i = 0; i < operations.length; i += batchSize) { const batch = operations.slice(i, i + batchSize) // Process batch in parallel const promises = batch.map(async (op) => { try { switch (op.type) { case 'write': await this.writeFile(op.path, op.data || '', op.options) break case 'delete': await this.unlink(op.path) break case 'mkdir': await this.mkdir(op.path, op.options) break case 'update': { // Update only metadata without changing content const entityId = await this.pathResolver.resolve(op.path) await this.brain.update({ id: entityId, metadata: op.options?.metadata }) break } } result.successful++ } catch (error: any) { result.failed.push({ operation: op, error: error.message || 'Unknown error' }) } }) await Promise.all(promises) } return result } /** * Get project statistics for a path */ async getProjectStats(path: string = '/'): Promise<{ fileCount: number directoryCount: number totalSize: number todoCount: number averageFileSize: number largestFile: { path: string, size: number } | null modifiedRange: { earliest: Date, latest: Date } | null }> { await this.ensureInitialized() const stats = { fileCount: 0, directoryCount: 0, totalSize: 0, todoCount: 0, averageFileSize: 0, largestFile: null as { path: string, size: number } | null, modifiedRange: null as { earliest: Date, latest: Date } | null } let earliestModified: number | null = null let latestModified: number | null = null const traverse = async (currentPath: string) => { try { const entityId = await this.pathResolver.resolve(currentPath) const entity = await this.getEntityById(entityId) if (entity.metadata.vfsType === 'directory') { stats.directoryCount++ // Traverse children const children = await this.readdir(currentPath) for (const child of children) { const childPath = currentPath === '/' ? `/${child}` : `${currentPath}/${child}` await traverse(childPath) } } else if (entity.metadata.vfsType === 'file') { stats.fileCount++ const size = entity.metadata.size || 0 stats.totalSize += size // Track largest file if (!stats.largestFile || size > stats.largestFile.size) { stats.largestFile = { path: currentPath, size } } // Track modification times const modified = entity.metadata.modified if (modified) { if (!earliestModified || modified < earliestModified) { earliestModified = modified } if (!latestModified || modified > latestModified) { latestModified = modified } } // Count todos if (entity.metadata.todos) { stats.todoCount += entity.metadata.todos.length } } } catch (error) { // Skip if path doesn't exist } } await traverse(path) // Calculate averages if (stats.fileCount > 0) { stats.averageFileSize = Math.round(stats.totalSize / stats.fileCount) } // Set date range if (earliestModified && latestModified) { stats.modifiedRange = { earliest: new Date(earliestModified), latest: new Date(latestModified) } } return stats } /** * Get all versions of a file (semantic versioning) */ createReadStream(path: string, options?: ReadStreamOptions): NodeJS.ReadableStream { // Lazy import to avoid circular dependencies const { VFSReadStream } = require('./streams/VFSReadStream.js') return new VFSReadStream(this, path, options) } createWriteStream(path: string, options?: WriteStreamOptions): NodeJS.WritableStream { // Lazy import to avoid circular dependencies const { VFSWriteStream } = require('./streams/VFSWriteStream.js') return new VFSWriteStream(this, path, options) } watch(path: string, listener: WatchListener): { close(): void } { if (!this.watchers.has(path)) { this.watchers.set(path, new Set()) } this.watchers.get(path)!.add(listener) return { close: () => { const watchers = this.watchers.get(path) if (watchers) { watchers.delete(listener) if (watchers.size === 0) { this.watchers.delete(path) } } } } } // ============= Import/Export Operations ============= /** * Import a single file from the real filesystem into VFS */ async importFile(sourcePath: string, targetPath: string): Promise { const fs = await import('fs/promises') const pathModule = await import('path') // Read file from local filesystem const content = await fs.readFile(sourcePath) const stats = await fs.stat(sourcePath) // Ensure parent directory exists in VFS const parentPath = pathModule.dirname(targetPath) if (parentPath !== '/' && parentPath !== '.') { try { await this.mkdir(parentPath, { recursive: true }) } catch (error: any) { if (error.code !== 'EEXIST') throw error } } // Write to VFS with metadata from source await this.writeFile(targetPath, content, { metadata: { imported: true, importedFrom: sourcePath, sourceSize: stats.size, sourceMtime: stats.mtime.getTime(), sourceMode: stats.mode } }) } /** * Import a directory from the real filesystem into VFS */ async importDirectory(sourcePath: string, options?: any): Promise { const { DirectoryImporter } = await import('./importers/DirectoryImporter.js') const importer = new DirectoryImporter(this, this.brain) return await importer.import(sourcePath, options) } /** * Import a directory with progress tracking */ async *importStream(sourcePath: string, options?: any): AsyncGenerator { const { DirectoryImporter } = await import('./importers/DirectoryImporter.js') const importer = new DirectoryImporter(this, this.brain) yield* importer.importStream(sourcePath, options) } watchFile(path: string, listener: WatchListener): void { this.watch(path, listener) } unwatchFile(path: string): void { this.watchers.delete(path) } async getEntity(path: string): Promise { const entityId = await this.pathResolver.resolve(path) return this.getEntityById(entityId) } async resolvePath(path: string, from?: string): Promise { // Handle relative paths if (!path.startsWith('/') && from) { path = `${from}/${path}` } // Normalize path const normalizedPath = path.replace(/\/+/g, '/').replace(/\/$/, '') || '/' // Special case for root if (normalizedPath === '/') { return this.rootEntityId! } // Resolve the path to an entity ID return await this.pathResolver.resolve(normalizedPath) } }