From 8393d01209773082ca50324a16e6e21b589f9e1e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 27 Oct 2025 10:44:06 -0700 Subject: [PATCH] feat(vfs): fix VFS visibility by removing broken filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove includeVFS parameter and broken isVFS filtering logic - Add excludeVFS parameter for optional VFS entity filtering - VFS entities now part of knowledge graph by default - Enable O(1) graph adjacency optimizations for VFS operations - Update all VFS projections and PathResolver - Add comprehensive VFS visibility documentation This fixes the bug where VFS operations returned empty results due to operator object mismatch in storage adapters. VFS relationships now use proper graph traversal without metadata filtering. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docs/FIND_SYSTEM.md | 77 +++++++++++++++++++ src/brainy.ts | 52 ++++++------- src/cli/commands/core.ts | 6 +- src/types/brainy.types.ts | 15 +--- src/vfs/PathResolver.ts | 13 ++-- src/vfs/VirtualFileSystem.ts | 12 +-- .../semantic/projections/AuthorProjection.ts | 13 ++-- src/vfs/semantic/projections/TagProjection.ts | 17 ++-- .../projections/TemporalProjection.ts | 14 ++-- 9 files changed, 134 insertions(+), 85 deletions(-) diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 6e125e06..fc5cba23 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -484,4 +484,81 @@ async function getDocumentsByDate(page: number, pageSize: number = 20) { } ``` +## VFS (Virtual File System) Visibility (v4.7.0+) + +### Default Behavior + +**VFS entities are now part of the knowledge graph** and included in query results by default: + +```typescript +// Default: Searches ALL entities including VFS files +await brain.find({ query: 'authentication setup' }) +// Returns: concepts, papers, AND markdown documentation files +``` + +**Why this change?** VFS files (imported markdown, PDFs, etc.) ARE knowledge entities. When you import documentation or papers into Brainy, you want to search them! + +### Excluding VFS Entities + +If you need to exclude VFS entities from specific queries, use the `excludeVFS` parameter: + +```typescript +// Exclude VFS files from results +await brain.find({ + query: 'machine learning', + excludeVFS: true // Only return non-file entities +}) +``` + +**Alternative**: Use explicit where clause for more control: + +```typescript +// Explicit filtering (same as excludeVFS: true) +await brain.find({ + query: 'machine learning', + where: { vfsType: { exists: false } } +}) + +// Or only search VFS files +await brain.find({ + query: 'setup instructions', + where: { vfsType: 'file' } // Only files +}) +``` + +### Performance + +**VFS filtering is production-scale:** +- Uses MetadataIndex (O(1) for exists checks) +- No performance penalty - same speed as any metadata filter +- Works seamlessly with vector + metadata + graph queries + +### Migration from v4.6.x + +**BREAKING CHANGE** (v4.7.0): The `includeVFS` parameter has been removed: + +```typescript +// ❌ Old (v4.6.x and earlier) +await brain.find({ + query: 'docs', + includeVFS: true // No longer needed! +}) + +// ✅ New (v4.7.0+) +await brain.find({ + query: 'docs' // VFS included by default +}) + +// ✅ To exclude VFS (if needed) +await brain.find({ + query: 'concepts', + excludeVFS: true +}) +``` + +**Why removed?** The old `includeVFS` parameter was: +1. Broken (metadata filter incompatibility with storage adapters) +2. Confusing (double-negative logic) +3. Wrong default (VFS should be searchable) + This system represents the most advanced query intelligence available in any database, combining the speed of specialized indices with the intelligence of natural language understanding and the power of graph relationships. \ No newline at end of file diff --git a/src/brainy.ts b/src/brainy.ts index c6d4c163..11c1a480 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1022,13 +1022,8 @@ export class Brainy implements BrainyInterface { filter.service = params.service } - // v4.5.1: Exclude VFS relationships by default (same pattern as brain.find()) - // VFS relationships have metadata.isVFS = true - // Only include VFS relationships if explicitly requested - if (params.includeVFS !== true) { - filter.metadata = filter.metadata || {} - filter.metadata.isVFS = { notEquals: true } - } + // v4.7.0: VFS relationships are no longer filtered + // VFS is part of the knowledge graph - users can filter explicitly if needed // Fetch from storage with pagination at storage layer (efficient!) const result = await this.storage.getVerbs({ @@ -1207,18 +1202,23 @@ export class Brainy implements BrainyInterface { * // Returns only knowledge entities, VFS files excluded * * @example - * // Include VFS entities when needed + * // v4.7.0: VFS entities included by default * const everything = await brainy.find({ - * query: 'documentation', - * includeVFS: true // Opt-in to include VFS files + * query: 'documentation' * }) * // Returns both knowledge entities AND VFS files * * @example * // Search only VFS files * const files = await brainy.find({ - * where: { vfsType: 'file', extension: '.md' }, - * includeVFS: true // Required to find VFS entities + * where: { vfsType: 'file', extension: '.md' } + * }) + * + * @example + * // Exclude VFS entities (if needed) + * const concepts = await brainy.find({ + * query: 'machine learning', + * excludeVFS: true // v4.7.0: Exclude VFS files * }) */ async find(query: string | FindParams): Promise[]> { @@ -1274,11 +1274,10 @@ export class Brainy implements BrainyInterface { if (params.where) Object.assign(filter, params.where) if (params.service) filter.service = params.service - // v4.3.3: Exclude VFS entities by default (Option 3C architecture) - // Only include VFS if explicitly requested via includeVFS: true - // BUT: Don't add automatic exclusion if user explicitly queries isVFS in where clause - if (params.includeVFS !== true && !params.where?.hasOwnProperty('isVFS')) { - filter.isVFS = { notEquals: true } + // v4.7.0: excludeVFS helper for cleaner UX + // Use vfsType field (more semantic than isVFS) + if (params.excludeVFS === true) { + filter.vfsType = { exists: false } } if (params.type) { @@ -1330,13 +1329,13 @@ export class Brainy implements BrainyInterface { const limit = params.limit || 20 const offset = params.offset || 0 - // v4.3.3: Apply VFS filtering even for empty queries + // v4.7.0: excludeVFS helper let filter: any = {} - if (params.includeVFS !== true) { - filter.isVFS = { notEquals: true } + if (params.excludeVFS === true) { + filter.vfsType = { exists: false } } - // Use metadata index if we need to filter VFS + // Use metadata index if we need to filter if (Object.keys(filter).length > 0) { const filteredIds = await this.metadataIndex.getIdsForFilter(filter) const pageIds = filteredIds.slice(offset, offset + limit) @@ -1399,7 +1398,7 @@ export class Brainy implements BrainyInterface { } // Apply O(log n) metadata filtering using core MetadataIndexManager - if (params.where || params.type || params.service || params.includeVFS !== true) { + if (params.where || params.type || params.service || params.excludeVFS) { // Build filter object for metadata index let filter: any = {} @@ -1407,10 +1406,9 @@ export class Brainy implements BrainyInterface { if (params.where) Object.assign(filter, params.where) if (params.service) filter.service = params.service - // v4.3.3: Exclude VFS entities by default (Option 3C architecture) - // BUT: Don't add automatic exclusion if user explicitly queries isVFS in where clause - if (params.includeVFS !== true && !params.where?.hasOwnProperty('isVFS')) { - filter.isVFS = { notEquals: true } + // v4.7.0: excludeVFS helper for cleaner UX + if (params.excludeVFS === true) { + filter.vfsType = { exists: false } } if (params.type) { @@ -1704,7 +1702,7 @@ export class Brainy implements BrainyInterface { type: params.type, where: params.where, service: params.service, - includeVFS: params.includeVFS // v4.4.0: Pass through VFS filtering + excludeVFS: params.excludeVFS // v4.7.0: Pass through VFS filtering }) } diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index 40ebb119..5b2f8473 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -347,10 +347,8 @@ export const coreCommands = { searchParams.includeRelations = true } - // Include VFS files (v4.4.0 - find excludes VFS by default) - if (options.includeVfs) { - searchParams.includeVFS = true - } + // v4.7.0: VFS is now part of the knowledge graph (included by default) + // Users can exclude VFS with --where vfsType exists:false if needed // Triple Intelligence Fusion - custom weighting if (options.fusion || options.vectorWeight || options.graphWeight || options.fieldWeight) { diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index d80c147f..743eff40 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -189,7 +189,7 @@ export interface FindParams { mode?: SearchMode // Search strategy explain?: boolean // Return scoring explanation includeRelations?: boolean // Include entity relationships - includeVFS?: boolean // v4.3.3: Include VFS entities (default: false for knowledge queries) + excludeVFS?: boolean // v4.7.0: Exclude VFS entities from results (default: false - VFS included) service?: string // Multi-tenancy filter // Triple Intelligence Fusion @@ -239,7 +239,7 @@ export interface SimilarParams { type?: NounType | NounType[] // Restrict to types where?: Partial // Additional filters service?: string // Multi-tenancy - includeVFS?: boolean // v4.4.0: Include VFS entities (default: false) + excludeVFS?: boolean // v4.7.0: Exclude VFS entities (default: false - VFS included) } /** @@ -328,17 +328,6 @@ export interface GetRelationsParams { * Only return relationships belonging to this service. */ service?: string - - /** - * Include VFS relationships (v4.5.1) - * - * By default, getRelations() excludes VFS relationships (since v4.4.0). - * Set this to true when you need to traverse VFS structure. - * - * @default false - * @since v4.5.1 - */ - includeVFS?: boolean } // ============= Batch Operations ============= diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index 4d8d3de5..c683dbb2 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -195,12 +195,11 @@ export class PathResolver { // Still need to verify it exists } - // Use proper graph traversal to find children - // Get all relationships where parentId contains other entities + // v4.7.0: Use proper graph traversal to find children + // VFS relationships are now part of the knowledge graph const relations = await this.brain.getRelations({ from: parentId, - type: VerbType.Contains, - includeVFS: true // v4.5.1: Required to see VFS relationships + type: VerbType.Contains }) // Find the child with matching name @@ -225,11 +224,11 @@ export class PathResolver { * Uses proper graph relationships to traverse the tree */ async getChildren(dirId: string): Promise { - // Production-ready: Use graph relationships (VFS creates these in mkdir/writeFile) + // v4.7.0: Use O(1) graph relationships (VFS creates these in mkdir/writeFile) + // VFS relationships are now part of the knowledge graph (no special filtering needed) const relations = await this.brain.getRelations({ from: dirId, - type: VerbType.Contains, - includeVFS: true // v4.5.1: Required to see VFS relationships + type: VerbType.Contains }) const validChildren: VFSEntity[]= [] diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 8a9ce6a1..ef7e8ff4 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -167,8 +167,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { path: '/', // ✅ Correct field name vfsType: 'directory' // ✅ Correct field name }, - limit: 10, - includeVFS: true // v4.4.0: CRITICAL - Must find VFS root entity! + limit: 10 }) if (existing.length > 0) { @@ -961,9 +960,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { limit: options?.limit || 10, offset: options?.offset, explain: options?.explain, - includeVFS: true, // v4.4.0: VFS search must include VFS entities! where: { - vfsType: 'file' // v4.4.0: Only search VFS files, not knowledge documents + vfsType: 'file' // v4.7.0: Search VFS files } } @@ -1012,9 +1010,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { limit: options?.limit || 10, threshold: options?.threshold || 0.7, type: [NounType.File, NounType.Document, NounType.Media], - includeVFS: true, // v4.4.0: VFS similarity search must include VFS entities! where: { - vfsType: 'file' // v4.4.0: Only find similar VFS files, not knowledge documents + vfsType: 'file' // v4.7.0: Find similar VFS files } }) @@ -2341,8 +2338,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { ...query.where, vfsType: 'entity' }, - limit: query.limit || 100, - includeVFS: true // v4.4.0: VFS entity search must include VFS entities! + limit: query.limit || 100 } if (query.type) { diff --git a/src/vfs/semantic/projections/AuthorProjection.ts b/src/vfs/semantic/projections/AuthorProjection.ts index 8c35781a..f77e018d 100644 --- a/src/vfs/semantic/projections/AuthorProjection.ts +++ b/src/vfs/semantic/projections/AuthorProjection.ts @@ -31,8 +31,7 @@ export class AuthorProjection extends BaseProjectionStrategy { vfsType: 'file', owner: authorName }, - limit: 1000, - includeVFS: true // v4.4.0: Must include VFS entities! + limit: 1000 } // Filter by filename if subpath specified @@ -53,14 +52,13 @@ export class AuthorProjection extends BaseProjectionStrategy { * Resolve author to entity IDs using REAL Brainy.find() */ async resolve(brain: Brainy, vfs: VirtualFileSystem, authorName: string): Promise { - // Use REAL Brainy metadata filtering + // v4.7.0: VFS entities are part of the knowledge graph const results = await brain.find({ where: { vfsType: 'file', owner: authorName }, - limit: 1000, - includeVFS: true // v4.4.0: Must include VFS entities! + limit: 1000 }) return this.extractIds(results) @@ -75,10 +73,9 @@ export class AuthorProjection extends BaseProjectionStrategy { const results = await brain.find({ where: { vfsType: 'file', - owner: { $exists: true } + owner: { exists: true } }, - limit, - includeVFS: true // v4.4.0: Must include VFS entities! + limit }) return results.map(r => r.entity as VFSEntity) diff --git a/src/vfs/semantic/projections/TagProjection.ts b/src/vfs/semantic/projections/TagProjection.ts index 97e737b2..1ebe18eb 100644 --- a/src/vfs/semantic/projections/TagProjection.ts +++ b/src/vfs/semantic/projections/TagProjection.ts @@ -29,10 +29,9 @@ export class TagProjection extends BaseProjectionStrategy { const query: FindParams = { where: { vfsType: 'file', - tags: { contains: tagName } // BFO operator for array contains + tags: { contains: tagName } // contains operator for array search }, - limit: 1000, - includeVFS: true // v4.4.0: Must include VFS entities! + limit: 1000 } // Filter by filename if subpath specified @@ -53,14 +52,13 @@ export class TagProjection extends BaseProjectionStrategy { * Resolve tag to entity IDs using REAL Brainy.find() */ async resolve(brain: Brainy, vfs: VirtualFileSystem, tagName: string): Promise { - // Use REAL Brainy metadata filtering + // v4.7.0: VFS entities are part of the knowledge graph const results = await brain.find({ where: { vfsType: 'file', - tags: { contains: tagName } // BFO operator + tags: { contains: tagName } }, - limit: 1000, - includeVFS: true // v4.4.0: Must include VFS entities! + limit: 1000 }) return this.extractIds(results) @@ -74,10 +72,9 @@ export class TagProjection extends BaseProjectionStrategy { const results = await brain.find({ where: { vfsType: 'file', - tags: { exists: true } // BFO operator + tags: { exists: true } // exists operator }, - limit, - includeVFS: true // v4.4.0: Must include VFS entities! + limit }) return results.map(r => r.entity as VFSEntity) diff --git a/src/vfs/semantic/projections/TemporalProjection.ts b/src/vfs/semantic/projections/TemporalProjection.ts index ba83ff04..4347e830 100644 --- a/src/vfs/semantic/projections/TemporalProjection.ts +++ b/src/vfs/semantic/projections/TemporalProjection.ts @@ -69,17 +69,16 @@ export class TemporalProjection extends BaseProjectionStrategy { const endOfDay = new Date(date) endOfDay.setHours(23, 59, 59, 999) - // Use REAL Brainy metadata filtering with range operators + // v4.7.0: VFS entities are part of the knowledge graph const results = await brain.find({ where: { vfsType: 'file', modified: { - greaterEqual: startOfDay.getTime(), // BFO operator - lessEqual: endOfDay.getTime() // BFO operator + greaterEqual: startOfDay.getTime(), + lessEqual: endOfDay.getTime() } }, - limit: 1000, - includeVFS: true // v4.4.0: Must include VFS entities! + limit: 1000 }) return this.extractIds(results) @@ -94,10 +93,9 @@ export class TemporalProjection extends BaseProjectionStrategy { const results = await brain.find({ where: { vfsType: 'file', - modified: { greaterEqual: oneDayAgo } // BFO operator + modified: { greaterEqual: oneDayAgo } }, - limit, - includeVFS: true // v4.4.0: Must include VFS entities! + limit }) return results.map(r => r.entity as VFSEntity)