feat: brain.find() excludes VFS by default (Option 3C)

Added includeVFS parameter to FindParams:
- brain.find() excludes VFS entities by default (clean knowledge graph)
- Opt-in with brain.find({ includeVFS: true })
- Automatically excludes VFS in all query paths (empty, metadata, vector)
- Respects explicit where: { isVFS: ... } queries

Implementation:
- Empty query path: Apply VFS filtering even with no criteria
- Metadata query path: Filter out isVFS: true by default
- Vector search path: Apply VFS filter after search
- Skip auto-exclusion if where clause explicitly queries isVFS

Architecture (Option 3C):
- VFS entities are first-class graph entities
- Marked with isVFS: true flag
- Separated via filtering, not storage
- Enables VFS-knowledge relationships

Moved internal docs to .strategy/:
- README_STORAGE_EXPLORATION.md
- EXPLORATION_SUMMARY.md
- STORAGE_FILES_REFERENCE.md
- STORAGE_ADAPTER_QUICK_REFERENCE.md
- SECURITY.md
This commit is contained in:
David Snelling 2025-10-24 11:42:47 -07:00
parent 86f5956d59
commit 014b8104da
14 changed files with 1172 additions and 1807 deletions

View file

@ -0,0 +1,86 @@
/**
* Debug: Check metadata index for isVFS field
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import * as fs from 'fs'
import * as path from 'path'
describe('Metadata Index Debug', () => {
const testDir = path.join(process.cwd(), 'test-metadata-index-debug')
let brain: Brainy
beforeAll(async () => {
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
fs.mkdirSync(testDir, { recursive: true })
brain = new Brainy({
storage: {
type: 'filesystem',
options: { path: testDir }
}
})
await brain.init()
})
afterAll(() => {
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
})
it('should index custom metadata fields', async () => {
// Add entity with custom metadata field
const entity1 = await brain.add({
data: 'Test entity with custom field',
type: NounType.Document,
metadata: {
customFlag: true,
customString: 'hello',
customNumber: 42
}
})
console.log('\n📋 Added entity:', entity1.id)
console.log(' Metadata:', entity1.metadata)
// Try to find by custom fields
console.log('\n📋 Query 1: where: { customFlag: true }')
const result1 = await brain.find({
where: { customFlag: true },
limit: 10
})
console.log(` Results: ${result1.length}`)
if (result1.length > 0) {
console.log(` Found: ${result1[0].id}`)
console.log(` Metadata.customFlag: ${result1[0].metadata?.customFlag}`)
}
console.log('\n📋 Query 2: where: { customString: "hello" }')
const result2 = await brain.find({
where: { customString: 'hello' },
limit: 10
})
console.log(` Results: ${result2.length}`)
console.log('\n📋 Query 3: where: { customNumber: 42 }')
const result3 = await brain.find({
where: { customNumber: 42 },
limit: 10
})
console.log(` Results: ${result3.length}`)
// Check what fields are indexed
console.log('\n📋 Indexed fields:')
const fields = await brain.getFilterFields()
console.log(` ${fields.join(', ')}`)
expect(result1.length).toBeGreaterThan(0)
expect(result2.length).toBeGreaterThan(0)
expect(result3.length).toBeGreaterThan(0)
})
})

View file

@ -0,0 +1,90 @@
/**
* Diagnostic: Check if isVFS flag is actually being stored
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import * as fs from 'fs'
import * as path from 'path'
describe('isVFS Flag Diagnostic', () => {
const testDir = path.join(process.cwd(), 'test-isvfs-diagnostic')
let brain: Brainy
beforeAll(async () => {
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
fs.mkdirSync(testDir, { recursive: true })
brain = new Brainy({
storage: {
type: 'filesystem',
options: { path: testDir }
}
})
await brain.init()
})
afterAll(() => {
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
})
it('should show what entities are created', async () => {
// Create VFS file
const vfs = brain.vfs()
await vfs.init()
await vfs.writeFile('/test.txt', 'Hello World')
// Get ALL entities
console.log('\n📋 All entities in database:')
const allEntities = await brain.find({ limit: 100, includeVFS: true })
console.log(` Total: ${allEntities.length}`)
for (const entity of allEntities) {
console.log(`\n Entity: ${entity.id}`)
console.log(` Type: ${entity.type}`)
console.log(` Metadata keys: ${Object.keys(entity.metadata || {}).join(', ')}`)
if (entity.metadata?.path) {
console.log(` Path: ${entity.metadata.path}`)
}
if (entity.metadata?.vfsType) {
console.log(` VfsType: ${entity.metadata.vfsType}`)
}
if (entity.metadata?.isVFS !== undefined) {
console.log(` isVFS: ${entity.metadata.isVFS}`)
}
if (entity.metadata?.name) {
console.log(` Name: ${entity.metadata.name}`)
}
}
// Try different queries
console.log('\n\n📋 Query 1: brain.find({ limit: 100 }) [default, no includeVFS]')
const query1 = await brain.find({ limit: 100 })
console.log(` Results: ${query1.length}`)
console.log('\n📋 Query 2: brain.find({ limit: 100, includeVFS: true })')
const query2 = await brain.find({ limit: 100, includeVFS: true })
console.log(` Results: ${query2.length}`)
console.log('\n📋 Query 3: brain.find({ where: { isVFS: true } })')
const query3 = await brain.find({ where: { isVFS: true }, limit: 100 })
console.log(` Results: ${query3.length}`)
console.log('\n📋 Query 4: brain.find({ where: { path: "/test.txt" } })')
const query4 = await brain.find({ where: { path: '/test.txt' }, limit: 100 })
console.log(` Results: ${query4.length}`)
console.log('\n📋 Query 5: brain.find({ type: NounType.Document })')
const query5 = await brain.find({ type: NounType.Document, limit: 100 })
console.log(` Results: ${query5.length}`)
console.log('\n📋 Query 6: brain.find({ type: NounType.Document, includeVFS: true })')
const query6 = await brain.find({ type: NounType.Document, includeVFS: true, limit: 100 })
console.log(` Results: ${query6.length}`)
})
})

View file

@ -0,0 +1,207 @@
/**
* VFS Multiple Init() Diagnostic Test
*
* Tests Workshop team's issue: Does calling vfs.init() multiple times
* create duplicate root entities?
*
* Scenario:
* - Create brain instance
* - Call vfs.init() multiple times (simulating multiple requests)
* - Check if multiple root entities are created
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as fs from 'fs'
import * as path from 'path'
describe('VFS Multiple Init Diagnostic', () => {
const testDir = path.join(process.cwd(), 'test-vfs-multi-init')
let brain: Brainy
beforeAll(async () => {
// Clean up test directory
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
fs.mkdirSync(testDir, { recursive: true })
// Create brain with filesystem storage
brain = new Brainy({
storage: {
type: 'filesystem',
path: testDir
}
})
await brain.init()
})
afterAll(() => {
// Clean up
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true })
}
})
it('should not create duplicate roots when calling vfs.init() multiple times on SAME instance', async () => {
const vfs = brain.vfs()
// Call init multiple times
await vfs.init()
await vfs.init()
await vfs.init()
// Query for root entities using workaround (where clause is broken)
const collections = await brain.find({
type: 'collection',
limit: 100
})
const roots = collections.filter(e =>
e.metadata?.path === '/' &&
e.metadata?.vfsType === 'directory'
)
console.log(`\n✅ Test 1: Same VFS instance`)
console.log(` Total collections: ${collections.length}`)
console.log(` Roots found: ${roots.length}`)
roots.forEach((root, i) => {
console.log(` Root ${i + 1}: ${root.id}`)
})
expect(roots.length).toBe(1)
})
it('should not create duplicate roots when creating MULTIPLE VFS instances (Workshop scenario)', async () => {
// Simulate Workshop's scenario: Getting VFS on multiple requests
// brain.vfs() returns cached instance, so this should be safe
const vfs1 = brain.vfs()
const vfs2 = brain.vfs()
const vfs3 = brain.vfs()
await vfs1.init()
await vfs2.init()
await vfs3.init()
// Query for root entities using workaround
const collections = await brain.find({
type: 'collection',
limit: 100
})
const roots = collections.filter(e =>
e.metadata?.path === '/' &&
e.metadata?.vfsType === 'directory'
)
console.log(`\n✅ Test 2: Multiple VFS references (cached)`)
console.log(` VFS instances are same? ${vfs1 === vfs2 && vfs2 === vfs3}`)
console.log(` Roots found: ${roots.length}`)
roots.forEach((root, i) => {
console.log(` Root ${i + 1}: ${root.id}`)
})
expect(vfs1).toBe(vfs2) // Should be same instance (cached)
expect(roots.length).toBe(1) // Should still be 1 root
})
it('should handle NEW brain instances gracefully (per-user scenario)', async () => {
// Simulate creating separate brain instances per user
// This is closer to Workshop's getUserBrainy() pattern
const brain2 = new Brainy({
storage: {
type: 'filesystem',
path: testDir // Same storage!
}
})
await brain2.init()
const vfs2 = brain2.vfs()
await vfs2.init()
// Query for roots using workaround
const collections = await brain.find({
type: 'collection',
limit: 100
})
const roots = collections.filter(e =>
e.metadata?.path === '/' &&
e.metadata?.vfsType === 'directory'
)
console.log(`\n✅ Test 3: New Brainy instance (same storage)`)
console.log(` Roots found: ${roots.length}`)
roots.forEach((root, i) => {
console.log(` Root ${i + 1}: ${root.id} (created: ${root.metadata?.createdAt})`)
})
// This is the CRITICAL test - should find existing root, not create new one
expect(roots.length).toBe(1)
})
it('should verify readdir works after multiple inits', async () => {
// Create a test directory
const vfs = brain.vfs()
await vfs.mkdir('/test-dir', { recursive: true })
await vfs.writeFile('/test-dir/test.txt', 'Hello')
// Call init again (simulating new request)
const brain3 = new Brainy({
storage: {
type: 'filesystem',
path: testDir
}
})
await brain3.init()
const vfs3 = brain3.vfs()
await vfs3.init()
// Try to read root directory
const entries = await vfs3.readdir('/', { withFileTypes: true })
console.log(`\n✅ Test 4: readdir after new brain instance`)
console.log(` Entries found: ${entries.length}`)
entries.forEach((entry: any) => {
console.log(` - ${entry.name} (${entry.type})`)
})
expect(entries.length).toBeGreaterThan(0)
})
it('should show if Contains relationships are created', async () => {
const vfs = brain.vfs()
// Get root entity ID using workaround
const collections = await brain.find({
type: 'collection',
limit: 100
})
const roots = collections.filter(e =>
e.metadata?.path === '/' &&
e.metadata?.vfsType === 'directory'
)
expect(roots.length).toBeGreaterThanOrEqual(1)
const rootId = roots[0].id
// Check for Contains relationships FROM root
const relations = await brain.getRelations({
from: rootId
})
console.log(`\n✅ Test 5: Contains relationships`)
console.log(` Root ID: ${rootId}`)
console.log(` Relationships from root: ${relations.length}`)
relations.forEach((rel) => {
console.log(` - ${rel.from} -> ${rel.to} (${rel.type})`)
})
// Root should have at least one Contains relationship (to /test-dir)
const containsRels = relations.filter(r => r.type === 'contains')
console.log(` Contains relationships: ${containsRels.length}`)
expect(containsRels.length).toBeGreaterThan(0)
})
})

View file

@ -0,0 +1,171 @@
/**
* Manual test to reproduce Workshop team's type filtering issue
*
* This test verifies that brain.find({ type: NounType.Person }) actually works
* as documented in the API.
*/
import { Brainy, NounType } from '../../src/index.js'
async function testTypeFiltering() {
console.log('\n🔬 Testing Type Filtering Issue from Workshop Team\n')
console.log('=' .repeat(60))
// Create in-memory instance for testing
const brain = new Brainy({
storage: { type: 'memory' }
})
await brain.init()
console.log('\n1⃣ Adding test entities with different types...\n')
// Add 5 person entities
const people = []
for (let i = 0; i < 5; i++) {
const id = await brain.add({
data: `Person ${i + 1}`,
type: NounType.Person,
metadata: { name: `Person ${i + 1}` }
})
people.push(id)
}
console.log(`✅ Added ${people.length} person entities`)
// Add 3 location entities
const locations = []
for (let i = 0; i < 3; i++) {
const id = await brain.add({
data: `Location ${i + 1}`,
type: NounType.Location,
metadata: { name: `Location ${i + 1}` }
})
locations.push(id)
}
console.log(`✅ Added ${locations.length} location entities`)
// Add 2 concept entities
const concepts = []
for (let i = 0; i < 2; i++) {
const id = await brain.add({
data: `Concept ${i + 1}`,
type: NounType.Concept,
metadata: { name: `Concept ${i + 1}` }
})
concepts.push(id)
}
console.log(`✅ Added ${concepts.length} concept entities`)
console.log(`\n📊 Total: ${people.length + locations.length + concepts.length} entities`)
console.log('\n' + '='.repeat(60))
console.log('\n2⃣ Testing find() without type filter...\n')
const allResults = await brain.find({ limit: 100 })
console.log(`✅ brain.find({}) returned ${allResults.length} entities`)
// Check types in results
const typeCounts = allResults.reduce((acc, r) => {
acc[r.type || 'unknown'] = (acc[r.type || 'unknown'] || 0) + 1
return acc
}, {} as Record<string, number>)
console.log('\nTypes in results:')
for (const [type, count] of Object.entries(typeCounts)) {
console.log(` - ${type}: ${count}`)
}
console.log('\n' + '='.repeat(60))
console.log('\n3⃣ Testing find() WITH type filter (using enum)...\n')
// Test 1: Filter by NounType.Person (enum value)
console.log('Test 1: brain.find({ type: NounType.Person })')
const personResults = await brain.find({ type: NounType.Person, limit: 100 })
console.log(` Result: ${personResults.length} entities`)
console.log(` Expected: ${people.length} entities`)
if (personResults.length === people.length) {
console.log(' ✅ PASS')
} else {
console.log(' ❌ FAIL - Type filtering not working!')
}
// Test 2: Filter by NounType.Location
console.log('\nTest 2: brain.find({ type: NounType.Location })')
const locationResults = await brain.find({ type: NounType.Location, limit: 100 })
console.log(` Result: ${locationResults.length} entities`)
console.log(` Expected: ${locations.length} entities`)
if (locationResults.length === locations.length) {
console.log(' ✅ PASS')
} else {
console.log(' ❌ FAIL - Type filtering not working!')
}
// Test 3: Filter by NounType.Concept
console.log('\nTest 3: brain.find({ type: NounType.Concept })')
const conceptResults = await brain.find({ type: NounType.Concept, limit: 100 })
console.log(` Result: ${conceptResults.length} entities`)
console.log(` Expected: ${concepts.length} entities`)
if (conceptResults.length === concepts.length) {
console.log(' ✅ PASS')
} else {
console.log(' ❌ FAIL - Type filtering not working!')
}
console.log('\n' + '='.repeat(60))
console.log('\n4⃣ Testing find() WITH type filter (using string)...\n')
// Test 4: Filter by string 'person'
console.log('Test 4: brain.find({ type: "person" })')
const personResults2 = await brain.find({ type: 'person' as any, limit: 100 })
console.log(` Result: ${personResults2.length} entities`)
console.log(` Expected: ${people.length} entities`)
if (personResults2.length === people.length) {
console.log(' ✅ PASS')
} else {
console.log(' ❌ FAIL - String type filtering not working!')
}
console.log('\n' + '='.repeat(60))
console.log('\n5⃣ Checking entity storage (metadata.noun)...\n')
// Get a person entity and check its storage
const personEntity = await brain.get(people[0])
console.log('Person entity structure:')
console.log(' - id:', personEntity?.id)
console.log(' - type:', personEntity?.type)
console.log(' - metadata.noun:', (personEntity as any)?.metadata?.noun)
console.log(' - metadata.name:', personEntity?.metadata?.name)
console.log('\n' + '='.repeat(60))
console.log('\n📋 Summary\n')
const tests = [
{ name: 'Filter by NounType.Person', passed: personResults.length === people.length },
{ name: 'Filter by NounType.Location', passed: locationResults.length === locations.length },
{ name: 'Filter by NounType.Concept', passed: conceptResults.length === concepts.length },
{ name: 'Filter by string "person"', passed: personResults2.length === people.length }
]
const passedCount = tests.filter(t => t.passed).length
const totalCount = tests.length
console.log(`Tests passed: ${passedCount}/${totalCount}\n`)
for (const test of tests) {
console.log(`${test.passed ? '✅' : '❌'} ${test.name}`)
}
if (passedCount === totalCount) {
console.log('\n🎉 All tests passed! Type filtering works correctly.')
console.log('\n💡 The Workshop team might be experiencing a different issue.')
console.log(' Check: storage persistence, Brainy instance reuse, or data migration')
} else {
console.log('\n❌ Type filtering is BROKEN in Brainy!')
console.log('\n🐛 This is a bug that needs to be fixed.')
console.log(' Workshop team was right - it\'s not user error.')
}
console.log('\n' + '='.repeat(60) + '\n')
}
// Run the test
testTypeFiltering().catch(console.error)