open-brainy/tests/integration/vfs-knowledge-separation.test.ts

/**
 * VFS-Knowledge Separation Test (8.0)
 *
 * Exercises how VFS infrastructure entities coexist with knowledge-graph
 * entities in the same brain, and how a query opts in/out of the VFS layer:
 *
 * - VFS files/directories are real entities, marked in metadata with
 *   `isVFS: true` / `isVFSEntity: true` and `vfsType: 'file' | 'directory'`.
 * - `brain.find()` INCLUDES VFS entities by default (8.0 semantics).
 * - `brain.find({ excludeVFS: true })` drops the VFS infrastructure layer,
 *   returning only knowledge entities — works on both the metadata-filter
 *   path and the vector/`query` path.
 * - `where: { vfsType: 'file' }` selects VFS file entities explicitly.
 * - Relationships can span VFS files and knowledge entities.
 *
 * Runs under the deterministic embedder (tests/setup-integration.ts), so
 * cross-text semantic ranking is not meaningful; self-retrieval (querying an
 * entity by its own text) and `excludeVFS` filtering are.
 */

import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NounType, VerbType } from '../../src/types/graphTypes.js'
import * as fs from 'fs'
import * as path from 'path'

describe('VFS-Knowledge Separation (8.0)', () => {
  const testDir = path.join(process.cwd(), 'test-vfs-knowledge-separation')
  let brain: Brainy

  beforeAll(async () => {
    // Clean up
    if (fs.existsSync(testDir)) {
      fs.rmSync(testDir, { recursive: true, force: true })
    }
    fs.mkdirSync(testDir, { recursive: true })

    brain = new Brainy({
      requireSubtype: false,
      storage: {
        type: 'filesystem',
        options: { path: testDir }
      }
    })
    await brain.init()

    // Seed the VFS layer + one knowledge entity used across the suite.
    const vfs = brain.vfs
    await vfs.init()
    await vfs.mkdir('/docs', { recursive: true })
    await vfs.writeFile('/docs/readme.md', '# Hello World')

    await brain.add({
      data: 'This is a knowledge document about AI',
      type: NounType.Document,
      metadata: {
        title: 'AI Research Paper',
        category: 'research'
      }
    })
  })

  afterAll(async () => {
    // Close releases the writer lock + flushes; prevents background-flush bleed.
    await brain.close()
    if (fs.existsSync(testDir)) {
      fs.rmSync(testDir, { recursive: true, force: true })
    }
  })

  it('includes VFS entities in brain.find() by default', async () => {
    // 8.0: VFS infrastructure entities are returned by default. The /docs/readme.md
    // file is a Document-typed entity carrying isVFS/vfsType markers, so a plain
    // type query surfaces BOTH it and the knowledge document.
    const results = await brain.find({
      type: NounType.Document,
      limit: 100
    })

    const vfsResults = results.filter((r) => r.metadata?.isVFS === true)
    const knowledgeResults = results.filter((r) => r.metadata?.isVFS !== true)

    // Default find() shows the VFS file alongside knowledge.
    expect(vfsResults.length).toBeGreaterThan(0)
    expect(knowledgeResults.length).toBeGreaterThan(0)
    expect(results.some((r) => r.metadata?.title === 'AI Research Paper')).toBe(true)
    expect(results.some((r) => r.metadata?.vfsType === 'file')).toBe(true)
  })

  it('excludes VFS entities when excludeVFS: true', async () => {
    // 8.0: excludeVFS drops the VFS infrastructure layer. Only knowledge entities remain.
    const results = await brain.find({
      type: NounType.Document,
      excludeVFS: true,
      limit: 100
    })

    const vfsResults = results.filter((r) => r.metadata?.isVFS === true)
    const knowledgeResults = results.filter((r) => r.metadata?.isVFS !== true)

    expect(vfsResults.length).toBe(0)
    expect(knowledgeResults.length).toBeGreaterThan(0)
    expect(results.some((r) => r.metadata?.title === 'AI Research Paper')).toBe(true)
  })

  it('should allow relationships between VFS files and knowledge entities', async () => {
    // Get VFS file entity. VFS files are identified by vfsType (the indexed,
    // queryable marker); they are included in find() by default.
    const vfsFile = await brain.find({
      where: {
        path: '/docs/readme.md'
      },
      limit: 1
    })
    expect(vfsFile.length).toBe(1)
    expect(vfsFile[0].metadata?.vfsType).toBe('file')

    // Get knowledge entity
    const knowledgeEntity = await brain.find({
      type: NounType.Document,
      where: {
        title: 'AI Research Paper'
      },
      excludeVFS: true,
      limit: 1
    })
    expect(knowledgeEntity.length).toBe(1)

    // Create relationship: knowledge entity -> references -> VFS file.
    // relate() returns the new relation's id (string).
    const relationId = await brain.relate({
      from: knowledgeEntity[0].id,
      to: vfsFile[0].id,
      type: VerbType.References
    })
    expect(typeof relationId).toBe('string')
    expect(relationId.length).toBeGreaterThan(0)

    // Verify relationship exists
    const relations = await brain.related({
      from: knowledgeEntity[0].id,
      to: vfsFile[0].id
    })

    expect(relations.length).toBe(1)
    expect(relations[0].type).toBe(VerbType.References)
  })

  it('should filter VFS entities using where clause', async () => {
    // Query for VFS files explicitly via the vfsType marker (indexed + queryable).
    const vfsFiles = await brain.find({
      where: {
        vfsType: 'file'
      },
      limit: 100
    })

    expect(vfsFiles.length).toBeGreaterThan(0)
    expect(vfsFiles.every((f) => f.metadata?.vfsType === 'file')).toBe(true)
    // Every vfsType:'file' entity is a VFS infrastructure entity.
    expect(vfsFiles.every((f) => f.metadata?.isVFS === true)).toBe(true)

    // Query for non-VFS knowledge entities explicitly via a user metadata field.
    const nonVFS = await brain.find({
      where: {
        category: 'research'
      },
      limit: 100
    })

    expect(nonVFS.length).toBeGreaterThan(0)
    expect(nonVFS.every((e) => e.metadata?.isVFS !== true)).toBe(true)
    expect(nonVFS.every((e) => e.metadata?.vfsType === undefined)).toBe(true)
  })

  it('should handle semantic search with VFS filtering', async () => {
    // Self-retrieval: querying with the knowledge doc's own text returns it
    // (deterministic embedder ⇒ cosine 1.0). With excludeVFS the VFS layer is dropped.
    const knowledgeOnly = await brain.find({
      query: 'This is a knowledge document about AI',
      excludeVFS: true,
      limit: 10
    })
    expect(knowledgeOnly.length).toBeGreaterThan(0)
    expect(knowledgeOnly.some((r) => r.metadata?.isVFS === true)).toBe(false)
    expect(knowledgeOnly.some((r) => r.metadata?.title === 'AI Research Paper')).toBe(true)

    // Default (no excludeVFS): self-retrieval of the VFS file by its own content
    // returns it — VFS entities participate in vector search by default.
    const withVFS = await brain.find({
      query: '# Hello World',
      limit: 10
    })
    expect(withVFS.some((r) => r.metadata?.path === '/docs/readme.md')).toBe(true)

    // excludeVFS on the vector path removes the VFS file even when its content matches.
    const withVFSExcluded = await brain.find({
      query: '# Hello World',
      excludeVFS: true,
      limit: 10
    })
    expect(withVFSExcluded.some((r) => r.metadata?.isVFS === true)).toBe(false)
  })
})