open-brainy/tests/integration/smart-import.test.ts

/**
 * Unified Import Integration Tests
 *
 * Tests the unified import system (brain.import()):
 * 1. Auto-detect format from Excel buffer
 * 2. Extract entities/relationships
 * 3. Create knowledge graph
 * 4. Create VFS structure
 *
 * Uses real data, no mocks
 */

import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import * as XLSX from 'xlsx'

describe('Unified Import System', () => {
  let brain: Brainy

  beforeEach(async () => {
    brain = new Brainy({ requireSubtype: false,
      storage: { type: 'memory' as const }
    })
    await brain.init()
  })

  // Close every brain a test opens. Without this the background dedup timer and
  // the writer-lock heartbeat keep running after the test ends; under the
  // sequential integration runner that bleeds across tests (surfacing as stray
  // EntityNotFoundError rejections from a prior import's deferred VFS work) and
  // adds ~8s teardown stalls.
  afterEach(async () => {
    await brain.close()
  })

  it('should extract entities and relationships from Excel data', async () => {
    // Create test Excel file
    const testData = [
      {
        'Term': 'Westland',
        'Definition': 'Ancient kingdom in the west, ruled by the royal dynasty',
        'Type': 'Place',
        'Related Terms': 'Capital City, Northern Mountains'
      },
      {
        'Term': 'Capital City',
        'Definition': 'Main city of Westland, known for its grand library',
        'Type': 'Place',
        'Related Terms': 'Westland'
      },
      {
        'Term': 'Royal Dynasty',
        'Definition': 'Noble family that has ruled Westland for centuries',
        'Type': 'Organization',
        'Related Terms': 'Westland'
      },
      {
        'Term': 'Grand Library',
        'Definition': 'Massive repository of knowledge in Capital City',
        'Type': 'Place',
        'Related Terms': 'Capital City'
      },
      {
        'Term': 'Northern Mountains',
        'Definition': 'Mountain range north of Westland',
        'Type': 'Place',
        'Related Terms': 'Westland'
      }
    ]

    // Create Excel workbook
    const worksheet = XLSX.utils.json_to_sheet(testData)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    // Import with unified API
    const result = await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-imports/data',
      groupBy: 'type',
      enableNeuralExtraction: true,
      enableRelationshipInference: true,
      enableConceptExtraction: true
    })

    // Verify format detection
    expect(result.format).toBe('excel')
    expect(result.formatConfidence).toBeGreaterThan(0.9)

    // Should extract entities
    expect(result.stats.entitiesExtracted).toBeGreaterThanOrEqual(5)
    expect(result.entities.length).toBeGreaterThanOrEqual(5)

    // Should create VFS structure
    expect(result.vfs.rootPath).toBe('/test-imports/data')
    expect(result.vfs.directories.length).toBeGreaterThan(0)

    // Verify we can query the created entities
    const entities = await brain.find({
      query: 'Westland',
      limit: 5
    })
    expect(entities.length).toBeGreaterThan(0)
  }, 60000) // 60s timeout for neural processing

  it('should handle progress callbacks', async () => {
    const testData = [
      { 'Term': 'Test1', 'Definition': 'Test definition 1', 'Type': 'Concept' },
      { 'Term': 'Test2', 'Definition': 'Test definition 2', 'Type': 'Concept' }
    ]

    const worksheet = XLSX.utils.json_to_sheet(testData)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    const progressStages: string[] = []

    await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-progress',
      onProgress: (progress) => {
        progressStages.push(progress.stage)
      }
    })

    // Should receive progress updates for different stages
    expect(progressStages.length).toBeGreaterThan(0)
  }, 30000)

  it('should group entities by type in VFS', async () => {
    const testData = [
      { 'Term': 'Alice', 'Definition': 'A person', 'Type': 'Person' },
      { 'Term': 'New York', 'Definition': 'A city', 'Type': 'Place' },
      { 'Term': 'Gravity', 'Definition': 'A concept', 'Type': 'Concept' }
    ]

    const worksheet = XLSX.utils.json_to_sheet(testData)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    const result = await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-types',
      groupBy: 'type'
    })

    // Verify VFS structure created
    expect(result.vfs.rootPath).toBe('/test-types')
    expect(result.vfs.directories.length).toBeGreaterThan(0)

    // Verify entities were extracted
    expect(result.entities.length).toBeGreaterThanOrEqual(3)
  }, 30000)

  it('should extract relationships from natural language definitions', async () => {
    const testData = [
      {
        'Term': 'Paris',
        'Definition': 'Capital city of France, located on the Seine river',
        'Type': 'Place'
      },
      {
        'Term': 'France',
        'Definition': 'A country in Western Europe',
        'Type': 'Place'
      }
    ]

    const worksheet = XLSX.utils.json_to_sheet(testData)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    const result = await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-relationships',
      enableRelationshipInference: true
    })

    // Verify entities extracted
    expect(result.entities.length).toBeGreaterThanOrEqual(2)

    // Find the Paris entity
    const parisResults = await brain.find({
      query: 'Paris',
      limit: 5
    })
    expect(parisResults.length).toBeGreaterThan(0)
  }, 30000)

  // v4.2.0: YAML import tests
  it('should import YAML files and extract entities', async () => {
    const yamlContent = `
---
name: John Doe
role: Software Engineer
company:
  name: Tech Corp
  location: San Francisco
skills:
  - TypeScript
  - Node.js
  - GraphQL
projects:
  - name: Brainy
    description: Knowledge graph database
    status: active
`

    const result = await brain.import(yamlContent, {
      format: 'yaml',
      vfsPath: '/test-yaml',
      enableNeuralExtraction: true,
      enableHierarchicalRelationships: true
    })

    // Verify format detection
    expect(result.format).toBe('yaml')

    // Should extract entities
    expect(result.stats.entitiesExtracted).toBeGreaterThan(0)
    expect(result.entities.length).toBeGreaterThan(0)

    // Should create VFS structure
    expect(result.vfs.rootPath).toBe('/test-yaml')
    expect(result.vfs.directories.length).toBeGreaterThan(0)
  }, 30000)

  // v4.2.0: Confidence and weight fields
  it('should populate confidence and weight fields on entities and relationships', async () => {
    const testData = [
      {
        'Term': 'Einstein',
        'Definition': 'German physicist who developed the theory of relativity',
        'Type': 'Person'
      }
    ]

    const worksheet = XLSX.utils.json_to_sheet(testData)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    const result = await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-confidence',
      enableNeuralExtraction: true,
      enableRelationshipInference: true,
      confidenceThreshold: 0.6
    })

    // Verify entities were created with confidence threshold applied
    expect(result.entities.length).toBeGreaterThan(0)
    expect(result.stats.entitiesExtracted).toBeGreaterThan(0)

    // Query the entity to verify it was stored successfully
    const entities = await brain.find({
      query: 'Einstein',
      limit: 1
    })

    expect(entities.length).toBeGreaterThan(0)
  }, 30000)

  // v4.2.0: Range queries with confidence
  it('should support range queries on confidence field', async () => {
    const testData = [
      { 'Term': 'Test1', 'Definition': 'First test entity', 'Type': 'Concept' },
      { 'Term': 'Test2', 'Definition': 'Second test entity', 'Type': 'Concept' },
      { 'Term': 'Test3', 'Definition': 'Third test entity', 'Type': 'Concept' }
    ]

    const worksheet = XLSX.utils.json_to_sheet(testData)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-range-query',
      enableNeuralExtraction: true,
      confidenceThreshold: 0.6
    })

    // Query with confidence range filter
    const highConfidenceEntities = await brain.find({
      where: {
        confidence: { gte: 0.8 }
      },
      limit: 10
    })

    // Should return some entities
    expect(highConfidenceEntities).toBeDefined()
    expect(Array.isArray(highConfidenceEntities)).toBe(true)
  }, 30000)

  // v4.2.0: Per-sheet VFS extraction
  it('should extract Excel sheets to separate VFS directories', async () => {
    const testData1 = [
      { 'Term': 'Sheet1Entity', 'Definition': 'Entity from sheet 1', 'Type': 'Concept' }
    ]
    const testData2 = [
      { 'Term': 'Sheet2Entity', 'Definition': 'Entity from sheet 2', 'Type': 'Concept' }
    ]

    const worksheet1 = XLSX.utils.json_to_sheet(testData1)
    const worksheet2 = XLSX.utils.json_to_sheet(testData2)
    const workbook = XLSX.utils.book_new()
    XLSX.utils.book_append_sheet(workbook, worksheet1, 'FirstSheet')
    XLSX.utils.book_append_sheet(workbook, worksheet2, 'SecondSheet')
    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

    const result = await brain.import(buffer, {
      format: 'excel',
      vfsPath: '/test-sheets',
      groupBy: 'sheet',
      enableNeuralExtraction: true
    })

    // Verify multiple directories created (one per sheet)
    expect(result.vfs.rootPath).toBe('/test-sheets')
    expect(result.vfs.directories.length).toBeGreaterThan(1)

    // Should extract entities from both sheets
    expect(result.entities.length).toBeGreaterThanOrEqual(2)
  }, 30000)

  // v4.2.0: SmartRelationshipExtractor Integration Tests
  describe('SmartRelationshipExtractor', () => {
    // TIER2: semantic relevance (real-model suite) — SmartRelationshipExtractor's
    // ensemble is dominated by the verb-embedding signal (55% weight). Asserting a
    // specific inferred VerbType (Creates vs the generic RelatedTo fallback) needs
    // real cross-entity embedding similarity, which the deterministic embedder
    // cannot provide. The same applies to locating the subject entity by a plain
    // `find({ query: 'Mona Lisa' })` (cross-text relevance, not self-retrieval).
    it.skip('should classify CreatedBy relationships using exact keywords', async () => {
      const testData = [
        {
          'Term': 'Mona Lisa',
          'Definition': 'Famous painting created by Leonardo da Vinci in the 16th century',
          'Type': 'Product'
        },
        {
          'Term': 'Leonardo da Vinci',
          'Definition': 'Italian Renaissance artist and inventor',
          'Type': 'Person'
        }
      ]

      const worksheet = XLSX.utils.json_to_sheet(testData)
      const workbook = XLSX.utils.book_new()
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-created-by',
        enableNeuralExtraction: true,
        enableRelationshipInference: true
      })

      // Verify relationships were extracted
      expect(result.stats.relationshipsInferred).toBeGreaterThan(0)

      // Find "Mona Lisa" entity
      const monaLisaEntities = await brain.find({
        query: 'Mona Lisa',
        limit: 1
      })
      expect(monaLisaEntities.length).toBeGreaterThan(0)

      // Get relationships for Mona Lisa
      const relationships = await brain.related(monaLisaEntities[0].id)
      expect(relationships.length).toBeGreaterThan(0)

      // Should have CreatedBy relationship (not just RelatedTo)
      const createdByRels = relationships.filter(r =>
        r.type === 'CreatedBy' && r.targetName?.toLowerCase().includes('leonardo')
      )
      expect(createdByRels.length).toBeGreaterThan(0)
    }, 60000)

    // TIER2: semantic relevance (real-model suite) — see note above. The ensemble
    // returns the generic RelatedTo fallback here under the deterministic embedder
    // instead of LocatedAt, and the subject is located by cross-text relevance.
    it.skip('should classify LocatedAt relationships using pattern matching', async () => {
      const testData = [
        {
          'Term': 'Stanford University',
          'Definition': 'Private research university located in Stanford, California',
          'Type': 'Organization'
        },
        {
          'Term': 'California',
          'Definition': 'State on the west coast of the United States',
          'Type': 'Place'
        }
      ]

      const worksheet = XLSX.utils.json_to_sheet(testData)
      const workbook = XLSX.utils.book_new()
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-located-at',
        enableNeuralExtraction: true,
        enableRelationshipInference: true
      })

      // Verify relationships were extracted
      expect(result.stats.relationshipsInferred).toBeGreaterThan(0)

      // Find Stanford entity
      const stanfordEntities = await brain.find({
        query: 'Stanford',
        limit: 1
      })
      expect(stanfordEntities.length).toBeGreaterThan(0)

      // Get relationships
      const relationships = await brain.related(stanfordEntities[0].id)
      expect(relationships.length).toBeGreaterThan(0)

      // Should have LocatedAt relationship
      const locatedAtRels = relationships.filter(r =>
        r.type === 'LocatedAt' && r.targetName?.toLowerCase().includes('california')
      )
      expect(locatedAtRels.length).toBeGreaterThan(0)
    }, 60000)

    // TIER2: semantic relevance (real-model suite) — see note above. Inferring
    // PartOf vs the generic fallback needs real verb-embedding similarity.
    it.skip('should classify PartOf relationships using hierarchical context', async () => {
      const testData = [
        {
          'Term': 'Heart',
          'Definition': 'Muscular organ that is part of the circulatory system, pumps blood',
          'Type': 'Thing'
        },
        {
          'Term': 'Circulatory System',
          'Definition': 'Organ system that transports nutrients and oxygen throughout the body',
          'Type': 'Thing'
        }
      ]

      const worksheet = XLSX.utils.json_to_sheet(testData)
      const workbook = XLSX.utils.book_new()
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-part-of',
        enableNeuralExtraction: true,
        enableRelationshipInference: true
      })

      // Verify relationships were extracted
      expect(result.stats.relationshipsInferred).toBeGreaterThan(0)

      // Find Heart entity
      const heartEntities = await brain.find({
        query: 'Heart',
        limit: 1
      })
      expect(heartEntities.length).toBeGreaterThan(0)

      // Get relationships
      const relationships = await brain.related(heartEntities[0].id)
      expect(relationships.length).toBeGreaterThan(0)

      // Should have PartOf relationship
      const partOfRels = relationships.filter(r =>
        r.type === 'PartOf' && r.targetName?.toLowerCase().includes('circulatory')
      )
      expect(partOfRels.length).toBeGreaterThan(0)
    }, 60000)

    // TIER2: semantic relevance (real-model suite) — see note above. Inferring
    // WorksWith/MemberOf vs the generic fallback needs real verb-embedding similarity.
    it.skip('should classify WorksWith relationships using type hints', async () => {
      const testData = [
        {
          'Term': 'Alice Johnson',
          'Definition': 'Senior engineer who works at Google on cloud infrastructure',
          'Type': 'Person'
        },
        {
          'Term': 'Google',
          'Definition': 'Technology company specializing in internet services',
          'Type': 'Organization'
        }
      ]

      const worksheet = XLSX.utils.json_to_sheet(testData)
      const workbook = XLSX.utils.book_new()
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-works-with',
        enableNeuralExtraction: true,
        enableRelationshipInference: true
      })

      // Verify relationships were extracted
      expect(result.stats.relationshipsInferred).toBeGreaterThan(0)

      // Find Alice entity
      const aliceEntities = await brain.find({
        query: 'Alice Johnson',
        limit: 1
      })
      expect(aliceEntities.length).toBeGreaterThan(0)

      // Get relationships
      const relationships = await brain.related(aliceEntities[0].id)
      expect(relationships.length).toBeGreaterThan(0)

      // Should have WorksWith or MemberOf relationship (not just RelatedTo)
      const workRels = relationships.filter(r =>
        (r.type === 'WorksWith' || r.type === 'MemberOf') &&
        r.targetName?.toLowerCase().includes('google')
      )
      expect(workRels.length).toBeGreaterThan(0)
    }, 60000)

    // TIER2: semantic relevance (real-model suite) — see note above. Ensemble
    // agreement on CreatedBy relies on the embedding signal, which is meaningless
    // under the deterministic embedder.
    it.skip('should use ensemble voting when multiple signals agree', async () => {
      const testData = [
        {
          'Term': 'iPhone',
          'Definition': 'Smartphone created by Apple Inc. in 2007, revolutionized mobile computing',
          'Type': 'Product'
        },
        {
          'Term': 'Apple Inc.',
          'Definition': 'Technology company that designs consumer electronics',
          'Type': 'Organization'
        }
      ]

      const worksheet = XLSX.utils.json_to_sheet(testData)
      const workbook = XLSX.utils.book_new()
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Terms')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-ensemble',
        enableNeuralExtraction: true,
        enableRelationshipInference: true
      })

      // Verify relationships extracted
      expect(result.stats.relationshipsInferred).toBeGreaterThan(0)

      // Find iPhone entity
      const iphoneEntities = await brain.find({
        query: 'iPhone',
        limit: 1
      })
      expect(iphoneEntities.length).toBeGreaterThan(0)

      // Get relationships
      const relationships = await brain.related(iphoneEntities[0].id)
      expect(relationships.length).toBeGreaterThan(0)

      // Should have CreatedBy relationship with high confidence
      // (exact match "created by" + embedding similarity + pattern match + type hint all agree)
      const createdByRels = relationships.filter(r =>
        r.type === 'CreatedBy' && r.targetName?.toLowerCase().includes('apple')
      )
      expect(createdByRels.length).toBeGreaterThan(0)
    }, 60000)

    // TIER2: semantic relevance (real-model suite) — see note above. Asserting the
    // Contains/PartOf hierarchical verb type and locating the project entity by a
    // cross-text `find({ query: 'Brainy' })` both need the real embedding model.
    it.skip('should extract relationships from YAML hierarchical structures', async () => {
      const yamlContent = `
---
project:
  name: Brainy
  description: Knowledge graph database that contains multiple modules
  modules:
    - name: Storage
      description: Handles data persistence
    - name: Query
      description: Processes natural language queries
    - name: VFS
      description: Virtual file system
`

      const result = await brain.import(yamlContent, {
        format: 'yaml',
        vfsPath: '/test-yaml-relationships',
        enableNeuralExtraction: true,
        enableHierarchicalRelationships: true
      })

      // Verify entities and relationships extracted
      expect(result.stats.entitiesExtracted).toBeGreaterThan(0)
      expect(result.stats.relationshipsInferred).toBeGreaterThan(0)

      // Find project entity
      const projectEntities = await brain.find({
        query: 'Brainy',
        limit: 1
      })
      expect(projectEntities.length).toBeGreaterThan(0)

      // Get relationships
      const relationships = await brain.related(projectEntities[0].id)
      expect(relationships.length).toBeGreaterThan(0)

      // Should have Contains relationship (hierarchical)
      const containsRels = relationships.filter(r =>
        r.type === 'Contains' || r.type === 'PartOf'
      )
      expect(containsRels.length).toBeGreaterThan(0)
    }, 60000)
  })

  // REAL LIBRARY BUG (left failing on purpose — not test rot, not embedder-related).
  // Three of the four tests below fail because brain.import() does NOT honor its own
  // documented "always-on streaming" contract for imports larger than 100 entities.
  // ImportCoordinator auto-disables deduplication at DEDUPLICATION_AUTO_DISABLE_THRESHOLD
  // = 100 and routes those imports to a batch "fast path" (ImportCoordinator.ts ~L993)
  // that calls addMany() and NEVER emits a progress event with `queryable: true`.
  // The progressive per-flush `queryable` callback exists only on the ≤100-entity slow
  // path (ImportCoordinator.ts ~L1293/L1323). This contradicts both the ImportProgress
  // JSDoc ("progress.queryable is true after each flush") and the public docs
  // (docs/guides/streaming-imports.md: "All imports stream by default ... flush every
  // 100 entities for <1K imports"). Reproduced deterministically with the real embedder
  // OFF (enableNeuralExtraction: false) and in a plain non-vitest process, so it is a
  // genuine import behavioral regression, not a deterministic-embedder artifact. These
  // assertions are correct and must stay; the fix belongs in src/import/ImportCoordinator.ts.
  describe('Always-On Streaming Imports (v4.2.0+)', () => {
    it('should always stream with adaptive flush intervals', async () => {
      // Create file with 250 entities (adaptive interval: flush every 100)
      const testData = Array.from({ length: 250 }, (_, i) => ({
        'Term': `Entity_${i}`,
        'Definition': `Test entity number ${i} for streaming import`,
        'Type': 'Thing'
      }))

      const workbook = XLSX.utils.book_new()
      const worksheet = XLSX.utils.json_to_sheet(testData)
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Test')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      let queryableCount = 0

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-streaming',
        enableNeuralExtraction: false,
        enableRelationshipInference: false,
        onProgress: (progress) => {
          if (progress.queryable === true) {
            queryableCount++
          }
        }
      })

      // Verify streaming happened (should flush 2-3 times for 250 entities)
      expect(queryableCount).toBeGreaterThan(0)

      // Verify all entities were imported
      expect(result.entities.length).toBe(250)
    }, 60000)

    it('should allow live queries during import', async () => {
      // Create file with 250 entities
      const testData = Array.from({ length: 250 }, (_, i) => ({
        'Term': `Product_${i}`,
        'Definition': `Product ${i} description`,
        'Type': 'product'
      }))

      const workbook = XLSX.utils.book_new()
      const worksheet = XLSX.utils.json_to_sheet(testData)
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Products')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const queryCounts: number[] = []

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-queryable',
        enableNeuralExtraction: true,  // Enable for proper type classification
        enableRelationshipInference: false,
        onProgress: async (progress) => {
          // Query data during import
          if (progress.queryable) {
            const products = await brain.find({ type: 'product', limit: 1000 })
            queryCounts.push(products.length)
          }
        }
      })

      // Should have queried at least once (flushes every 100 entities)
      expect(queryCounts.length).toBeGreaterThan(0)

      // Counts should be increasing
      for (let i = 1; i < queryCounts.length; i++) {
        expect(queryCounts[i]).toBeGreaterThanOrEqual(queryCounts[i - 1])
      }

      // Final count should match total
      const finalProducts = await brain.find({ type: 'product', limit: 1000 })
      expect(finalProducts.length).toBe(250)
    }, 60000)

    it('should maintain data integrity with streaming flushes', async () => {
      // Create file with mix of entity types (300 total)
      const testData = [
        ...Array.from({ length: 100 }, (_, i) => ({
          'Term': `Person_${i}`,
          'Definition': `Person ${i}`,
          'Type': 'person'
        })),
        ...Array.from({ length: 100 }, (_, i) => ({
          'Term': `Product_${i}`,
          'Definition': `Product ${i}`,
          'Type': 'product'
        })),
        ...Array.from({ length: 100 }, (_, i) => ({
          'Term': `Place_${i}`,
          'Definition': `Place ${i}`,
          'Type': 'location'
        }))
      ]

      const workbook = XLSX.utils.book_new()
      const worksheet = XLSX.utils.json_to_sheet(testData)
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Mixed')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      const result = await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-integrity',
        enableNeuralExtraction: true,  // Enable for proper type classification
        enableRelationshipInference: false
      })

      // Verify all entities imported
      expect(result.entities.length).toBe(300)

      // Verify type distribution
      const persons = await brain.find({ type: 'person', limit: 1000 })
      const products = await brain.find({ type: 'product', limit: 1000 })
      const places = await brain.find({ type: 'location', limit: 1000 })

      expect(persons.length).toBe(100)
      expect(products.length).toBe(100)
      expect(places.length).toBe(100)

      // Verify data integrity (sum of all types should equal total imported)
      expect(persons.length + products.length + places.length).toBe(300)
    }, 60000)

    it('should use adaptive intervals based on import size', async () => {
      // Test small import (< 1000): should flush every 100 entities
      const smallData = Array.from({ length: 250 }, (_, i) => ({
        'Term': `Small_${i}`,
        'Definition': `Small ${i}`,
        'Type': 'Thing'
      }))

      const workbook = XLSX.utils.book_new()
      const worksheet = XLSX.utils.json_to_sheet(smallData)
      XLSX.utils.book_append_sheet(workbook, worksheet, 'Small')
      const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })

      let flushCount = 0

      await brain.import(buffer, {
        format: 'excel',
        vfsPath: '/test-adaptive',
        enableNeuralExtraction: false,
        enableRelationshipInference: false,
        onProgress: (progress) => {
          if (progress.queryable) {
            flushCount++
          }
        }
      })

      // For 250 entities with flush every 100: expect 2-3 flushes
      expect(flushCount).toBeGreaterThanOrEqual(2)
      expect(flushCount).toBeLessThanOrEqual(3)
    }, 60000)
  })
})