David Snelling
773c5171c3
Shutdown/close/flush now properly flushes all 4 components in parallel: metadataIndex, graphIndex, HNSW dirty nodes, and storage counts. Previously only counts were flushed, causing native provider data loss on restart. Also: - Wire roaring, msgpack, entityIdMapper provider consumption from plugins - Fix allOf filter O(n²) intersection → O(n) Set-based - Fix ne/exists negation filter to use Set-based exclusion - Add setMsgpackImplementation() swap in SSTable for native msgpack - Add setRoaringImplementation() swap for native CRoaring bitmaps - Add getAllIntIds() to EntityIdMapper for bitmap operations - Remove TypeAwareHNSWIndex from default index creation path - Export memory detection utilities from internals - Clean up 26 permanently-skipped dead tests
114 lines
3.5 KiB
TypeScript
114 lines
3.5 KiB
TypeScript
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { Brainy } from '../../../src/brainy.js'
|
|
import * as fs from 'fs'
|
|
import * as path from 'path'
|
|
import * as os from 'os'
|
|
|
|
/**
|
|
* Test for v5.1.2 fix: preserveSource now works with file paths
|
|
*
|
|
* Bug: sourceBuffer was undefined for file paths because normalizeSource
|
|
* returned type='path', but code checked for type='buffer'
|
|
*
|
|
* Fix: Changed to Buffer.isBuffer(normalizedSource.data)
|
|
*/
|
|
|
|
describe('Import preserveSource Fix (v5.1.2)', () => {
|
|
let brain: Brainy
|
|
let tempDir: string
|
|
let testFilePath: string
|
|
|
|
beforeEach(async () => {
|
|
brain = new Brainy({
|
|
storage: { type: 'memory' },
|
|
silent: true
|
|
})
|
|
await brain.init()
|
|
|
|
// Create temp directory and test file
|
|
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-test-'))
|
|
testFilePath = path.join(tempDir, 'test-data.csv')
|
|
|
|
// Create a test CSV (simple text content that won't fail parsing)
|
|
const csvContent = 'name,value\ntest1,100\ntest2,200'
|
|
fs.writeFileSync(testFilePath, csvContent)
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await brain.close()
|
|
|
|
// Cleanup temp files (recursive)
|
|
if (fs.existsSync(tempDir)) {
|
|
const files = fs.readdirSync(tempDir)
|
|
for (const file of files) {
|
|
fs.unlinkSync(path.join(tempDir, file))
|
|
}
|
|
fs.rmdirSync(tempDir)
|
|
}
|
|
})
|
|
|
|
it('should preserve source file when importing from file path with preserveSource: true', async () => {
|
|
// Import CSV from file path
|
|
await brain.import(testFilePath, {
|
|
format: 'csv',
|
|
vfsPath: '/imported',
|
|
preserveSource: true,
|
|
enableNeuralExtraction: false, // Disable to speed up test
|
|
createEntities: true
|
|
})
|
|
|
|
// Verify source file was preserved in VFS
|
|
const sourceFilePath = '/imported/_source.csv'
|
|
const exists = await brain.vfs.exists(sourceFilePath)
|
|
expect(exists).toBe(true)
|
|
|
|
// Verify we can read the file back
|
|
const content = await brain.vfs.readFile(sourceFilePath)
|
|
expect(Buffer.isBuffer(content)).toBe(true)
|
|
expect(content.length).toBeGreaterThan(0)
|
|
|
|
// Verify content matches original
|
|
const originalContent = fs.readFileSync(testFilePath)
|
|
expect(Buffer.compare(content, originalContent)).toBe(0)
|
|
})
|
|
|
|
it('should NOT preserve source file when preserveSource: false', async () => {
|
|
// Import CSV without preserving source
|
|
await brain.import(testFilePath, {
|
|
format: 'csv',
|
|
vfsPath: '/imported-no-source',
|
|
preserveSource: false,
|
|
enableNeuralExtraction: false,
|
|
createEntities: true
|
|
})
|
|
|
|
// Verify source file was NOT preserved
|
|
const sourceFilePath = '/imported-no-source/_source.csv'
|
|
const exists = await brain.vfs.exists(sourceFilePath)
|
|
expect(exists).toBe(false)
|
|
})
|
|
|
|
it('should handle binary files correctly (no corruption)', async () => {
|
|
// Create a simple JSON file to test without parsing errors
|
|
const jsonContent = JSON.stringify({ test: 'data', binary: [0xFF, 0xFE, 0xFD] })
|
|
const jsonFilePath = path.join(tempDir, 'test.json')
|
|
fs.writeFileSync(jsonFilePath, jsonContent)
|
|
|
|
// Import JSON
|
|
await brain.import(jsonFilePath, {
|
|
format: 'json',
|
|
vfsPath: '/json-test',
|
|
preserveSource: true,
|
|
enableNeuralExtraction: false,
|
|
createEntities: true
|
|
})
|
|
|
|
// Read back and verify no corruption
|
|
const content = await brain.vfs.readFile('/json-test/_source.json')
|
|
expect(content.toString('utf-8')).toBe(jsonContent)
|
|
|
|
// Cleanup
|
|
fs.unlinkSync(jsonFilePath)
|
|
})
|
|
|
|
})
|