fix: Resolve soft delete and add encrypted config support
- Fix soft delete functionality by filtering out deleted items in search results - Add support for encrypted configuration storage and retrieval - Copy test suite and vitest config to ensure all functionality works - Maintain all core brainy functionality during repository cleanup
This commit is contained in:
parent
7432fab3bc
commit
3d80df1726
62 changed files with 13792 additions and 6 deletions
255
tests/pagination.test.ts
Normal file
255
tests/pagination.test.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* Tests for offset-based pagination in search results
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { BrainyData } from '../src/brainyData.js'
|
||||
import { cleanupWorkerPools } from '../src/utils/index.js'
|
||||
|
||||
describe('Pagination with Offset', () => {
|
||||
let db: BrainyData
|
||||
|
||||
beforeEach(async () => {
|
||||
// Initialize BrainyData with in-memory storage for testing
|
||||
db = new BrainyData({
|
||||
storage: {
|
||||
forceMemoryStorage: true
|
||||
},
|
||||
logging: {
|
||||
verbose: false
|
||||
}
|
||||
})
|
||||
await db.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await db.clear()
|
||||
await cleanupWorkerPools()
|
||||
})
|
||||
|
||||
describe('Basic Offset Pagination', () => {
|
||||
it('should return correct results with offset=0', async () => {
|
||||
// Add test data
|
||||
const testData = []
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const item = {
|
||||
id: `item-${i}`,
|
||||
text: `test document ${i}`,
|
||||
value: i
|
||||
}
|
||||
testData.push(item)
|
||||
await db.add(item)
|
||||
}
|
||||
|
||||
// Search without offset (default offset=0)
|
||||
const results = await db.search('test document', 5)
|
||||
expect(results.length).toBe(5)
|
||||
|
||||
// Results should be the top 5 most similar
|
||||
const resultIds = results.map(r => r.metadata.id)
|
||||
expect(resultIds.length).toBe(5)
|
||||
})
|
||||
|
||||
it('should skip results with offset > 0', async () => {
|
||||
// Add test data
|
||||
const testData = []
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const item = {
|
||||
id: `item-${i}`,
|
||||
text: `test document ${i}`,
|
||||
value: i
|
||||
}
|
||||
testData.push(item)
|
||||
await db.add(item)
|
||||
}
|
||||
|
||||
// Get first page (no offset)
|
||||
const firstPage = await db.search('test document', 5)
|
||||
expect(firstPage.length).toBe(5)
|
||||
const firstPageIds = firstPage.map(r => r.metadata.id)
|
||||
|
||||
// Get second page (offset=5)
|
||||
const secondPage = await db.search('test document', 5, { offset: 5 })
|
||||
expect(secondPage.length).toBe(5)
|
||||
const secondPageIds = secondPage.map(r => r.metadata.id)
|
||||
|
||||
// Ensure no overlap between pages
|
||||
const overlap = firstPageIds.filter(id => secondPageIds.includes(id))
|
||||
expect(overlap.length).toBe(0)
|
||||
})
|
||||
|
||||
it('should handle offset beyond available results', async () => {
|
||||
// Add limited test data
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await db.add({
|
||||
id: `item-${i}`,
|
||||
text: `test document ${i}`
|
||||
})
|
||||
}
|
||||
|
||||
// Search with offset beyond available results
|
||||
const results = await db.search('test document', 5, { offset: 15 })
|
||||
expect(results.length).toBe(0)
|
||||
})
|
||||
|
||||
it('should return partial results when offset + k exceeds total', async () => {
|
||||
// Add limited test data
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await db.add({
|
||||
id: `item-${i}`,
|
||||
text: `test document ${i}`
|
||||
})
|
||||
}
|
||||
|
||||
// Search with offset that allows only partial results
|
||||
const results = await db.search('test document', 5, { offset: 7 })
|
||||
expect(results.length).toBe(3) // Only 3 results available after offset 7
|
||||
})
|
||||
})
|
||||
|
||||
describe('Pagination with Filters', () => {
|
||||
it.skip('should paginate with noun type filters', async () => {
|
||||
// TODO: This test requires proper noun type support in the add method
|
||||
// Currently skipped as noun types are not directly supported in the add method
|
||||
// Add test data with different noun types
|
||||
for (let i = 0; i < 15; i++) {
|
||||
await db.add({
|
||||
id: `doc-${i}`,
|
||||
text: `document ${i}`,
|
||||
type: 'document'
|
||||
}, undefined, 'document')
|
||||
}
|
||||
|
||||
for (let i = 0; i < 15; i++) {
|
||||
await db.add({
|
||||
id: `note-${i}`,
|
||||
text: `note ${i}`,
|
||||
type: 'note'
|
||||
}, undefined, 'note')
|
||||
}
|
||||
|
||||
// Get first page of documents
|
||||
const firstPage = await db.search('document', 5, {
|
||||
nounTypes: ['document']
|
||||
})
|
||||
expect(firstPage.length).toBe(5)
|
||||
expect(firstPage.every(r => r.metadata.type === 'document')).toBe(true)
|
||||
|
||||
// Get second page of documents
|
||||
const secondPage = await db.search('document', 5, {
|
||||
nounTypes: ['document'],
|
||||
offset: 5
|
||||
})
|
||||
expect(secondPage.length).toBe(5)
|
||||
expect(secondPage.every(r => r.metadata.type === 'document')).toBe(true)
|
||||
|
||||
// Ensure pages are different
|
||||
const firstIds = firstPage.map(r => r.metadata.id)
|
||||
const secondIds = secondPage.map(r => r.metadata.id)
|
||||
const overlap = firstIds.filter(id => secondIds.includes(id))
|
||||
expect(overlap.length).toBe(0)
|
||||
})
|
||||
|
||||
it('should paginate with service filters', async () => {
|
||||
// Add test data with different services
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const metadata = {
|
||||
id: `item-${i}`,
|
||||
text: `test item ${i}`,
|
||||
createdBy: {
|
||||
augmentation: i < 10 ? 'service-a' : 'service-b'
|
||||
}
|
||||
}
|
||||
await db.add(metadata)
|
||||
}
|
||||
|
||||
// Get paginated results for service-a
|
||||
const page1 = await db.search('test item', 3, {
|
||||
service: 'service-a'
|
||||
})
|
||||
const page2 = await db.search('test item', 3, {
|
||||
service: 'service-a',
|
||||
offset: 3
|
||||
})
|
||||
|
||||
// Check that results are from service-a
|
||||
expect(page1.every(r => r.metadata.createdBy?.augmentation === 'service-a')).toBe(true)
|
||||
expect(page2.every(r => r.metadata.createdBy?.augmentation === 'service-a')).toBe(true)
|
||||
|
||||
// Check no overlap
|
||||
const page1Ids = page1.map(r => r.metadata.id)
|
||||
const page2Ids = page2.map(r => r.metadata.id)
|
||||
expect(page1Ids.filter(id => page2Ids.includes(id)).length).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Pagination Consistency', () => {
|
||||
it('should maintain consistent ordering across pages', async () => {
|
||||
// Add test data
|
||||
for (let i = 0; i < 30; i++) {
|
||||
await db.add({
|
||||
id: `item-${i.toString().padStart(2, '0')}`,
|
||||
text: `consistent test ${i}`,
|
||||
score: Math.random()
|
||||
})
|
||||
}
|
||||
|
||||
// Get all results in one query
|
||||
const allResults = await db.search('consistent test', 30)
|
||||
const allIds = allResults.map(r => r.metadata.id)
|
||||
|
||||
// Get results in pages
|
||||
const page1 = await db.search('consistent test', 10, { offset: 0 })
|
||||
const page2 = await db.search('consistent test', 10, { offset: 10 })
|
||||
const page3 = await db.search('consistent test', 10, { offset: 20 })
|
||||
|
||||
const pagedIds = [
|
||||
...page1.map(r => r.metadata.id),
|
||||
...page2.map(r => r.metadata.id),
|
||||
...page3.map(r => r.metadata.id)
|
||||
]
|
||||
|
||||
// Check that paginated results match the full query
|
||||
expect(pagedIds).toEqual(allIds)
|
||||
})
|
||||
|
||||
it('should handle empty results gracefully', async () => {
|
||||
// Search empty database with offset
|
||||
const results = await db.search('nonexistent', 10, { offset: 5 })
|
||||
expect(results).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Vector Search with Offset', () => {
|
||||
it('should paginate vector searches', async () => {
|
||||
// Add test vectors
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const vector = new Array(384).fill(0).map(() => Math.random())
|
||||
await db.add({
|
||||
id: `vec-${i}`,
|
||||
vector: vector,
|
||||
index: i
|
||||
})
|
||||
}
|
||||
|
||||
// Create a query vector
|
||||
const queryVector = new Array(384).fill(0).map(() => Math.random())
|
||||
|
||||
// Get first page
|
||||
const page1 = await db.search(queryVector, 5, { forceEmbed: false })
|
||||
expect(page1.length).toBe(5)
|
||||
|
||||
// Get second page
|
||||
const page2 = await db.search(queryVector, 5, {
|
||||
forceEmbed: false,
|
||||
offset: 5
|
||||
})
|
||||
expect(page2.length).toBe(5)
|
||||
|
||||
// Ensure different results
|
||||
const page1Ids = page1.map(r => r.metadata.id)
|
||||
const page2Ids = page2.map(r => r.metadata.id)
|
||||
expect(page1Ids.filter(id => page2Ids.includes(id)).length).toBe(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue