David Snelling
33afd715a0
- Added SearchCursor and PaginatedSearchResult interfaces for cursor-based pagination support. - Introduced SearchCache class to cache search results, improving performance. - Implemented tests for automatic cache configuration and performance improvements. - Enhanced existing tests to validate pagination and caching behavior.
255 lines
No EOL
7.8 KiB
TypeScript
255 lines
No EOL
7.8 KiB
TypeScript
/**
|
|
* Tests for offset-based pagination in search results
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { BrainyData } from '../src/brainyData.js'
|
|
import { cleanupWorkerPools } from '../src/utils/index.js'
|
|
|
|
describe('Pagination with Offset', () => {
|
|
let db: BrainyData
|
|
|
|
beforeEach(async () => {
|
|
// Initialize BrainyData with in-memory storage for testing
|
|
db = new BrainyData({
|
|
storage: {
|
|
forceMemoryStorage: true
|
|
},
|
|
logging: {
|
|
verbose: false
|
|
}
|
|
})
|
|
await db.init()
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await db.clear()
|
|
await cleanupWorkerPools()
|
|
})
|
|
|
|
describe('Basic Offset Pagination', () => {
|
|
it('should return correct results with offset=0', async () => {
|
|
// Add test data
|
|
const testData = []
|
|
for (let i = 0; i < 20; i++) {
|
|
const item = {
|
|
id: `item-${i}`,
|
|
text: `test document ${i}`,
|
|
value: i
|
|
}
|
|
testData.push(item)
|
|
await db.add(item)
|
|
}
|
|
|
|
// Search without offset (default offset=0)
|
|
const results = await db.search('test document', 5)
|
|
expect(results.length).toBe(5)
|
|
|
|
// Results should be the top 5 most similar
|
|
const resultIds = results.map(r => r.metadata.id)
|
|
expect(resultIds.length).toBe(5)
|
|
})
|
|
|
|
it('should skip results with offset > 0', async () => {
|
|
// Add test data
|
|
const testData = []
|
|
for (let i = 0; i < 20; i++) {
|
|
const item = {
|
|
id: `item-${i}`,
|
|
text: `test document ${i}`,
|
|
value: i
|
|
}
|
|
testData.push(item)
|
|
await db.add(item)
|
|
}
|
|
|
|
// Get first page (no offset)
|
|
const firstPage = await db.search('test document', 5)
|
|
expect(firstPage.length).toBe(5)
|
|
const firstPageIds = firstPage.map(r => r.metadata.id)
|
|
|
|
// Get second page (offset=5)
|
|
const secondPage = await db.search('test document', 5, { offset: 5 })
|
|
expect(secondPage.length).toBe(5)
|
|
const secondPageIds = secondPage.map(r => r.metadata.id)
|
|
|
|
// Ensure no overlap between pages
|
|
const overlap = firstPageIds.filter(id => secondPageIds.includes(id))
|
|
expect(overlap.length).toBe(0)
|
|
})
|
|
|
|
it('should handle offset beyond available results', async () => {
|
|
// Add limited test data
|
|
for (let i = 0; i < 10; i++) {
|
|
await db.add({
|
|
id: `item-${i}`,
|
|
text: `test document ${i}`
|
|
})
|
|
}
|
|
|
|
// Search with offset beyond available results
|
|
const results = await db.search('test document', 5, { offset: 15 })
|
|
expect(results.length).toBe(0)
|
|
})
|
|
|
|
it('should return partial results when offset + k exceeds total', async () => {
|
|
// Add limited test data
|
|
for (let i = 0; i < 10; i++) {
|
|
await db.add({
|
|
id: `item-${i}`,
|
|
text: `test document ${i}`
|
|
})
|
|
}
|
|
|
|
// Search with offset that allows only partial results
|
|
const results = await db.search('test document', 5, { offset: 7 })
|
|
expect(results.length).toBe(3) // Only 3 results available after offset 7
|
|
})
|
|
})
|
|
|
|
describe('Pagination with Filters', () => {
|
|
it.skip('should paginate with noun type filters', async () => {
|
|
// TODO: This test requires proper noun type support in the add method
|
|
// Currently skipped as noun types are not directly supported in the add method
|
|
// Add test data with different noun types
|
|
for (let i = 0; i < 15; i++) {
|
|
await db.add({
|
|
id: `doc-${i}`,
|
|
text: `document ${i}`,
|
|
type: 'document'
|
|
}, undefined, 'document')
|
|
}
|
|
|
|
for (let i = 0; i < 15; i++) {
|
|
await db.add({
|
|
id: `note-${i}`,
|
|
text: `note ${i}`,
|
|
type: 'note'
|
|
}, undefined, 'note')
|
|
}
|
|
|
|
// Get first page of documents
|
|
const firstPage = await db.search('document', 5, {
|
|
nounTypes: ['document']
|
|
})
|
|
expect(firstPage.length).toBe(5)
|
|
expect(firstPage.every(r => r.metadata.type === 'document')).toBe(true)
|
|
|
|
// Get second page of documents
|
|
const secondPage = await db.search('document', 5, {
|
|
nounTypes: ['document'],
|
|
offset: 5
|
|
})
|
|
expect(secondPage.length).toBe(5)
|
|
expect(secondPage.every(r => r.metadata.type === 'document')).toBe(true)
|
|
|
|
// Ensure pages are different
|
|
const firstIds = firstPage.map(r => r.metadata.id)
|
|
const secondIds = secondPage.map(r => r.metadata.id)
|
|
const overlap = firstIds.filter(id => secondIds.includes(id))
|
|
expect(overlap.length).toBe(0)
|
|
})
|
|
|
|
it('should paginate with service filters', async () => {
|
|
// Add test data with different services
|
|
for (let i = 0; i < 20; i++) {
|
|
const metadata = {
|
|
id: `item-${i}`,
|
|
text: `test item ${i}`,
|
|
createdBy: {
|
|
augmentation: i < 10 ? 'service-a' : 'service-b'
|
|
}
|
|
}
|
|
await db.add(metadata)
|
|
}
|
|
|
|
// Get paginated results for service-a
|
|
const page1 = await db.search('test item', 3, {
|
|
service: 'service-a'
|
|
})
|
|
const page2 = await db.search('test item', 3, {
|
|
service: 'service-a',
|
|
offset: 3
|
|
})
|
|
|
|
// Check that results are from service-a
|
|
expect(page1.every(r => r.metadata.createdBy?.augmentation === 'service-a')).toBe(true)
|
|
expect(page2.every(r => r.metadata.createdBy?.augmentation === 'service-a')).toBe(true)
|
|
|
|
// Check no overlap
|
|
const page1Ids = page1.map(r => r.metadata.id)
|
|
const page2Ids = page2.map(r => r.metadata.id)
|
|
expect(page1Ids.filter(id => page2Ids.includes(id)).length).toBe(0)
|
|
})
|
|
})
|
|
|
|
describe('Pagination Consistency', () => {
|
|
it('should maintain consistent ordering across pages', async () => {
|
|
// Add test data
|
|
for (let i = 0; i < 30; i++) {
|
|
await db.add({
|
|
id: `item-${i.toString().padStart(2, '0')}`,
|
|
text: `consistent test ${i}`,
|
|
score: Math.random()
|
|
})
|
|
}
|
|
|
|
// Get all results in one query
|
|
const allResults = await db.search('consistent test', 30)
|
|
const allIds = allResults.map(r => r.metadata.id)
|
|
|
|
// Get results in pages
|
|
const page1 = await db.search('consistent test', 10, { offset: 0 })
|
|
const page2 = await db.search('consistent test', 10, { offset: 10 })
|
|
const page3 = await db.search('consistent test', 10, { offset: 20 })
|
|
|
|
const pagedIds = [
|
|
...page1.map(r => r.metadata.id),
|
|
...page2.map(r => r.metadata.id),
|
|
...page3.map(r => r.metadata.id)
|
|
]
|
|
|
|
// Check that paginated results match the full query
|
|
expect(pagedIds).toEqual(allIds)
|
|
})
|
|
|
|
it('should handle empty results gracefully', async () => {
|
|
// Search empty database with offset
|
|
const results = await db.search('nonexistent', 10, { offset: 5 })
|
|
expect(results).toEqual([])
|
|
})
|
|
})
|
|
|
|
describe('Vector Search with Offset', () => {
|
|
it('should paginate vector searches', async () => {
|
|
// Add test vectors
|
|
for (let i = 0; i < 20; i++) {
|
|
const vector = new Array(512).fill(0).map(() => Math.random())
|
|
await db.add({
|
|
id: `vec-${i}`,
|
|
vector: vector,
|
|
index: i
|
|
})
|
|
}
|
|
|
|
// Create a query vector
|
|
const queryVector = new Array(512).fill(0).map(() => Math.random())
|
|
|
|
// Get first page
|
|
const page1 = await db.search(queryVector, 5, { forceEmbed: false })
|
|
expect(page1.length).toBe(5)
|
|
|
|
// Get second page
|
|
const page2 = await db.search(queryVector, 5, {
|
|
forceEmbed: false,
|
|
offset: 5
|
|
})
|
|
expect(page2.length).toBe(5)
|
|
|
|
// Ensure different results
|
|
const page1Ids = page1.map(r => r.metadata.id)
|
|
const page2Ids = page2.map(r => r.metadata.id)
|
|
expect(page1Ids.filter(id => page2Ids.includes(id)).length).toBe(0)
|
|
})
|
|
})
|
|
}) |