perf: make getRelations() pagination consistent and efficient

**Problem**: Pagination behavior was inconsistent across different query patterns:
- getRelations({ from: id, limit: 10 }) fetched ALL relationships then sliced
- getRelations({ limit: 10 }) paginated at storage layer
- storage.getVerbs() offset parameter wasn't being passed to adapters

**Root Cause**:
1. getRelations() used different code paths for from/to vs no-filter queries
2. storage.getVerbs() called getVerbsWithPagination without offset
3. Then tried to slice results, which failed for paginated queries

**Solution**:
- Unified getRelations() to ALWAYS use storage.getVerbs() with pagination
- Fixed storage.getVerbs() to convert offset to cursor for adapters
- All query patterns now paginate efficiently at storage layer
- Eliminated inefficient "fetch all then slice" pattern

**Performance Impact**:
- Before: getRelations({ from: entityId, limit: 10 }) on entity with 1000 relationships = 1000 fetched
- After: Only 10 fetched 
- All tests passing (14/14)

**Breaking**: None - fully backward compatible
This commit is contained in:
David Snelling 2025-10-21 13:28:38 -07:00
parent 8d217f3b84
commit 54d819cfcf
3 changed files with 39 additions and 46 deletions

View file

@ -911,58 +911,45 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const limit = params.limit || 100
const offset = params.offset || 0
let relations: Relation<T>[] = []
// Production safety: warn for large unfiltered queries
if (!params.from && !params.to && !params.type && limit > 10000) {
console.warn(
`[Brainy] getRelations(): Fetching ${limit} relationships without filters. ` +
`Consider adding 'from', 'to', or 'type' filter for better performance.`
)
}
// Build filter for storage query
const filter: any = {}
// Case 1: Filter by source
if (params.from) {
const verbs = await this.storage.getVerbsBySource(params.from)
relations.push(...this.verbsToRelations(verbs as any))
}
// Case 2: Filter by target
else if (params.to) {
const verbs = await this.storage.getVerbsByTarget(params.to)
relations.push(...this.verbsToRelations(verbs as any))
}
// Case 3: Get ALL relationships (NEW - fixes v4.1.2 bug)
else {
// Production safety: warn for large unfiltered queries
if (!params.type && limit > 10000) {
console.warn(
`[Brainy] getRelations(): Fetching ${limit} relationships without filters. ` +
`Consider adding 'type' filter or reducing 'limit' for better performance.`
)
}
// Fetch from storage using pagination
const result = await this.storage.getVerbs({
pagination: {
limit: limit + offset, // Fetch enough for offset + limit
offset: 0,
cursor: params.cursor
},
filter: params.type
? { verbType: Array.isArray(params.type) ? params.type : [params.type] as any }
: undefined
})
relations = this.verbsToRelations(result.items as any)
filter.sourceId = params.from
}
// Filter by type (only if not already filtered at storage level)
let filtered = relations
if (params.type && (params.from || params.to)) {
// Type filter only needed for from/to queries
const types = Array.isArray(params.type) ? params.type : [params.type]
filtered = relations.filter((r) => types.includes(r.type))
if (params.to) {
filter.targetId = params.to
}
if (params.type) {
filter.verbType = Array.isArray(params.type) ? params.type : [params.type]
}
// Filter by service
if (params.service) {
filtered = filtered.filter((r) => r.service === params.service)
filter.service = params.service
}
// Apply pagination (for from/to queries, or trim excess from storage query)
return filtered.slice(offset, offset + limit)
// Fetch from storage with pagination at storage layer (efficient!)
const result = await this.storage.getVerbs({
pagination: {
limit,
offset,
cursor: params.cursor
},
filter: Object.keys(filter).length > 0 ? filter : undefined
})
// Convert to Relation format
return this.verbsToRelations(result.items as any)
}
// ============= SEARCH & DISCOVERY =============