perf: make getRelations() pagination consistent and efficient

**Problem**: Pagination behavior was inconsistent across different query patterns:
- getRelations({ from: id, limit: 10 }) fetched ALL relationships then sliced
- getRelations({ limit: 10 }) paginated at storage layer
- storage.getVerbs() offset parameter wasn't being passed to adapters

**Root Cause**:
1. getRelations() used different code paths for from/to vs no-filter queries
2. storage.getVerbs() called getVerbsWithPagination without offset
3. Then tried to slice results, which failed for paginated queries

**Solution**:
- Unified getRelations() to ALWAYS use storage.getVerbs() with pagination
- Fixed storage.getVerbs() to convert offset to cursor for adapters
- All query patterns now paginate efficiently at storage layer
- Eliminated inefficient "fetch all then slice" pattern

**Performance Impact**:
- Before: getRelations({ from: entityId, limit: 10 }) on entity with 1000 relationships = 1000 fetched
- After: Only 10 fetched 
- All tests passing (14/14)

**Breaking**: None - fully backward compatible
This commit is contained in:
David Snelling 2025-10-21 13:28:38 -07:00
parent 8d217f3b84
commit 54d819cfcf
3 changed files with 39 additions and 46 deletions

View file

@ -11,6 +11,9 @@ All notable changes to this project will be documented in this file. See [standa
- Fixed critical bug where `brain.getRelations()` returned `[]` instead of all relationships - Fixed critical bug where `brain.getRelations()` returned `[]` instead of all relationships
- Added support for retrieving all relationships with pagination (default limit: 100) - Added support for retrieving all relationships with pagination (default limit: 100)
- Added string ID shorthand syntax: `brain.getRelations(entityId)` as alias for `brain.getRelations({ from: entityId })` - Added string ID shorthand syntax: `brain.getRelations(entityId)` as alias for `brain.getRelations({ from: entityId })`
- **Performance**: Made pagination consistent - now ALL query patterns paginate at storage layer
- **Efficiency**: `getRelations({ from: id, limit: 10 })` now fetches only 10 instead of fetching ALL then slicing
- Fixed storage.getVerbs() offset handling - now properly converts offset to cursor for adapters
- Production safety: Warns when fetching >10k relationships without filters - Production safety: Warns when fetching >10k relationships without filters
- Fixed broken method calls in improvedNeuralAPI.ts (replaced non-existent `getVerbsForNoun` with `getRelations`) - Fixed broken method calls in improvedNeuralAPI.ts (replaced non-existent `getVerbsForNoun` with `getRelations`)
- Fixed property access bugs: `verb.target``verb.to`, `verb.verb``verb.type` - Fixed property access bugs: `verb.target``verb.to`, `verb.verb``verb.type`

View file

@ -911,58 +911,45 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const limit = params.limit || 100 const limit = params.limit || 100
const offset = params.offset || 0 const offset = params.offset || 0
let relations: Relation<T>[] = []
// Case 1: Filter by source
if (params.from) {
const verbs = await this.storage.getVerbsBySource(params.from)
relations.push(...this.verbsToRelations(verbs as any))
}
// Case 2: Filter by target
else if (params.to) {
const verbs = await this.storage.getVerbsByTarget(params.to)
relations.push(...this.verbsToRelations(verbs as any))
}
// Case 3: Get ALL relationships (NEW - fixes v4.1.2 bug)
else {
// Production safety: warn for large unfiltered queries // Production safety: warn for large unfiltered queries
if (!params.type && limit > 10000) { if (!params.from && !params.to && !params.type && limit > 10000) {
console.warn( console.warn(
`[Brainy] getRelations(): Fetching ${limit} relationships without filters. ` + `[Brainy] getRelations(): Fetching ${limit} relationships without filters. ` +
`Consider adding 'type' filter or reducing 'limit' for better performance.` `Consider adding 'from', 'to', or 'type' filter for better performance.`
) )
} }
// Fetch from storage using pagination // Build filter for storage query
const filter: any = {}
if (params.from) {
filter.sourceId = params.from
}
if (params.to) {
filter.targetId = params.to
}
if (params.type) {
filter.verbType = Array.isArray(params.type) ? params.type : [params.type]
}
if (params.service) {
filter.service = params.service
}
// Fetch from storage with pagination at storage layer (efficient!)
const result = await this.storage.getVerbs({ const result = await this.storage.getVerbs({
pagination: { pagination: {
limit: limit + offset, // Fetch enough for offset + limit limit,
offset: 0, offset,
cursor: params.cursor cursor: params.cursor
}, },
filter: params.type filter: Object.keys(filter).length > 0 ? filter : undefined
? { verbType: Array.isArray(params.type) ? params.type : [params.type] as any }
: undefined
}) })
relations = this.verbsToRelations(result.items as any) // Convert to Relation format
} return this.verbsToRelations(result.items as any)
// Filter by type (only if not already filtered at storage level)
let filtered = relations
if (params.type && (params.from || params.to)) {
// Type filter only needed for from/to queries
const types = Array.isArray(params.type) ? params.type : [params.type]
filtered = relations.filter((r) => types.includes(r.type))
}
// Filter by service
if (params.service) {
filtered = filtered.filter((r) => r.service === params.service)
}
// Apply pagination (for from/to queries, or trim excess from storage query)
return filtered.slice(offset, offset + limit)
} }
// ============= SEARCH & DISCOVERY ============= // ============= SEARCH & DISCOVERY =============

View file

@ -752,14 +752,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Check if the adapter has a paginated method for getting verbs // Check if the adapter has a paginated method for getting verbs
if (typeof (this as any).getVerbsWithPagination === 'function') { if (typeof (this as any).getVerbsWithPagination === 'function') {
// Use the adapter's paginated method // Use the adapter's paginated method
// Convert offset to cursor if no cursor provided (adapters use cursor for offset)
const effectiveCursor = cursor || (offset > 0 ? offset.toString() : undefined)
const result = await (this as any).getVerbsWithPagination({ const result = await (this as any).getVerbsWithPagination({
limit, limit,
cursor, cursor: effectiveCursor,
filter: options?.filter filter: options?.filter
}) })
// Apply offset if needed (some adapters might not support offset) // Items are already offset by the adapter via cursor, no need to slice
const items = result.items.slice(offset) const items = result.items
// CRITICAL SAFETY CHECK: Prevent infinite loops // CRITICAL SAFETY CHECK: Prevent infinite loops
// If we have no items but hasMore is true, force hasMore to false // If we have no items but hasMore is true, force hasMore to false