**feat(storage): add pagination and filtering support for nouns and verbs**
- Introduced `PaginationOptions`, `NounFilterOptions`, and `VerbFilterOptions` types for improved query flexibility in data retrieval operations. - Added `getNouns` and `getVerbs` methods with pagination and filtering capabilities, replacing existing methods for broader use cases and scalability. - Marked legacy methods (`getAllNouns`, `getAllVerbs`, `getVerbsBySource`, `getVerbsByTarget`, `getVerbsByType`) as deprecated, directing users to use new methods. - Updated `coreTypes`, `memoryStorage`, and related modules to support new functionality, including cursor and offset-based pagination handling. - Updated fallback logic for storage adapters, ensuring compatibility with non-paginated operations when required. **Purpose**: Enhance scalability and query precision by implementing paginated and filtered retrieval of nouns and verbs, aligning query methods with modern requirements.
This commit is contained in:
parent
59caa6ab5b
commit
c3c4ca31e1
8 changed files with 1309 additions and 143 deletions
|
|
@ -124,6 +124,393 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
await this.ensureInitialized()
|
||||
return this.getVerbsByType_internal(type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of nouns
|
||||
*/
|
||||
public async getNouns(options?: {
|
||||
pagination?: {
|
||||
offset?: number
|
||||
limit?: number
|
||||
cursor?: string
|
||||
}
|
||||
filter?: {
|
||||
nounType?: string | string[]
|
||||
service?: string | string[]
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: HNSWNoun[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Set default pagination values
|
||||
const pagination = options?.pagination || {}
|
||||
const limit = pagination.limit || 100
|
||||
const offset = pagination.offset || 0
|
||||
|
||||
// Optimize for common filter cases to avoid loading all nouns
|
||||
if (options?.filter) {
|
||||
// If filtering by nounType only, use the optimized method
|
||||
if (options.filter.nounType && !options.filter.service && !options.filter.metadata) {
|
||||
const nounType = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType[0]
|
||||
: options.filter.nounType
|
||||
|
||||
// Get nouns by type directly
|
||||
const nounsByType = await this.getNounsByNounType_internal(nounType)
|
||||
|
||||
// Apply pagination
|
||||
const paginatedNouns = nounsByType.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < nounsByType.length
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedNouns.length > 0) {
|
||||
const lastItem = paginatedNouns[paginatedNouns.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedNouns,
|
||||
totalCount: nounsByType.length,
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For more complex filtering or no filtering, we need to get all nouns
|
||||
// but limit the number we load to avoid memory issues
|
||||
const maxNouns = offset + limit + 1 // Get one extra to check if there are more
|
||||
let allNouns: HNSWNoun[] = []
|
||||
|
||||
try {
|
||||
// Try to get only the nouns we need
|
||||
allNouns = await this.getAllNouns_internal()
|
||||
|
||||
// If we have too many nouns, truncate the array to avoid memory issues
|
||||
if (allNouns.length > maxNouns * 10) {
|
||||
console.warn(`Large number of nouns (${allNouns.length}), truncating to ${maxNouns * 10} for filtering`)
|
||||
allNouns = allNouns.slice(0, maxNouns * 10)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting all nouns:', error)
|
||||
// Return empty result on error
|
||||
return {
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filtering if needed
|
||||
let filteredNouns = allNouns
|
||||
|
||||
if (options?.filter) {
|
||||
// Filter by noun type
|
||||
if (options.filter.nounType) {
|
||||
const nounTypes = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType
|
||||
: [options.filter.nounType]
|
||||
|
||||
filteredNouns = filteredNouns.filter(noun => {
|
||||
// HNSWNoun doesn't have a type property directly, check metadata
|
||||
const nounType = noun.metadata?.type
|
||||
return typeof nounType === 'string' && nounTypes.includes(nounType)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
|
||||
filteredNouns = filteredNouns.filter(noun => {
|
||||
// HNSWNoun doesn't have a service property directly, check metadata
|
||||
const service = noun.metadata?.service
|
||||
return typeof service === 'string' && services.includes(service)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
const metadataFilter = options.filter.metadata
|
||||
filteredNouns = filteredNouns.filter(noun => {
|
||||
if (!noun.metadata) return false
|
||||
|
||||
// Check if all metadata keys match
|
||||
return Object.entries(metadataFilter).every(([key, value]) =>
|
||||
noun.metadata && noun.metadata[key] === value
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Get total count before pagination
|
||||
const totalCount = filteredNouns.length
|
||||
|
||||
// Apply pagination
|
||||
const paginatedNouns = filteredNouns.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < totalCount
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedNouns.length > 0) {
|
||||
const lastItem = paginatedNouns[paginatedNouns.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedNouns,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
* @returns Promise that resolves to a paginated result of verbs
|
||||
*/
|
||||
public async getVerbs(options?: {
|
||||
pagination?: {
|
||||
offset?: number
|
||||
limit?: number
|
||||
cursor?: string
|
||||
}
|
||||
filter?: {
|
||||
verbType?: string | string[]
|
||||
sourceId?: string | string[]
|
||||
targetId?: string | string[]
|
||||
service?: string | string[]
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: GraphVerb[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Set default pagination values
|
||||
const pagination = options?.pagination || {}
|
||||
const limit = pagination.limit || 100
|
||||
const offset = pagination.offset || 0
|
||||
|
||||
// Optimize for common filter cases to avoid loading all verbs
|
||||
if (options?.filter) {
|
||||
// If filtering by sourceId only, use the optimized method
|
||||
if (options.filter.sourceId && !options.filter.verbType &&
|
||||
!options.filter.targetId && !options.filter.service &&
|
||||
!options.filter.metadata) {
|
||||
|
||||
const sourceId = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId[0]
|
||||
: options.filter.sourceId
|
||||
|
||||
// Get verbs by source directly
|
||||
const verbsBySource = await this.getVerbsBySource_internal(sourceId)
|
||||
|
||||
// Apply pagination
|
||||
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < verbsBySource.length
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedVerbs.length > 0) {
|
||||
const lastItem = paginatedVerbs[paginatedVerbs.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount: verbsBySource.length,
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
|
||||
// If filtering by targetId only, use the optimized method
|
||||
if (options.filter.targetId && !options.filter.verbType &&
|
||||
!options.filter.sourceId && !options.filter.service &&
|
||||
!options.filter.metadata) {
|
||||
|
||||
const targetId = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId[0]
|
||||
: options.filter.targetId
|
||||
|
||||
// Get verbs by target directly
|
||||
const verbsByTarget = await this.getVerbsByTarget_internal(targetId)
|
||||
|
||||
// Apply pagination
|
||||
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < verbsByTarget.length
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedVerbs.length > 0) {
|
||||
const lastItem = paginatedVerbs[paginatedVerbs.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount: verbsByTarget.length,
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
|
||||
// If filtering by verbType only, use the optimized method
|
||||
if (options.filter.verbType && !options.filter.sourceId &&
|
||||
!options.filter.targetId && !options.filter.service &&
|
||||
!options.filter.metadata) {
|
||||
|
||||
const verbType = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType[0]
|
||||
: options.filter.verbType
|
||||
|
||||
// Get verbs by type directly
|
||||
const verbsByType = await this.getVerbsByType_internal(verbType)
|
||||
|
||||
// Apply pagination
|
||||
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < verbsByType.length
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedVerbs.length > 0) {
|
||||
const lastItem = paginatedVerbs[paginatedVerbs.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount: verbsByType.length,
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For more complex filtering or no filtering, we need to get all verbs
|
||||
// but limit the number we load to avoid memory issues
|
||||
const maxVerbs = offset + limit + 1 // Get one extra to check if there are more
|
||||
let allVerbs: GraphVerb[] = []
|
||||
|
||||
try {
|
||||
// Try to get only the verbs we need
|
||||
allVerbs = await this.getAllVerbs_internal()
|
||||
|
||||
// If we have too many verbs, truncate the array to avoid memory issues
|
||||
if (allVerbs.length > maxVerbs * 10) {
|
||||
console.warn(`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs * 10} for filtering`)
|
||||
allVerbs = allVerbs.slice(0, maxVerbs * 10)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting all verbs:', error)
|
||||
// Return empty result on error
|
||||
return {
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filtering if needed
|
||||
let filteredVerbs = allVerbs
|
||||
|
||||
if (options?.filter) {
|
||||
// Filter by verb type
|
||||
if (options.filter.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType
|
||||
: [options.filter.verbType]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(verb =>
|
||||
verb.type !== undefined && verbTypes.includes(verb.type)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by source ID
|
||||
if (options.filter.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId
|
||||
: [options.filter.sourceId]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(verb =>
|
||||
verb.sourceId !== undefined && sourceIds.includes(verb.sourceId)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by target ID
|
||||
if (options.filter.targetId) {
|
||||
const targetIds = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId
|
||||
: [options.filter.targetId]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(verb =>
|
||||
verb.targetId !== undefined && targetIds.includes(verb.targetId)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(verb => {
|
||||
// GraphVerb doesn't have a service property directly, check metadata
|
||||
const service = verb.metadata?.service
|
||||
return typeof service === 'string' && services.includes(service)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
const metadataFilter = options.filter.metadata
|
||||
filteredVerbs = filteredVerbs.filter(verb => {
|
||||
if (!verb.metadata) return false
|
||||
|
||||
// Check if all metadata keys match
|
||||
return Object.entries(metadataFilter).every(([key, value]) =>
|
||||
verb.metadata && verb.metadata[key] === value
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Get total count before pagination
|
||||
const totalCount = filteredVerbs.length
|
||||
|
||||
// Apply pagination
|
||||
const paginatedVerbs = filteredVerbs.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < totalCount
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedVerbs.length > 0) {
|
||||
const lastItem = paginatedVerbs[paginatedVerbs.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a verb from storage
|
||||
|
|
@ -261,4 +648,4 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* @returns Promise that resolves to the statistics data or null if not found
|
||||
*/
|
||||
protected abstract getStatisticsData(): Promise<StatisticsData | null>
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue