feat: v0.49 - Filter discovery API, remove deprecated methods, improve performance
BREAKING CHANGES: - Removed deprecated getAllNouns() and getAllVerbs() methods - All internal usage migrated to pagination-based methods New Features: - Filter Discovery API: - getFilterValues(field): Get all available values for a field - getFilterFields(): Get all filterable fields - Enables dynamic filter UI generation with O(1) field discovery - Hybrid metadata indexing with field-level indexes - Adaptive auto-flush for optimal performance - LRU caching for metadata indexes Improvements: - Fixed ENAMETOOLONG errors from vector-based filenames - Safe filename generation using hash-based approach - Scalable chunked value storage for millions of entries - Performance optimization with adaptive flush thresholds - Added support for $includes operator in metadata filters Technical: - Replaced vector-based filenames with safe hash approach - Implemented MetadataIndexCache with existing SearchCache pattern - Field indexes enable O(1) filter discovery - Adaptive flush based on performance metrics (20-200 entries) - All tests passing with improved metadata filtering
This commit is contained in:
parent
ac5b3183e3
commit
2dc909909a
17 changed files with 942 additions and 486 deletions
|
|
@ -17,8 +17,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
|
||||
abstract getNoun(id: string): Promise<any | null>
|
||||
|
||||
abstract getAllNouns(): Promise<any[]>
|
||||
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>
|
||||
|
||||
abstract deleteNoun(id: string): Promise<void>
|
||||
|
|
@ -27,8 +25,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
|
||||
abstract getVerb(id: string): Promise<any | null>
|
||||
|
||||
abstract getAllVerbs(): Promise<any[]>
|
||||
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>
|
||||
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>
|
||||
|
|
@ -54,6 +50,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
details?: Record<string, any>
|
||||
}>
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @returns Promise that resolves to an array of all nouns
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
|
||||
*/
|
||||
abstract getAllNouns(): Promise<any[]>
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* @returns Promise that resolves to an array of all HNSWVerbs
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
|
||||
*/
|
||||
abstract getAllVerbs(): Promise<any[]>
|
||||
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
|
|||
|
|
@ -752,12 +752,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
return this.getNode(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
return this.getAllNodes()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
|
|
@ -789,12 +783,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by source
|
||||
|
|
|
|||
|
|
@ -83,33 +83,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
return nounCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
const allNouns: HNSWNoun[] = []
|
||||
|
||||
// Iterate through all nouns in the nouns map
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allNouns.push(nounCopy)
|
||||
}
|
||||
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
@ -297,32 +270,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
return verbCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
const allVerbs: HNSWVerb[] = []
|
||||
|
||||
// Iterate through all verbs in the verbs map
|
||||
for (const [verbId, verb] of this.verbs.entries()) {
|
||||
// Create a deep copy of the HNSWVerb
|
||||
const verbCopy: HNSWVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allVerbs.push(verbCopy)
|
||||
}
|
||||
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
|
|||
|
|
@ -255,46 +255,6 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNouns: HNSWNoun_internal[] = []
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the noun data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nounIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
allNouns.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
level: data.level || 0
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading noun file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
|
|
@ -469,12 +429,6 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage (internal implementation)
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
|
|
|
|||
|
|
@ -466,17 +466,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage (internal implementation)
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
// Use paginated method to avoid deprecation warning
|
||||
const result = await this.getNodesWithPagination({
|
||||
limit: 1000,
|
||||
useCache: true
|
||||
})
|
||||
return result.nodes
|
||||
}
|
||||
|
||||
// Node cache to avoid redundant API calls
|
||||
private nodeCache = new Map<string, HNSWNode>()
|
||||
|
|
@ -854,15 +843,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage (internal implementation)
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getVerbsWithPagination() instead.
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
this.logger.warn('getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.')
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
|
|
|
|||
|
|
@ -92,17 +92,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
return this.getNoun_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getNouns() with pagination instead.
|
||||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
console.warn('WARNING: getAllNouns() is deprecated and will be removed in a future version. Use getNouns() with pagination instead.')
|
||||
return this.getAllNouns_internal()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
|
|
@ -215,24 +204,34 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getVerbs() with pagination instead.
|
||||
* @returns Promise that resolves to an array of all HNSWVerbs
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
|
||||
*/
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
public async getAllVerbs(): Promise<HNSWVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
console.warn('WARNING: getAllVerbs() is deprecated and will be removed in a future version. Use getVerbs() with pagination instead.')
|
||||
|
||||
const hnswVerbs = await this.getAllVerbs_internal()
|
||||
const graphVerbs: GraphVerb[] = []
|
||||
console.warn('getAllVerbs() is deprecated and may cause memory issues with large datasets. Consider using getVerbs() with pagination instead.')
|
||||
|
||||
for (const hnswVerb of hnswVerbs) {
|
||||
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb)
|
||||
if (graphVerb) {
|
||||
graphVerbs.push(graphVerb)
|
||||
// Get all verbs using the paginated method with a very large limit
|
||||
const result = await this.getVerbs({
|
||||
pagination: {
|
||||
limit: Number.MAX_SAFE_INTEGER
|
||||
}
|
||||
})
|
||||
|
||||
// Convert GraphVerbs back to HNSWVerbs since that's what this method should return
|
||||
const hnswVerbs: HNSWVerb[] = []
|
||||
for (const graphVerb of result.items) {
|
||||
// Create an HNSWVerb from the GraphVerb (reverse conversion)
|
||||
const hnswVerb: HNSWVerb = {
|
||||
id: graphVerb.id,
|
||||
vector: graphVerb.vector,
|
||||
connections: new Map() // HNSWVerbs need connections, but GraphVerbs don't have them
|
||||
}
|
||||
hnswVerbs.push(hnswVerb)
|
||||
}
|
||||
|
||||
return graphVerbs
|
||||
return hnswVerbs
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -241,11 +240,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get all verbs and filter by source
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter(verb =>
|
||||
verb.sourceId === sourceId || verb.source === sourceId
|
||||
)
|
||||
// Use the paginated getVerbs method with source filter
|
||||
const result = await this.getVerbs({
|
||||
filter: { sourceId }
|
||||
})
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -254,11 +253,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get all verbs and filter by target
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter(verb =>
|
||||
verb.targetId === targetId || verb.target === targetId
|
||||
)
|
||||
// Use the paginated getVerbs method with target filter
|
||||
const result = await this.getVerbs({
|
||||
filter: { targetId }
|
||||
})
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -267,11 +266,30 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get all verbs and filter by type
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter(verb =>
|
||||
verb.type === type || verb.verb === type
|
||||
)
|
||||
// Use the paginated getVerbs method with type filter
|
||||
const result = await this.getVerbs({
|
||||
filter: { verbType: type }
|
||||
})
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @returns Promise that resolves to an array of all nouns
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
|
||||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
console.warn('getAllNouns() is deprecated and may cause memory issues with large datasets. Consider using getNouns() with pagination instead.')
|
||||
|
||||
const result = await this.getNouns({
|
||||
pagination: {
|
||||
limit: Number.MAX_SAFE_INTEGER
|
||||
}
|
||||
})
|
||||
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -374,100 +392,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// If the adapter doesn't have a paginated method, fall back to the old approach
|
||||
// but with a warning and a reasonable limit
|
||||
console.warn(
|
||||
'Storage adapter does not support pagination, falling back to loading all nouns. This may cause performance issues with large datasets.'
|
||||
// Storage adapter does not support pagination
|
||||
console.error(
|
||||
'Storage adapter does not support pagination. The deprecated getAllNouns_internal() method has been removed. Please implement getNounsWithPagination() in your storage adapter.'
|
||||
)
|
||||
|
||||
// Get nouns with a reasonable limit to avoid memory issues
|
||||
const maxNouns = Math.min(offset + limit + 100, 1000) // Reasonable limit
|
||||
let allNouns: HNSWNoun[] = []
|
||||
|
||||
try {
|
||||
// Try to get only the nouns we need
|
||||
allNouns = await this.getAllNouns_internal()
|
||||
|
||||
// If we have too many nouns, truncate the array to avoid memory issues
|
||||
if (allNouns.length > maxNouns) {
|
||||
console.warn(
|
||||
`Large number of nouns (${allNouns.length}), truncating to ${maxNouns} for filtering`
|
||||
)
|
||||
allNouns = allNouns.slice(0, maxNouns)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting all nouns:', error)
|
||||
// Return empty result on error
|
||||
return {
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filtering if needed
|
||||
let filteredNouns = allNouns
|
||||
|
||||
if (options?.filter) {
|
||||
// Filter by noun type
|
||||
if (options.filter.nounType) {
|
||||
const nounTypes = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType
|
||||
: [options.filter.nounType]
|
||||
|
||||
filteredNouns = filteredNouns.filter((noun) => {
|
||||
// HNSWNoun doesn't have a type property directly, check metadata
|
||||
const nounType = noun.metadata?.type
|
||||
return typeof nounType === 'string' && nounTypes.includes(nounType)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
|
||||
filteredNouns = filteredNouns.filter((noun) => {
|
||||
// HNSWNoun doesn't have a service property directly, check metadata
|
||||
const service = noun.metadata?.service
|
||||
return typeof service === 'string' && services.includes(service)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
const metadataFilter = options.filter.metadata
|
||||
filteredNouns = filteredNouns.filter((noun) => {
|
||||
if (!noun.metadata) return false
|
||||
|
||||
// Check if all metadata keys match
|
||||
return Object.entries(metadataFilter).every(
|
||||
([key, value]) => noun.metadata && noun.metadata[key] === value
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Get total count before pagination
|
||||
totalCount = totalCount || filteredNouns.length
|
||||
|
||||
// Apply pagination
|
||||
const paginatedNouns = filteredNouns.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < filteredNouns.length || filteredNouns.length >= maxNouns
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedNouns.length > 0) {
|
||||
const lastItem = paginatedNouns[paginatedNouns.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
items: paginatedNouns,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting nouns with pagination:', error)
|
||||
|
|
@ -651,122 +584,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// If the adapter doesn't have a paginated method, fall back to the old approach
|
||||
// but with a warning and a reasonable limit
|
||||
console.warn(
|
||||
'Storage adapter does not support pagination, falling back to loading all verbs. This may cause performance issues with large datasets.'
|
||||
// Storage adapter does not support pagination
|
||||
console.error(
|
||||
'Storage adapter does not support pagination. The deprecated getAllVerbs_internal() method has been removed. Please implement getVerbsWithPagination() in your storage adapter.'
|
||||
)
|
||||
|
||||
// Get verbs with a reasonable limit to avoid memory issues
|
||||
const maxVerbs = Math.min(offset + limit + 100, 1000) // Reasonable limit
|
||||
let allVerbs: GraphVerb[] = []
|
||||
|
||||
try {
|
||||
// Try to get only the verbs we need
|
||||
allVerbs = await this.getAllVerbs()
|
||||
|
||||
// If we have too many verbs, truncate the array to avoid memory issues
|
||||
if (allVerbs.length > maxVerbs) {
|
||||
console.warn(
|
||||
`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs} for filtering`
|
||||
)
|
||||
allVerbs = allVerbs.slice(0, maxVerbs)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting all verbs:', error)
|
||||
// Return empty result on error
|
||||
return {
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filtering if needed
|
||||
let filteredVerbs = allVerbs
|
||||
|
||||
if (options?.filter) {
|
||||
// Filter by verb type
|
||||
if (options.filter.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType
|
||||
: [options.filter.verbType]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(
|
||||
(verb) => verb.type !== undefined && verbTypes.includes(verb.type)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by source ID
|
||||
if (options.filter.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId
|
||||
: [options.filter.sourceId]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(
|
||||
(verb) =>
|
||||
verb.sourceId !== undefined && sourceIds.includes(verb.sourceId)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by target ID
|
||||
if (options.filter.targetId) {
|
||||
const targetIds = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId
|
||||
: [options.filter.targetId]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(
|
||||
(verb) =>
|
||||
verb.targetId !== undefined && targetIds.includes(verb.targetId)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter((verb) => {
|
||||
// GraphVerb doesn't have a service property directly, check metadata
|
||||
const service = verb.metadata?.service
|
||||
return typeof service === 'string' && services.includes(service)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
const metadataFilter = options.filter.metadata
|
||||
filteredVerbs = filteredVerbs.filter((verb) => {
|
||||
if (!verb.metadata) return false
|
||||
|
||||
// Check if all metadata keys match
|
||||
return Object.entries(metadataFilter).every(
|
||||
([key, value]) => verb.metadata && verb.metadata[key] === value
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Get total count before pagination
|
||||
totalCount = totalCount || filteredVerbs.length
|
||||
|
||||
// Apply pagination
|
||||
const paginatedVerbs = filteredVerbs.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < filteredVerbs.length || filteredVerbs.length >= maxVerbs
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedVerbs.length > 0) {
|
||||
const lastItem = paginatedVerbs[paginatedVerbs.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting verbs with pagination:', error)
|
||||
|
|
@ -851,12 +677,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
*/
|
||||
protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllNouns_internal(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* This method should be implemented by each specific adapter
|
||||
|
|
@ -883,12 +703,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
*/
|
||||
protected abstract getVerb_internal(id: string): Promise<HNSWVerb | null>
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllVerbs_internal(): Promise<HNSWVerb[]>
|
||||
|
||||
/**
|
||||
* Get verbs by source
|
||||
* This method should be implemented by each specific adapter
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue