From 3d46dea814ed70daebb05dff65672e95bfc501a4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 6 Aug 2025 15:00:34 -0700 Subject: [PATCH] feat: implement pagination methods for OPFSStorage adapter - Add getNounsWithPagination and getVerbsWithPagination methods - Update mock to provide async iterators for entries/values/keys - Fix compatibility with BaseStorage pagination requirements - Resolve test failures related to getAllNouns/getAllVerbs deprecation --- src/storage/adapters/opfsStorage.ts | 249 ++++++++++++++++++++++++++++ tests/mocks/opfs-mock.ts | 20 ++- 2 files changed, 268 insertions(+), 1 deletion(-) diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index e3beda53..04c84a3d 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -1276,4 +1276,253 @@ export class OPFSStorage extends BaseStorage { throw new Error(`Failed to get statistics data: ${error}`) } } + + /** + * Get nouns with pagination support + * @param options Pagination and filter options + * @returns Promise that resolves to a paginated result of nouns + */ + public async getNounsWithPagination(options: { + limit?: number + cursor?: string + filter?: { + nounType?: string | string[] + service?: string | string[] + metadata?: Record + } + } = {}): Promise<{ + items: HNSWNoun[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + await this.ensureInitialized() + + const limit = options.limit || 100 + const cursor = options.cursor + + // Get all noun files + const nounFiles: string[] = [] + if (this.nounsDir) { + for await (const [name, handle] of this.nounsDir.entries()) { + if (handle.kind === 'file' && name.endsWith('.json')) { + nounFiles.push(name) + } + } + } + + // Sort files for consistent ordering + nounFiles.sort() + + // Apply cursor-based pagination + let startIndex = 0 + if (cursor) { + const cursorIndex = nounFiles.findIndex(file => file > cursor) + if (cursorIndex >= 0) { + startIndex = cursorIndex + } + } + + // Get the subset of files for this page + const pageFiles = nounFiles.slice(startIndex, startIndex + limit) + + // Load nouns from files + const items: HNSWNoun[] = [] + for (const fileName of pageFiles) { + const id = fileName.replace('.json', '') + const noun = await this.getNoun_internal(id) + if (noun) { + // Apply filters if provided + if (options.filter) { + const metadata = await this.getNounMetadata(id) + + // Filter by noun type + if (options.filter.nounType) { + const nounTypes = Array.isArray(options.filter.nounType) + ? options.filter.nounType + : [options.filter.nounType] + if (metadata && !nounTypes.includes(metadata.type || metadata.noun)) { + continue + } + } + + // Filter by service + if (options.filter.service) { + const services = Array.isArray(options.filter.service) + ? options.filter.service + : [options.filter.service] + if (metadata && !services.includes(metadata.createdBy?.augmentation)) { + continue + } + } + + // Filter by metadata + if (options.filter.metadata) { + if (!metadata) continue + let matches = true + for (const [key, value] of Object.entries(options.filter.metadata)) { + if (metadata[key] !== value) { + matches = false + break + } + } + if (!matches) continue + } + } + + items.push(noun) + } + } + + // Determine if there are more items + const hasMore = startIndex + limit < nounFiles.length + + // Generate next cursor if there are more items + const nextCursor = hasMore && pageFiles.length > 0 + ? pageFiles[pageFiles.length - 1] + : undefined + + return { + items, + totalCount: nounFiles.length, + hasMore, + nextCursor + } + } + + /** + * Get verbs with pagination support + * @param options Pagination and filter options + * @returns Promise that resolves to a paginated result of verbs + */ + public async getVerbsWithPagination(options: { + limit?: number + cursor?: string + filter?: { + verbType?: string | string[] + sourceId?: string | string[] + targetId?: string | string[] + service?: string | string[] + metadata?: Record + } + } = {}): Promise<{ + items: GraphVerb[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + await this.ensureInitialized() + + const limit = options.limit || 100 + const cursor = options.cursor + + // Get all verb files + const verbFiles: string[] = [] + if (this.verbsDir) { + for await (const [name, handle] of this.verbsDir.entries()) { + if (handle.kind === 'file' && name.endsWith('.json')) { + verbFiles.push(name) + } + } + } + + // Sort files for consistent ordering + verbFiles.sort() + + // Apply cursor-based pagination + let startIndex = 0 + if (cursor) { + const cursorIndex = verbFiles.findIndex(file => file > cursor) + if (cursorIndex >= 0) { + startIndex = cursorIndex + } + } + + // Get the subset of files for this page + const pageFiles = verbFiles.slice(startIndex, startIndex + limit) + + // Load verbs from files and convert to GraphVerb + const items: GraphVerb[] = [] + for (const fileName of pageFiles) { + const id = fileName.replace('.json', '') + const hnswVerb = await this.getVerb_internal(id) + if (hnswVerb) { + // Convert HNSWVerb to GraphVerb + const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb) + if (graphVerb) { + // Apply filters if provided + if (options.filter) { + // Filter by verb type + if (options.filter.verbType) { + const verbTypes = Array.isArray(options.filter.verbType) + ? options.filter.verbType + : [options.filter.verbType] + if (graphVerb.verb && !verbTypes.includes(graphVerb.verb)) { + continue + } + } + + // Filter by source ID + if (options.filter.sourceId) { + const sourceIds = Array.isArray(options.filter.sourceId) + ? options.filter.sourceId + : [options.filter.sourceId] + if (graphVerb.source && !sourceIds.includes(graphVerb.source)) { + continue + } + } + + // Filter by target ID + if (options.filter.targetId) { + const targetIds = Array.isArray(options.filter.targetId) + ? options.filter.targetId + : [options.filter.targetId] + if (graphVerb.target && !targetIds.includes(graphVerb.target)) { + continue + } + } + + // Filter by service + if (options.filter.service) { + const services = Array.isArray(options.filter.service) + ? options.filter.service + : [options.filter.service] + if (graphVerb.createdBy?.augmentation && !services.includes(graphVerb.createdBy.augmentation)) { + continue + } + } + + // Filter by metadata + if (options.filter.metadata && graphVerb.metadata) { + let matches = true + for (const [key, value] of Object.entries(options.filter.metadata)) { + if (graphVerb.metadata[key] !== value) { + matches = false + break + } + } + if (!matches) continue + } + } + + items.push(graphVerb) + } + } + } + + // Determine if there are more items + const hasMore = startIndex + limit < verbFiles.length + + // Generate next cursor if there are more items + const nextCursor = hasMore && pageFiles.length > 0 + ? pageFiles[pageFiles.length - 1] + : undefined + + return { + items, + totalCount: verbFiles.length, + hasMore, + nextCursor + } + } } diff --git a/tests/mocks/opfs-mock.ts b/tests/mocks/opfs-mock.ts index bafe4a48..fd432e87 100644 --- a/tests/mocks/opfs-mock.ts +++ b/tests/mocks/opfs-mock.ts @@ -138,7 +138,7 @@ export function createMockDirectoryHandle(dirName: string, entries: Map