diff --git a/package-lock.json b/package-lock.json index 11977190..cab898fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "2.14.0", + "version": "2.14.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "2.14.0", + "version": "2.14.1", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index ccaeb7ef..f0e92208 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "2.14.0", + "version": "2.14.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 191dde8b..53964bb9 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -147,6 +147,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { /** * Count total number of nouns (optional) + * WARNING: Implementations should be efficient for large datasets. + * Consider caching counts or using database COUNT operations. * @param filter Optional filter criteria * @returns Promise that resolves to the count */ @@ -154,6 +156,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { /** * Count total number of verbs (optional) + * WARNING: Implementations should be efficient for large datasets. + * Consider caching counts or using database COUNT operations. * @param filter Optional filter criteria * @returns Promise that resolves to the count */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index ac47df98..1250e2bf 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -997,6 +997,8 @@ export class FileSystemStorage extends BaseStorage { try { // List all verb files in the verbs directory + // Note: For very large directories (millions of files), this could be memory-intensive + // Future optimization: Use fs.opendir() for streaming directory reads const files = await fs.promises.readdir(this.verbsDir) const verbFiles = files.filter((f: string) => f.endsWith('.json')) @@ -1008,6 +1010,11 @@ export class FileSystemStorage extends BaseStorage { const endIndex = Math.min(startIndex + limit, totalCount) const hasMore = endIndex < totalCount + // Safety check for large datasets + if (totalCount > 100000) { + console.warn(`Large verb dataset detected (${totalCount} verbs). Consider using a database for better performance.`) + } + // Load the requested page of verbs const verbs: GraphVerb[] = []