diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index dc6f5acc..9351a654 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -548,6 +548,185 @@ const valid = await security.verify(data, signature, secret) --- +## Storage Configuration + +Brainy supports multiple storage backends for different deployment scenarios. + +### Storage Types + +#### Memory Storage (Default) +Fast in-memory storage, ideal for testing and development. + +```typescript +const brain = new Brainy({ + storage: { type: 'memory' } +}) +``` + +#### File System Storage +Persistent local storage using the filesystem. + +```typescript +const brain = new Brainy({ + storage: { + type: 'filesystem', + rootDirectory: './brainy-data' + } +}) +``` + +#### Amazon S3 Storage +Scalable cloud storage with S3. + +```typescript +const brain = new Brainy({ + storage: { + type: 's3', + s3Storage: { + bucketName: 'my-bucket', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + } + } +}) +``` + +#### Cloudflare R2 Storage +Scalable cloud storage with Cloudflare R2 (S3-compatible). + +```typescript +const brain = new Brainy({ + storage: { + type: 'r2', + r2Storage: { + bucketName: 'my-bucket', + accountId: process.env.CF_ACCOUNT_ID, + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY + } + } +}) +``` + +#### Google Cloud Storage (S3-Compatible) +GCS using HMAC keys for S3-compatible access. + +```typescript +const brain = new Brainy({ + storage: { + type: 'gcs', + gcsStorage: { + bucketName: 'my-bucket', + region: 'us-central1', + accessKeyId: process.env.GCS_ACCESS_KEY_ID, + secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY, + endpoint: 'https://storage.googleapis.com' + } + } +}) +``` + +#### Google Cloud Storage (Native SDK) ๐Ÿ†• +**Recommended for GCS deployments.** Uses native `@google-cloud/storage` SDK with automatic authentication. + +**Key Benefits:** +- โœ… Application Default Credentials (ADC) - Zero config in Cloud Run/GCE +- โœ… Better performance with native GCS optimizations +- โœ… No HMAC key management required +- โœ… Automatic service account integration + +**With Application Default Credentials (Cloud Run/GCE):** +```typescript +const brain = new Brainy({ + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket' + // No credentials needed - ADC automatic! + } + } +}) +``` + +**With Service Account Key File:** +```typescript +const brain = new Brainy({ + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket', + keyFilename: '/path/to/service-account.json' + } + } +}) +``` + +**With Service Account Credentials Object:** +```typescript +const brain = new Brainy({ + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket', + credentials: { + client_email: 'service@project.iam.gserviceaccount.com', + private_key: process.env.GCS_PRIVATE_KEY + } + } + } +}) +``` + +### Storage Features + +All storage adapters support: +- โœ… **UUID-based sharding** - 256 buckets (00-ff) for scalability +- โœ… **Pagination** - Efficient cursor-based pagination across shards +- โœ… **Statistics** - O(1) count operations +- โœ… **Caching** - Multi-level cache for performance +- โœ… **Backpressure** - Automatic flow control +- โœ… **Throttling detection** - Adaptive retry on rate limits + +### Migration from HMAC to Native GCS + +If you're currently using `type: 'gcs'` with HMAC keys, migrating to `type: 'gcs-native'` is straightforward: + +**Before (HMAC):** +```typescript +const brain = new Brainy({ + storage: { + type: 'gcs', + gcsStorage: { + bucketName: 'my-bucket', + accessKeyId: process.env.GCS_ACCESS_KEY_ID, + secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY + } + } +}) +``` + +**After (Native with ADC):** +```typescript +const brain = new Brainy({ + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket' + // ADC handles authentication automatically + } + } +}) +``` + +**Data Migration:** +No data migration required! Both adapters use the same path structure: +- `entities/nouns/vectors/{shard}/{id}.json` +- `entities/nouns/metadata/{shard}/{id}.json` +- `entities/verbs/vectors/{shard}/{id}.json` + +--- + ## Configuration API Access via: `const config = await brain.config()` diff --git a/package-lock.json b/package-lock.json index c85036bd..88a3749e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", + "@google-cloud/storage": "^7.14.0", "@huggingface/transformers": "^3.7.2", "boxen": "^8.0.1", "chalk": "^5.3.0", @@ -1689,6 +1690,111 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage": { + "version": "7.17.2", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.17.2.tgz", + "integrity": "sha512-6xN0KNO8L/LIA5zu3CJwHkJiB6n65eykBLOb0E+RooiHYgX8CSao6lvQiKT9TBk2gL5g33LL3fmhDodZnt56rw==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "<4.1.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "duplexify": "^4.1.3", + "fast-xml-parser": "^4.4.1", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "html-entities": "^2.5.2", + "mime": "^3.0.0", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", + "uuid": "^8.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/fast-xml-parser": { + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz", + "integrity": "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^1.1.1" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@google-cloud/storage/node_modules/strnum": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", + "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/@google-cloud/storage/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", @@ -4303,6 +4409,21 @@ "testcontainers": "^11.5.1" } }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, "node_modules/@types/chai": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.2.tgz", @@ -4396,6 +4517,18 @@ "license": "MIT", "optional": true }, + "node_modules/@types/request": { + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.5" + } + }, "node_modules/@types/resolve": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", @@ -4440,6 +4573,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -4871,7 +5010,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dev": true, "license": "MIT", "dependencies": { "event-target-shim": "^5.0.0" @@ -4920,6 +5058,15 @@ "node": ">=0.8" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -5192,6 +5339,30 @@ "dev": true, "license": "MIT" }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/async-retry/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -5330,7 +5501,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -5357,6 +5527,15 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -5556,6 +5735,12 @@ "node": ">=8.0.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -5616,7 +5801,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -6027,6 +6211,18 @@ "simple-swizzle": "^0.2.2" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", @@ -6524,7 +6720,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -6647,6 +6842,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/detect-indent": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", @@ -6913,7 +7117,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -6924,6 +7127,18 @@ "node": ">= 0.4" } }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -6931,6 +7146,15 @@ "dev": true, "license": "MIT" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -6942,7 +7166,6 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -6994,7 +7217,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -7003,6 +7225,21 @@ "node": ">= 0.4" } }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es6-error": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", @@ -7346,7 +7583,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -7379,6 +7615,12 @@ "node": ">=12.0.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -7647,6 +7889,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/form-data": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", + "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.35", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/frac": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", @@ -7682,12 +7941,41 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -7714,7 +8002,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -7815,7 +8102,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -7991,6 +8277,32 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -8017,6 +8329,19 @@ "dev": true, "license": "MIT" }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/guid-typescript": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", @@ -8081,7 +8406,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -8094,7 +8418,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -8110,7 +8433,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -8152,6 +8474,22 @@ "dev": true, "license": "ISC" }, + "node_modules/html-entities": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -8174,6 +8512,45 @@ "node": ">=8.0.0" } }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -8260,7 +8637,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, "node_modules/ini": { @@ -8498,7 +8874,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -8653,6 +9028,15 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -8742,6 +9126,27 @@ "html2canvas": "^1.0.0-rc.5" } }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -9026,7 +9431,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -9267,11 +9671,22 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -9281,7 +9696,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "license": "MIT", "dependencies": { "mime-db": "1.52.0" @@ -9468,7 +9882,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/mute-stream": { @@ -9521,6 +9934,26 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/normalize-package-data": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", @@ -9560,7 +9993,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -9693,7 +10125,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" @@ -10307,7 +10738,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -10441,6 +10871,20 @@ "node": ">= 4" } }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -10566,7 +11010,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -11097,6 +11540,15 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, "node_modules/stream-json": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", @@ -11107,6 +11559,12 @@ "stream-chain": "^2.2.5" } }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, "node_modules/streamx": { "version": "2.22.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.1.tgz", @@ -11135,7 +11593,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -11314,6 +11771,12 @@ ], "license": "MIT" }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -11395,6 +11858,47 @@ "streamx": "^2.15.0" } }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/terser": { "version": "5.43.1", "resolved": "https://registry.npmjs.org/terser/-/terser-5.43.1.tgz", @@ -11592,6 +12096,12 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/trim-newlines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz", @@ -11754,7 +12264,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/utrie": { @@ -11976,6 +12485,22 @@ "@zxing/text-encoding": "0.9.0" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -12207,7 +12732,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/ws": { @@ -12396,7 +12920,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" diff --git a/package.json b/package.json index c2df075a..b41dc67e 100644 --- a/package.json +++ b/package.json @@ -157,6 +157,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.540.0", + "@google-cloud/storage": "^7.14.0", "@huggingface/transformers": "^3.7.2", "boxen": "^8.0.1", "chalk": "^5.3.0", diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts new file mode 100644 index 00000000..00f9e38d --- /dev/null +++ b/src/storage/adapters/gcsStorage.ts @@ -0,0 +1,1533 @@ +/** + * Google Cloud Storage Adapter (Native) + * Uses the native @google-cloud/storage library for optimal performance and authentication + * + * Supports multiple authentication methods: + * 1. Application Default Credentials (ADC) - Automatic in Cloud Run/GCE + * 2. Service Account Key File + * 3. Service Account Credentials Object + * 4. HMAC Keys (fallback for backward compatibility) + */ + +import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js' +import { + BaseStorage, + NOUNS_DIR, + VERBS_DIR, + METADATA_DIR, + INDEX_DIR, + SYSTEM_DIR, + STATISTICS_KEY, + getDirectoryPath +} from '../baseStorage.js' +import { BrainyError } from '../../errors/brainyError.js' +import { CacheManager } from '../cacheManager.js' +import { createModuleLogger, prodLog } from '../../utils/logger.js' +import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js' +import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js' +import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js' +import { getCoalescer, RequestCoalescer } from '../../utils/requestCoalescer.js' +import { getShardIdFromUuid, getAllShardIds, getShardIdByIndex, TOTAL_SHARDS } from '../sharding.js' + +// Type aliases for better readability +type HNSWNode = HNSWNoun +type Edge = HNSWVerb + +// GCS client types - dynamically imported to avoid issues in browser environments +type Storage = any +type Bucket = any +type File = any + +/** + * Native Google Cloud Storage adapter for server environments + * Uses the @google-cloud/storage library with Application Default Credentials + * + * Authentication priority: + * 1. Application Default Credentials (if no credentials provided) + * 2. Service Account Key File (if keyFilename provided) + * 3. Service Account Credentials Object (if credentials provided) + * 4. HMAC Keys (if accessKeyId/secretAccessKey provided) + */ +export class GcsStorage extends BaseStorage { + private storage: Storage | null = null + private bucket: Bucket | null = null + private bucketName: string + private keyFilename?: string + private credentials?: object + private accessKeyId?: string + private secretAccessKey?: string + + // Prefixes for different types of data + private nounPrefix: string + private verbPrefix: string + private metadataPrefix: string // Noun metadata + private verbMetadataPrefix: string // Verb metadata + private systemPrefix: string // System data (_system) + + // Statistics caching for better performance + protected statisticsCache: StatisticsData | null = null + + // Backpressure and performance management + private pendingOperations: number = 0 + private maxConcurrentOperations: number = 100 + private baseBatchSize: number = 10 + private currentBatchSize: number = 10 + private lastMemoryCheck: number = 0 + private memoryCheckInterval: number = 5000 // Check every 5 seconds + private consecutiveErrors: number = 0 + private lastErrorReset: number = Date.now() + + // Adaptive backpressure for automatic flow control + private backpressure = getGlobalBackpressure() + + // Write buffers for bulk operations + private nounWriteBuffer: WriteBuffer | null = null + private verbWriteBuffer: WriteBuffer | null = null + + // Request coalescer for deduplication + private requestCoalescer: RequestCoalescer | null = null + + // High-volume mode detection - MUCH more aggressive + private highVolumeMode = false + private lastVolumeCheck = 0 + private volumeCheckInterval = 1000 // Check every second, not 5 + private forceHighVolumeMode = false // Environment variable override + + // Multi-level cache manager for efficient data access + private nounCacheManager: CacheManager + private verbCacheManager: CacheManager + + // Module logger + private logger = createModuleLogger('GcsStorage') + + /** + * Initialize the storage adapter + * @param options Configuration options for Google Cloud Storage + */ + constructor(options: { + bucketName: string + + // Service account authentication + keyFilename?: string + credentials?: object + + // HMAC authentication (backward compatibility) + accessKeyId?: string + secretAccessKey?: string + + // Cache and operation configuration + cacheConfig?: { + hotCacheMaxSize?: number + hotCacheEvictionThreshold?: number + warmCacheTTL?: number + } + readOnly?: boolean + }) { + super() + this.bucketName = options.bucketName + this.keyFilename = options.keyFilename + this.credentials = options.credentials + this.accessKeyId = options.accessKeyId + this.secretAccessKey = options.secretAccessKey + this.readOnly = options.readOnly || false + + // Set up prefixes for different types of data using entity-based structure + this.nounPrefix = `${getDirectoryPath('noun', 'vector')}/` + this.verbPrefix = `${getDirectoryPath('verb', 'vector')}/` + this.metadataPrefix = `${getDirectoryPath('noun', 'metadata')}/` // Noun metadata + this.verbMetadataPrefix = `${getDirectoryPath('verb', 'metadata')}/` // Verb metadata + this.systemPrefix = `${SYSTEM_DIR}/` // System data + + // Initialize cache managers + this.nounCacheManager = new CacheManager(options.cacheConfig) + this.verbCacheManager = new CacheManager(options.cacheConfig) + + // Check for high-volume mode override + if (typeof process !== 'undefined' && process.env?.BRAINY_FORCE_HIGH_VOLUME === 'true') { + this.forceHighVolumeMode = true + this.highVolumeMode = true + prodLog.info('๐Ÿš€ High-volume mode FORCED via BRAINY_FORCE_HIGH_VOLUME environment variable') + } + } + + /** + * Initialize the storage adapter + */ + public async init(): Promise { + if (this.isInitialized) { + return + } + + try { + // Import Google Cloud Storage SDK only when needed + const { Storage } = await import('@google-cloud/storage') + + // Configure the GCS client based on available credentials + const clientConfig: any = {} + + // Priority 1: Service Account Key File + if (this.keyFilename) { + clientConfig.keyFilename = this.keyFilename + prodLog.info('๐Ÿ” GCS: Using Service Account Key File') + } + // Priority 2: Service Account Credentials Object + else if (this.credentials) { + clientConfig.credentials = this.credentials + prodLog.info('๐Ÿ” GCS: Using Service Account Credentials') + } + // Priority 3: HMAC Keys (S3 compatibility) + else if (this.accessKeyId && this.secretAccessKey) { + clientConfig.credentials = { + client_email: 'hmac-user@example.com', + private_key: this.secretAccessKey + } + prodLog.warn('โš ๏ธ GCS: Using HMAC keys (consider migrating to ADC)') + } + // Priority 4: Application Default Credentials (default) + else { + // No credentials needed - ADC will be used automatically + prodLog.info('๐Ÿ” GCS: Using Application Default Credentials (ADC)') + } + + // Create the GCS client + this.storage = new Storage(clientConfig) + + // Get reference to the bucket + this.bucket = this.storage.bucket(this.bucketName) + + // Verify bucket exists and is accessible + const [exists] = await this.bucket.exists() + if (!exists) { + throw new Error(`Bucket ${this.bucketName} does not exist or is not accessible`) + } + + prodLog.info(`โœ… Connected to GCS bucket: ${this.bucketName}`) + + // Initialize write buffers for high-volume mode + const storageId = `gcs-${this.bucketName}` + this.nounWriteBuffer = getWriteBuffer( + `${storageId}-nouns`, + 'noun', + async (items) => { + await this.flushNounBuffer(items) + } + ) + + this.verbWriteBuffer = getWriteBuffer( + `${storageId}-verbs`, + 'verb', + async (items) => { + await this.flushVerbBuffer(items) + } + ) + + // Initialize request coalescer for deduplication + this.requestCoalescer = getCoalescer( + storageId, + async (batch) => { + // Process coalesced operations (placeholder for future optimization) + this.logger.trace(`Processing coalesced batch: ${batch.length} items`) + } + ) + + // Initialize counts from storage + await this.initializeCounts() + + this.isInitialized = true + } catch (error) { + this.logger.error('Failed to initialize GCS storage:', error) + throw new Error(`Failed to initialize GCS storage: ${error}`) + } + } + + /** + * Get the GCS object key for a noun using UUID-based sharding + * + * Uses first 2 hex characters of UUID for consistent sharding. + * Path format: entities/nouns/vectors/{shardId}/{uuid}.json + * + * @example + * getNounKey('ab123456-1234-5678-9abc-def012345678') + * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json' + */ + private getNounKey(id: string): string { + const shardId = getShardIdFromUuid(id) + return `${this.nounPrefix}${shardId}/${id}.json` + } + + /** + * Get the GCS object key for a verb using UUID-based sharding + * + * Uses first 2 hex characters of UUID for consistent sharding. + * Path format: entities/verbs/vectors/{shardId}/{uuid}.json + * + * @example + * getVerbKey('cd987654-4321-8765-cba9-fed543210987') + * // returns 'entities/verbs/vectors/cd/cd987654-4321-8765-cba9-fed543210987.json' + */ + private getVerbKey(id: string): string { + const shardId = getShardIdFromUuid(id) + return `${this.verbPrefix}${shardId}/${id}.json` + } + + /** + * Override base class method to detect GCS-specific throttling errors + */ + protected isThrottlingError(error: any): boolean { + // First check base class detection + if (super.isThrottlingError(error)) { + return true + } + + // GCS-specific throttling detection + const statusCode = error.code + const message = error.message?.toLowerCase() || '' + + return ( + statusCode === 429 || // Too Many Requests + statusCode === 503 || // Service Unavailable + statusCode === 'RATE_LIMIT_EXCEEDED' || + message.includes('quota') || + message.includes('rate limit') || + message.includes('too many requests') + ) + } + + /** + * Apply backpressure before starting an operation + * @returns Request ID for tracking + */ + private async applyBackpressure(): Promise { + const requestId = `${Date.now()}-${Math.random().toString(36).substr(2, 9)}` + await this.backpressure.requestPermission(requestId, 1) + this.pendingOperations++ + return requestId + } + + /** + * Release backpressure after completing an operation + * @param success Whether the operation succeeded + * @param requestId Request ID from applyBackpressure() + */ + private releaseBackpressure(success: boolean = true, requestId?: string): void { + this.pendingOperations = Math.max(0, this.pendingOperations - 1) + if (requestId) { + this.backpressure.releasePermission(requestId, success) + } + } + + /** + * Check if high-volume mode should be enabled + */ + private checkVolumeMode(): void { + if (this.forceHighVolumeMode) { + return // Already forced on + } + + const now = Date.now() + if (now - this.lastVolumeCheck < this.volumeCheckInterval) { + return + } + + this.lastVolumeCheck = now + + // Enable high-volume mode if we have many pending operations + const shouldEnable = this.pendingOperations > 20 + + if (shouldEnable && !this.highVolumeMode) { + this.highVolumeMode = true + prodLog.info('๐Ÿš€ High-volume mode ENABLED (pending operations:', this.pendingOperations, ')') + } else if (!shouldEnable && this.highVolumeMode && !this.forceHighVolumeMode) { + this.highVolumeMode = false + prodLog.info('๐ŸŒ High-volume mode DISABLED (pending operations:', this.pendingOperations, ')') + } + } + + /** + * Flush noun buffer to GCS + */ + private async flushNounBuffer(items: Map): Promise { + const writes = Array.from(items.values()).map(async (noun) => { + try { + await this.saveNodeDirect(noun) + } catch (error) { + this.logger.error(`Failed to flush noun ${noun.id}:`, error) + } + }) + + await Promise.all(writes) + } + + /** + * Flush verb buffer to GCS + */ + private async flushVerbBuffer(items: Map): Promise { + const writes = Array.from(items.values()).map(async (verb) => { + try { + await this.saveEdgeDirect(verb) + } catch (error) { + this.logger.error(`Failed to flush verb ${verb.id}:`, error) + } + }) + + await Promise.all(writes) + } + + /** + * Save a noun to storage (internal implementation) + */ + protected async saveNoun_internal(noun: HNSWNoun): Promise { + return this.saveNode(noun) + } + + /** + * Save a node to storage + */ + protected async saveNode(node: HNSWNode): Promise { + await this.ensureInitialized() + + // ALWAYS check if we should use high-volume mode (critical for detection) + this.checkVolumeMode() + + // Use write buffer in high-volume mode + if (this.highVolumeMode && this.nounWriteBuffer) { + this.logger.trace(`๐Ÿ“ BUFFERING: Adding noun ${node.id} to write buffer (high-volume mode active)`) + await this.nounWriteBuffer.add(node.id, node) + return + } else if (!this.highVolumeMode) { + this.logger.trace(`๐Ÿ“ DIRECT WRITE: Saving noun ${node.id} directly (high-volume mode inactive)`) + } + + // Direct write in normal mode + await this.saveNodeDirect(node) + } + + /** + * Save a node directly to GCS (bypass buffer) + */ + private async saveNodeDirect(node: HNSWNode): Promise { + // Apply backpressure before starting operation + const requestId = await this.applyBackpressure() + + try { + this.logger.trace(`Saving node ${node.id}`) + + // Convert connections Map to a serializable format + const serializableNode = { + ...node, + connections: Object.fromEntries( + Array.from(node.connections.entries()).map(([level, nounIds]) => [ + level, + Array.from(nounIds) + ]) + ) + } + + // Get the GCS key with UUID-based sharding + const key = this.getNounKey(node.id) + + // Save to GCS + const file = this.bucket!.file(key) + await file.save(JSON.stringify(serializableNode, null, 2), { + contentType: 'application/json', + resumable: false // For small objects, non-resumable is faster + }) + + // Update cache + this.nounCacheManager.set(node.id, node) + + // Increment noun count + const metadata = await this.getNounMetadata(node.id) + if (metadata && metadata.type) { + await this.incrementEntityCountSafe(metadata.type) + } + + this.logger.trace(`Node ${node.id} saved successfully`) + this.releaseBackpressure(true, requestId) + } catch (error: any) { + this.releaseBackpressure(false, requestId) + + // Handle throttling + if (this.isThrottlingError(error)) { + await this.handleThrottling(error) + throw error // Re-throw for retry at higher level + } + + this.logger.error(`Failed to save node ${node.id}:`, error) + throw new Error(`Failed to save node ${node.id}: ${error}`) + } + } + + /** + * Get a noun from storage (internal implementation) + */ + protected async getNoun_internal(id: string): Promise { + return this.getNode(id) + } + + /** + * Get a node from storage + */ + protected async getNode(id: string): Promise { + await this.ensureInitialized() + + // Check cache first + const cached = this.nounCacheManager.get(id) + if (cached) { + this.logger.trace(`Cache hit for noun ${id}`) + return cached + } + + // Apply backpressure + const requestId = await this.applyBackpressure() + + try { + this.logger.trace(`Getting node ${id}`) + + // Get the GCS key with UUID-based sharding + const key = this.getNounKey(id) + + // Download from GCS + const file = this.bucket!.file(key) + const [contents] = await file.download() + + // Parse JSON + const data = JSON.parse(contents.toString()) + + // Convert serialized connections back to Map> + const connections = new Map>() + for (const [level, nounIds] of Object.entries(data.connections || {})) { + connections.set(Number(level), new Set(nounIds as string[])) + } + + const node: HNSWNode = { + id: data.id, + vector: data.vector, + connections, + level: data.level || 0 + } + + // Update cache + this.nounCacheManager.set(id, node) + + this.logger.trace(`Successfully retrieved node ${id}`) + this.releaseBackpressure(true, requestId) + return node + } catch (error: any) { + this.releaseBackpressure(false, requestId) + + // Check if this is a "not found" error + if (error.code === 404) { + this.logger.trace(`Node not found: ${id}`) + return null + } + + // Handle throttling + if (this.isThrottlingError(error)) { + await this.handleThrottling(error) + throw error + } + + this.logger.error(`Failed to get node ${id}:`, error) + throw BrainyError.fromError(error, `getNoun(${id})`) + } + } + + /** + * Delete a noun from storage (internal implementation) + */ + protected async deleteNoun_internal(id: string): Promise { + await this.ensureInitialized() + + const requestId = await this.applyBackpressure() + + try { + this.logger.trace(`Deleting noun ${id}`) + + // Get the GCS key + const key = this.getNounKey(id) + + // Delete from GCS + const file = this.bucket!.file(key) + await file.delete() + + // Remove from cache + this.nounCacheManager.delete(id) + + // Decrement noun count + const metadata = await this.getNounMetadata(id) + if (metadata && metadata.type) { + await this.decrementEntityCountSafe(metadata.type) + } + + this.logger.trace(`Noun ${id} deleted successfully`) + this.releaseBackpressure(true, requestId) + } catch (error: any) { + this.releaseBackpressure(false, requestId) + + if (error.code === 404) { + // Already deleted + this.logger.trace(`Noun ${id} not found (already deleted)`) + return + } + + // Handle throttling + if (this.isThrottlingError(error)) { + await this.handleThrottling(error) + throw error + } + + this.logger.error(`Failed to delete noun ${id}:`, error) + throw new Error(`Failed to delete noun ${id}: ${error}`) + } + } + + /** + * Save noun metadata to storage (internal implementation) + */ + protected async saveNounMetadata_internal(id: string, metadata: any): Promise { + await this.ensureInitialized() + + try { + // Use UUID-based sharding for metadata (consistent with noun vectors) + const shardId = getShardIdFromUuid(id) + const key = `${this.metadataPrefix}${shardId}/${id}.json` + + this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`) + + // Save to GCS + const file = this.bucket!.file(key) + await file.save(JSON.stringify(metadata, null, 2), { + contentType: 'application/json', + resumable: false + }) + + this.logger.debug(`Noun metadata for ${id} saved successfully`) + } catch (error) { + this.logger.error(`Failed to save noun metadata for ${id}:`, error) + throw new Error(`Failed to save noun metadata for ${id}: ${error}`) + } + } + + /** + * Save metadata to storage (public API - delegates to saveNounMetadata_internal) + */ + public async saveMetadata(id: string, metadata: any): Promise { + return this.saveNounMetadata_internal(id, metadata) + } + + /** + * Get metadata from storage (public API - delegates to getNounMetadata) + */ + public async getMetadata(id: string): Promise { + return this.getNounMetadata(id) + } + + /** + * Get noun metadata from storage + */ + public async getNounMetadata(id: string): Promise { + await this.ensureInitialized() + + try { + // Use UUID-based sharding for metadata + const shardId = getShardIdFromUuid(id) + const key = `${this.metadataPrefix}${shardId}/${id}.json` + + this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`) + + // Download from GCS + const file = this.bucket!.file(key) + const [contents] = await file.download() + + // Parse JSON + const metadata = JSON.parse(contents.toString()) + + this.logger.trace(`Successfully retrieved noun metadata for ${id}`) + return metadata + } catch (error: any) { + // Check if this is a "not found" error + if (error.code === 404) { + this.logger.trace(`Noun metadata not found for ${id}`) + return null + } + + // For other types of errors, convert to BrainyError + throw BrainyError.fromError(error, `getNounMetadata(${id})`) + } + } + + /** + * Save verb metadata to storage (internal implementation) + */ + protected async saveVerbMetadata_internal(id: string, metadata: any): Promise { + await this.ensureInitialized() + + try { + const key = `${this.verbMetadataPrefix}${id}.json` + + this.logger.trace(`Saving verb metadata for ${id} to key: ${key}`) + + // Save to GCS + const file = this.bucket!.file(key) + await file.save(JSON.stringify(metadata, null, 2), { + contentType: 'application/json', + resumable: false + }) + + this.logger.debug(`Verb metadata for ${id} saved successfully`) + } catch (error) { + this.logger.error(`Failed to save verb metadata for ${id}:`, error) + throw new Error(`Failed to save verb metadata for ${id}: ${error}`) + } + } + + /** + * Get verb metadata from storage + */ + public async getVerbMetadata(id: string): Promise { + await this.ensureInitialized() + + try { + const key = `${this.verbMetadataPrefix}${id}.json` + + this.logger.trace(`Getting verb metadata for ${id} from key: ${key}`) + + // Download from GCS + const file = this.bucket!.file(key) + const [contents] = await file.download() + + // Parse JSON + const metadata = JSON.parse(contents.toString()) + + this.logger.trace(`Successfully retrieved verb metadata for ${id}`) + return metadata + } catch (error: any) { + // Check if this is a "not found" error + if (error.code === 404) { + this.logger.trace(`Verb metadata not found for ${id}`) + return null + } + + // For other types of errors, convert to BrainyError + throw BrainyError.fromError(error, `getVerbMetadata(${id})`) + } + } + + /** + * Save a verb to storage (internal implementation) + */ + protected async saveVerb_internal(verb: HNSWVerb): Promise { + return this.saveEdge(verb) + } + + /** + * Save an edge to storage + */ + protected async saveEdge(edge: Edge): Promise { + await this.ensureInitialized() + + // Check volume mode + this.checkVolumeMode() + + // Use write buffer in high-volume mode + if (this.highVolumeMode && this.verbWriteBuffer) { + this.logger.trace(`๐Ÿ“ BUFFERING: Adding verb ${edge.id} to write buffer`) + await this.verbWriteBuffer.add(edge.id, edge) + return + } + + // Direct write in normal mode + await this.saveEdgeDirect(edge) + } + + /** + * Save an edge directly to GCS (bypass buffer) + */ + private async saveEdgeDirect(edge: Edge): Promise { + const requestId = await this.applyBackpressure() + + try { + this.logger.trace(`Saving edge ${edge.id}`) + + // Convert connections Map to serializable format + const serializableEdge = { + ...edge, + connections: Object.fromEntries( + Array.from(edge.connections.entries()).map(([level, verbIds]) => [ + level, + Array.from(verbIds) + ]) + ) + } + + // Get the GCS key with UUID-based sharding + const key = this.getVerbKey(edge.id) + + // Save to GCS + const file = this.bucket!.file(key) + await file.save(JSON.stringify(serializableEdge, null, 2), { + contentType: 'application/json', + resumable: false + }) + + // Update cache + this.verbCacheManager.set(edge.id, edge) + + // Increment verb count + const metadata = await this.getVerbMetadata(edge.id) + if (metadata && metadata.type) { + await this.incrementVerbCount(metadata.type) + } + + this.logger.trace(`Edge ${edge.id} saved successfully`) + this.releaseBackpressure(true, requestId) + } catch (error: any) { + this.releaseBackpressure(false, requestId) + + if (this.isThrottlingError(error)) { + await this.handleThrottling(error) + throw error + } + + this.logger.error(`Failed to save edge ${edge.id}:`, error) + throw new Error(`Failed to save edge ${edge.id}: ${error}`) + } + } + + /** + * Get a verb from storage (internal implementation) + */ + protected async getVerb_internal(id: string): Promise { + return this.getEdge(id) + } + + /** + * Get an edge from storage + */ + protected async getEdge(id: string): Promise { + await this.ensureInitialized() + + // Check cache first + const cached = this.verbCacheManager.get(id) + if (cached) { + this.logger.trace(`Cache hit for verb ${id}`) + return cached + } + + const requestId = await this.applyBackpressure() + + try { + this.logger.trace(`Getting edge ${id}`) + + // Get the GCS key with UUID-based sharding + const key = this.getVerbKey(id) + + // Download from GCS + const file = this.bucket!.file(key) + const [contents] = await file.download() + + // Parse JSON + const data = JSON.parse(contents.toString()) + + // Convert serialized connections back to Map + const connections = new Map>() + for (const [level, verbIds] of Object.entries(data.connections || {})) { + connections.set(Number(level), new Set(verbIds as string[])) + } + + const edge: Edge = { + id: data.id, + vector: data.vector, + connections + } + + // Update cache + this.verbCacheManager.set(id, edge) + + this.logger.trace(`Successfully retrieved edge ${id}`) + this.releaseBackpressure(true, requestId) + return edge + } catch (error: any) { + this.releaseBackpressure(false, requestId) + + // Check if this is a "not found" error + if (error.code === 404) { + this.logger.trace(`Edge not found: ${id}`) + return null + } + + if (this.isThrottlingError(error)) { + await this.handleThrottling(error) + throw error + } + + this.logger.error(`Failed to get edge ${id}:`, error) + throw BrainyError.fromError(error, `getVerb(${id})`) + } + } + + /** + * Delete a verb from storage (internal implementation) + */ + protected async deleteVerb_internal(id: string): Promise { + await this.ensureInitialized() + + const requestId = await this.applyBackpressure() + + try { + this.logger.trace(`Deleting verb ${id}`) + + // Get the GCS key + const key = this.getVerbKey(id) + + // Delete from GCS + const file = this.bucket!.file(key) + await file.delete() + + // Remove from cache + this.verbCacheManager.delete(id) + + // Decrement verb count + const metadata = await this.getVerbMetadata(id) + if (metadata && metadata.type) { + await this.decrementVerbCount(metadata.type) + } + + this.logger.trace(`Verb ${id} deleted successfully`) + this.releaseBackpressure(true, requestId) + } catch (error: any) { + this.releaseBackpressure(false, requestId) + + if (error.code === 404) { + // Already deleted + this.logger.trace(`Verb ${id} not found (already deleted)`) + return + } + + if (this.isThrottlingError(error)) { + await this.handleThrottling(error) + throw error + } + + this.logger.error(`Failed to delete verb ${id}:`, error) + throw new Error(`Failed to delete verb ${id}: ${error}`) + } + } + + /** + * Get nouns with pagination + * Iterates through all UUID-based shards (00-ff) for consistent pagination + */ + public async getNounsWithPagination(options: { + limit?: number + cursor?: string + filter?: { + nounType?: string | string[] + service?: string | string[] + metadata?: Record + } + } = {}): Promise<{ + items: HNSWNoun[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + await this.ensureInitialized() + + const limit = options.limit || 100 + const cursor = options.cursor + + // Get paginated nodes + const result = await this.getNodesWithPagination({ + limit, + cursor, + useCache: true + }) + + // Apply filters if provided + let filteredNodes = result.nodes + + if (options.filter) { + // Filter by noun type + if (options.filter.nounType) { + const nounTypes = Array.isArray(options.filter.nounType) + ? options.filter.nounType + : [options.filter.nounType] + + const filteredByType: HNSWNoun[] = [] + for (const node of filteredNodes) { + const metadata = await this.getNounMetadata(node.id) + if (metadata && nounTypes.includes(metadata.type || metadata.noun)) { + filteredByType.push(node) + } + } + filteredNodes = filteredByType + } + + // Additional filter logic can be added here + } + + return { + items: filteredNodes, + totalCount: result.totalCount, + hasMore: result.hasMore, + nextCursor: result.nextCursor + } + } + + /** + * Get nodes with pagination (internal implementation) + * Iterates through UUID-based shards for consistent pagination + */ + private async getNodesWithPagination(options: { + limit: number + cursor?: string + useCache?: boolean + }): Promise<{ + nodes: HNSWNode[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + const limit = options.limit || 100 + const useCache = options.useCache !== false + + try { + const nodes: HNSWNode[] = [] + + // Parse cursor (format: "shardIndex:gcsPageToken") + let startShardIndex = 0 + let gcsPageToken: string | undefined + if (options.cursor) { + const parts = options.cursor.split(':', 2) + startShardIndex = parseInt(parts[0]) || 0 + gcsPageToken = parts[1] || undefined + } + + // Iterate through shards starting from cursor position + for (let shardIndex = startShardIndex; shardIndex < TOTAL_SHARDS; shardIndex++) { + const shardId = getShardIdByIndex(shardIndex) + const shardPrefix = `${this.nounPrefix}${shardId}/` + + // List objects in this shard + const [files, , response] = await this.bucket!.getFiles({ + prefix: shardPrefix, + maxResults: limit - nodes.length, + pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined + }) + + // Extract node IDs from file names + if (files && files.length > 0) { + const nodeIds = files + .filter((file: any) => file && file.name) + .map((file: any) => { + // Extract UUID from: entities/nouns/vectors/ab/ab123456-uuid.json + let name = file.name! + if (name.startsWith(shardPrefix)) { + name = name.substring(shardPrefix.length) + } + if (name.endsWith('.json')) { + name = name.substring(0, name.length - 5) + } + return name + }) + .filter((id: string) => id && id.length > 0) + + // Load nodes + for (const id of nodeIds) { + const node = await this.getNode(id) + if (node) { + nodes.push(node) + } + + if (nodes.length >= limit) { + break + } + } + } + + // Check if we have enough nodes or if there are more files in current shard + if (nodes.length >= limit) { + const nextCursor = response?.nextPageToken + ? `${shardIndex}:${response.nextPageToken}` + : shardIndex + 1 < TOTAL_SHARDS + ? `${shardIndex + 1}:` + : undefined + + return { + nodes, + hasMore: !!nextCursor, + nextCursor + } + } + + // If this shard has more pages, create cursor for next page + if (response?.nextPageToken) { + return { + nodes, + hasMore: true, + nextCursor: `${shardIndex}:${response.nextPageToken}` + } + } + + // Continue to next shard + } + + // No more shards or nodes + return { + nodes, + hasMore: false, + nextCursor: undefined + } + } catch (error) { + this.logger.error('Error in getNodesWithPagination:', error) + throw new Error(`Failed to get nodes with pagination: ${error}`) + } + } + + /** + * Get nouns by noun type (internal implementation) + */ + protected async getNounsByNounType_internal(nounType: string): Promise { + const result = await this.getNounsWithPagination({ + limit: 10000, // Large limit for backward compatibility + filter: { nounType } + }) + + return result.items + } + + /** + * Get verbs by source ID (internal implementation) + */ + protected async getVerbsBySource_internal(sourceId: string): Promise { + // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion + const result = await this.getVerbsWithPagination({ + limit: Number.MAX_SAFE_INTEGER, + filter: { sourceId: [sourceId] } + }) + + return result.items + } + + /** + * Get verbs by target ID (internal implementation) + */ + protected async getVerbsByTarget_internal(targetId: string): Promise { + // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion + const result = await this.getVerbsWithPagination({ + limit: Number.MAX_SAFE_INTEGER, + filter: { targetId: [targetId] } + }) + + return result.items + } + + /** + * Get verbs by type (internal implementation) + */ + protected async getVerbsByType_internal(type: string): Promise { + // Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion + const result = await this.getVerbsWithPagination({ + limit: Number.MAX_SAFE_INTEGER, + filter: { verbType: type } + }) + + return result.items + } + + /** + * Get verbs with pagination + */ + public async getVerbsWithPagination(options: { + limit?: number + cursor?: string + filter?: { + verbType?: string | string[] + sourceId?: string | string[] + targetId?: string | string[] + service?: string | string[] + metadata?: Record + } + } = {}): Promise<{ + items: GraphVerb[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + await this.ensureInitialized() + + const limit = options.limit || 100 + + try { + // List verbs (simplified - not sharded yet in original implementation) + const [files, , response] = await this.bucket!.getFiles({ + prefix: this.verbPrefix, + maxResults: limit, + pageToken: options.cursor + }) + + // If no files, return empty result + if (!files || files.length === 0) { + return { + items: [], + totalCount: 0, + hasMore: false, + nextCursor: undefined + } + } + + // Extract verb IDs and load verbs as HNSW verbs + const hnswVerbs: HNSWVerb[] = [] + for (const file of files) { + if (!file.name) continue + + // Extract UUID from path + let name = file.name + if (name.startsWith(this.verbPrefix)) { + name = name.substring(this.verbPrefix.length) + } + if (name.endsWith('.json')) { + name = name.substring(0, name.length - 5) + } + + const verb = await this.getEdge(name) + if (verb) { + hnswVerbs.push(verb) + } + } + + // Convert HNSWVerbs to GraphVerbs by combining with metadata + const graphVerbs: GraphVerb[] = [] + for (const hnswVerb of hnswVerbs) { + const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb) + if (graphVerb) { + graphVerbs.push(graphVerb) + } + } + + // Apply filters + let filteredVerbs = graphVerbs + if (options.filter) { + filteredVerbs = graphVerbs.filter((graphVerb) => { + // Filter by sourceId + if (options.filter!.sourceId) { + const sourceIds = Array.isArray(options.filter!.sourceId) + ? options.filter!.sourceId + : [options.filter!.sourceId] + if (!sourceIds.includes(graphVerb.sourceId)) { + return false + } + } + + // Filter by targetId + if (options.filter!.targetId) { + const targetIds = Array.isArray(options.filter!.targetId) + ? options.filter!.targetId + : [options.filter!.targetId] + if (!targetIds.includes(graphVerb.targetId)) { + return false + } + } + + // Filter by verbType + if (options.filter!.verbType) { + const verbTypes = Array.isArray(options.filter!.verbType) + ? options.filter!.verbType + : [options.filter!.verbType] + const verbType = graphVerb.verb || graphVerb.type || '' + if (!verbTypes.includes(verbType)) { + return false + } + } + + return true + }) + } + + return { + items: filteredVerbs, + hasMore: !!response?.nextPageToken, + nextCursor: response?.nextPageToken + } + } catch (error) { + this.logger.error('Error in getVerbsWithPagination:', error) + throw new Error(`Failed to get verbs with pagination: ${error}`) + } + } + + /** + * Get nouns with filtering and pagination (public API) + */ + public async getNouns(options?: { + pagination?: { + offset?: number + limit?: number + cursor?: string + } + filter?: { + nounType?: string | string[] + service?: string | string[] + metadata?: Record + } + }): Promise<{ + items: any[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + const limit = options?.pagination?.limit || 100 + const cursor = options?.pagination?.cursor + + return this.getNounsWithPagination({ + limit, + cursor, + filter: options?.filter + }) + } + + /** + * Get verbs with filtering and pagination (public API) + */ + public async getVerbs(options?: { + pagination?: { + offset?: number + limit?: number + cursor?: string + } + filter?: { + verbType?: string | string[] + sourceId?: string | string[] + targetId?: string | string[] + service?: string | string[] + metadata?: Record + } + }): Promise<{ + items: GraphVerb[] + totalCount?: number + hasMore: boolean + nextCursor?: string + }> { + const limit = options?.pagination?.limit || 100 + const cursor = options?.pagination?.cursor + + return this.getVerbsWithPagination({ + limit, + cursor, + filter: options?.filter + }) + } + + /** + * Clear all data from storage + */ + public async clear(): Promise { + await this.ensureInitialized() + + try { + this.logger.info('๐Ÿงน Clearing all data from GCS bucket...') + + // Helper function to delete all objects with a given prefix + const deleteObjectsWithPrefix = async (prefix: string): Promise => { + const [files] = await this.bucket!.getFiles({ prefix }) + + if (!files || files.length === 0) { + return + } + + // Delete each file + for (const file of files) { + await file.delete() + } + } + + // Clear all data directories + await deleteObjectsWithPrefix(this.nounPrefix) + await deleteObjectsWithPrefix(this.verbPrefix) + await deleteObjectsWithPrefix(this.metadataPrefix) + await deleteObjectsWithPrefix(this.verbMetadataPrefix) + await deleteObjectsWithPrefix(this.systemPrefix) + + // Clear caches + this.nounCacheManager.clear() + this.verbCacheManager.clear() + + // Reset counts + this.totalNounCount = 0 + this.totalVerbCount = 0 + this.entityCounts.clear() + this.verbCounts.clear() + + this.logger.info('โœ… All data cleared from GCS') + } catch (error) { + this.logger.error('Failed to clear GCS storage:', error) + throw new Error(`Failed to clear GCS storage: ${error}`) + } + } + + /** + * Get storage status + */ + public async getStorageStatus(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + await this.ensureInitialized() + + try { + // Get bucket metadata + const [metadata] = await this.bucket!.getMetadata() + + return { + type: 'gcs-native', + used: 0, // GCS doesn't provide usage info easily + quota: null, // No quota in GCS + details: { + bucket: this.bucketName, + location: metadata.location, + storageClass: metadata.storageClass, + created: metadata.timeCreated + } + } + } catch (error) { + this.logger.error('Failed to get storage status:', error) + return { + type: 'gcs-native', + used: 0, + quota: null + } + } + } + + /** + * Save statistics data to storage + */ + protected async saveStatisticsData(statistics: StatisticsData): Promise { + await this.ensureInitialized() + + try { + const key = `${this.systemPrefix}${STATISTICS_KEY}.json` + + this.logger.trace(`Saving statistics to ${key}`) + + const file = this.bucket!.file(key) + await file.save(JSON.stringify(statistics, null, 2), { + contentType: 'application/json', + resumable: false + }) + + this.logger.trace('Statistics saved successfully') + } catch (error) { + this.logger.error('Failed to save statistics:', error) + throw new Error(`Failed to save statistics: ${error}`) + } + } + + /** + * Get statistics data from storage + */ + protected async getStatisticsData(): Promise { + await this.ensureInitialized() + + try { + const key = `${this.systemPrefix}${STATISTICS_KEY}.json` + + this.logger.trace(`Getting statistics from ${key}`) + + const file = this.bucket!.file(key) + const [contents] = await file.download() + + const statistics = JSON.parse(contents.toString()) + + this.logger.trace('Statistics retrieved successfully') + return statistics + } catch (error: any) { + if (error.code === 404) { + this.logger.trace('Statistics not found (creating new)') + return null + } + + this.logger.error('Failed to get statistics:', error) + return null + } + } + + /** + * Initialize counts from storage + */ + protected async initializeCounts(): Promise { + try { + const key = `${this.systemPrefix}counts.json` + + const file = this.bucket!.file(key) + const [contents] = await file.download() + + const counts = JSON.parse(contents.toString()) + + this.totalNounCount = counts.totalNounCount || 0 + this.totalVerbCount = counts.totalVerbCount || 0 + this.entityCounts = new Map(Object.entries(counts.entityCounts || {})) as Map + this.verbCounts = new Map(Object.entries(counts.verbCounts || {})) as Map + + prodLog.info(`๐Ÿ“Š Loaded counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) + } catch (error: any) { + if (error.code === 404) { + // No counts file yet - initialize from scan + prodLog.info('๐Ÿ“Š No counts file found - initializing from storage scan...') + await this.initializeCountsFromScan() + } else { + this.logger.error('Error loading counts:', error) + } + } + } + + /** + * Initialize counts from storage scan (expensive - only for first-time init) + */ + private async initializeCountsFromScan(): Promise { + try { + // Count nouns + const [nounFiles] = await this.bucket!.getFiles({ prefix: this.nounPrefix }) + this.totalNounCount = nounFiles?.filter((f: any) => f.name?.endsWith('.json')).length || 0 + + // Count verbs + const [verbFiles] = await this.bucket!.getFiles({ prefix: this.verbPrefix }) + this.totalVerbCount = verbFiles?.filter((f: any) => f.name?.endsWith('.json')).length || 0 + + // Save initial counts + await this.persistCounts() + + prodLog.info(`โœ… Initialized counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) + } catch (error) { + this.logger.error('Error initializing counts from scan:', error) + } + } + + /** + * Persist counts to storage + */ + protected async persistCounts(): Promise { + try { + const key = `${this.systemPrefix}counts.json` + + const counts = { + totalNounCount: this.totalNounCount, + totalVerbCount: this.totalVerbCount, + entityCounts: Object.fromEntries(this.entityCounts), + verbCounts: Object.fromEntries(this.verbCounts), + lastUpdated: new Date().toISOString() + } + + const file = this.bucket!.file(key) + await file.save(JSON.stringify(counts, null, 2), { + contentType: 'application/json', + resumable: false + }) + } catch (error) { + this.logger.error('Error persisting counts:', error) + } + } +} diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index d7094b84..3de3eb3b 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -10,6 +10,7 @@ import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js' +import { GcsStorage } from './adapters/gcsStorage.js' // FileSystemStorage is dynamically imported to avoid issues in browser environments import { isBrowser } from '../utils/environment.js' import { OperationConfig } from '../utils/operationUtils.js' @@ -26,9 +27,10 @@ export interface StorageOptions { * - 'filesystem': Use file system storage (Node.js only) * - 's3': Use Amazon S3 storage * - 'r2': Use Cloudflare R2 storage - * - 'gcs': Use Google Cloud Storage + * - 'gcs': Use Google Cloud Storage (S3-compatible with HMAC keys) + * - 'gcs-native': Use Google Cloud Storage (native SDK with ADC) */ - type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' + type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' /** * Force the use of memory storage even if other storage types are available @@ -106,7 +108,7 @@ export interface StorageOptions { } /** - * Configuration for Google Cloud Storage + * Configuration for Google Cloud Storage (S3-compatible with HMAC keys) */ gcsStorage?: { /** @@ -135,6 +137,36 @@ export interface StorageOptions { endpoint?: string } + /** + * Configuration for Google Cloud Storage (native SDK with ADC) + */ + gcsNativeStorage?: { + /** + * GCS bucket name + */ + bucketName: string + + /** + * Service account key file path (optional, uses ADC if not provided) + */ + keyFilename?: string + + /** + * Service account credentials object (optional, uses ADC if not provided) + */ + credentials?: object + + /** + * HMAC access key ID (backward compatibility, not recommended) + */ + accessKeyId?: string + + /** + * HMAC secret access key (backward compatibility, not recommended) + */ + secretAccessKey?: string + } + /** * Configuration for custom S3-compatible storage */ @@ -357,7 +389,7 @@ export async function createStorage( case 'gcs': if (options.gcsStorage) { - console.log('Using Google Cloud Storage') + console.log('Using Google Cloud Storage (S3-compatible)') return new S3CompatibleStorage({ bucketName: options.gcsStorage.bucketName, region: options.gcsStorage.region, @@ -375,6 +407,24 @@ export async function createStorage( return new MemoryStorage() } + case 'gcs-native': + if (options.gcsNativeStorage) { + console.log('Using Google Cloud Storage (native SDK)') + return new GcsStorage({ + bucketName: options.gcsNativeStorage.bucketName, + keyFilename: options.gcsNativeStorage.keyFilename, + credentials: options.gcsNativeStorage.credentials, + accessKeyId: options.gcsNativeStorage.accessKeyId, + secretAccessKey: options.gcsNativeStorage.secretAccessKey, + cacheConfig: options.cacheConfig + }) + } else { + console.warn( + 'GCS native storage configuration is missing, falling back to memory storage' + ) + return new MemoryStorage() + } + default: console.warn( `Unknown storage type: ${options.type}, falling back to memory storage` @@ -426,9 +476,22 @@ export async function createStorage( }) } - // If GCS storage is specified, use it + // If GCS native storage is specified, use it (prioritize native over S3-compatible) + if (options.gcsNativeStorage) { + console.log('Using Google Cloud Storage (native SDK)') + return new GcsStorage({ + bucketName: options.gcsNativeStorage.bucketName, + keyFilename: options.gcsNativeStorage.keyFilename, + credentials: options.gcsNativeStorage.credentials, + accessKeyId: options.gcsNativeStorage.accessKeyId, + secretAccessKey: options.gcsNativeStorage.secretAccessKey, + cacheConfig: options.cacheConfig + }) + } + + // If GCS storage is specified, use it (S3-compatible) if (options.gcsStorage) { - console.log('Using Google Cloud Storage') + console.log('Using Google Cloud Storage (S3-compatible)') return new S3CompatibleStorage({ bucketName: options.gcsStorage.bucketName, region: options.gcsStorage.region, @@ -498,7 +561,8 @@ export { MemoryStorage, OPFSStorage, S3CompatibleStorage, - R2Storage + R2Storage, + GcsStorage } // Export FileSystemStorage conditionally diff --git a/tests/integration/gcs-native-storage.test.ts b/tests/integration/gcs-native-storage.test.ts new file mode 100644 index 00000000..cb329e76 --- /dev/null +++ b/tests/integration/gcs-native-storage.test.ts @@ -0,0 +1,489 @@ +/** + * GCS Native Storage Adapter Integration Tests + * + * This test verifies that the GCS native adapter: + * - Properly authenticates with ADC and service accounts + * - Implements UUID-based sharding correctly + * - Handles pagination across shards + * - Persists data correctly + * - Manages statistics and counts + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { GcsStorage } from '../../src/storage/adapters/gcsStorage.js' +import { randomUUID } from 'node:crypto' + +describe('GCS Native Storage Adapter', () => { + // Mock GCS client for testing + let mockGcsObjects: Map = new Map() + let storage: GcsStorage | null = null + + // Helper to create mock GCS client + function createMockGcsClient() { + const mockBucket = { + exists: async () => [true], + + file: (name: string) => ({ + save: async (data: string, options: any) => { + mockGcsObjects.set(name, JSON.parse(data)) + return {} + }, + + download: async () => { + const data = mockGcsObjects.get(name) + if (!data) { + const error: any = new Error('Not found') + error.code = 404 + throw error + } + return [Buffer.from(JSON.stringify(data))] + }, + + delete: async () => { + mockGcsObjects.delete(name) + return [{}] + } + }), + + getFiles: async (options: any) => { + const prefix = options.prefix || '' + const maxResults = options.maxResults || 1000 + const pageToken = options.pageToken + + console.log(`[Mock GCS] getFiles: Prefix="${prefix}", maxResults=${maxResults}`) + + // Filter objects by prefix + const allKeys = Array.from(mockGcsObjects.keys()) + const matchingKeys = allKeys.filter(key => key.startsWith(prefix)).sort() + + console.log(`[Mock GCS] Total keys=${allKeys.length}, Matching=${matchingKeys.length}`) + if (matchingKeys.length > 0) { + console.log(`[Mock GCS] First match: ${matchingKeys[0]}`) + } + + // Apply pagination + let startIndex = 0 + if (pageToken) { + startIndex = parseInt(pageToken) + } + + const endIndex = Math.min(startIndex + maxResults, matchingKeys.length) + const pageKeys = matchingKeys.slice(startIndex, endIndex) + + const files = pageKeys.map(key => ({ + name: key + })) + + const response = { + nextPageToken: endIndex < matchingKeys.length ? String(endIndex) : undefined + } + + return [files, {}, response] + }, + + getMetadata: async () => [{ + location: 'us-central1', + storageClass: 'STANDARD', + timeCreated: new Date().toISOString() + }] + } + + return { + bucket: (name: string) => mockBucket + } + } + + beforeEach(() => { + mockGcsObjects.clear() + }) + + afterEach(async () => { + if (storage) { + try { + await storage.clear() + } catch (error) { + // Ignore cleanup errors + } + storage = null + } + }) + + it('should initialize with Application Default Credentials (ADC)', async () => { + // Create storage without any credentials (ADC) + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Verify initialization + expect((storage as any).bucketName).toBe('test-bucket') + expect((storage as any).keyFilename).toBeUndefined() + expect((storage as any).credentials).toBeUndefined() + }) + + it('should initialize with service account key file', async () => { + // Create storage with key file + storage = new GcsStorage({ + bucketName: 'test-bucket', + keyFilename: '/path/to/service-account.json' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Verify initialization + expect((storage as any).keyFilename).toBe('/path/to/service-account.json') + }) + + it('should write and read data with UUID-based sharding', async () => { + console.log('\n๐Ÿ“ Test: Write and read with UUID sharding...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Generate UUIDs for testing + const testData = [ + { id: randomUUID(), vector: [0.1, 0.2, 0.3], metadata: { type: 'user', name: 'Alice' } }, + { id: randomUUID(), vector: [0.4, 0.5, 0.6], metadata: { type: 'user', name: 'Bob' } }, + { id: randomUUID(), vector: [0.7, 0.8, 0.9], metadata: { type: 'user', name: 'Charlie' } } + ] + + // Save nouns with metadata + for (const item of testData) { + const noun = { + id: item.id, + vector: item.vector, + connections: new Map(), + layer: 0 + } + await storage.saveNoun(noun) + await (storage as any).saveNounMetadata_internal(item.id, item.metadata) + } + + console.log(`โœ… Wrote ${testData.length} entities`) + console.log(`๐Ÿ“Š Objects in mock storage: ${mockGcsObjects.size}`) + + // Verify data was written to UUID-sharded paths + const shardedKeys = Array.from(mockGcsObjects.keys()).filter(k => + k.match(/entities\/nouns\/vectors\/[0-9a-f]{2}\//) + ) + console.log(`๐Ÿ”‘ UUID-sharded keys: ${shardedKeys.length}`) + expect(shardedKeys.length).toBe(testData.length) + + // Log shard distribution + const shards = new Set(shardedKeys.map(k => k.match(/entities\/nouns\/vectors\/([0-9a-f]{2})\//)?.[1])) + console.log(`๐Ÿ“ Data distributed across ${shards.size} shards: ${Array.from(shards).join(', ')}`) + + // Verify each entity can be retrieved + for (const item of testData) { + const noun = await storage.getNoun(item.id) + expect(noun).toBeTruthy() + expect(noun!.id).toBe(item.id) + expect(noun!.vector).toEqual(item.vector) + + const metadata = await storage.getNounMetadata(item.id) + expect(metadata).toBeTruthy() + expect(metadata.name).toBe(item.metadata.name) + } + + console.log('โœ… All entities retrieved successfully') + }) + + it('should handle pagination across UUID shards', async () => { + console.log('\n๐Ÿ”„ Test: Pagination across shards...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Write 10 entities with proper UUIDs + console.log('๐Ÿ“ Writing 10 entities...') + for (let i = 0; i < 10; i++) { + const id = randomUUID() + const noun = { + id, + vector: [0.1 * i, 0.2 * i, 0.3 * i], + connections: new Map(), + layer: 0 + } + await storage.saveNoun(noun) + } + + // Read with pagination (limit: 3) + console.log('\n๐Ÿ”„ Reading with pagination (limit: 3)...') + + let allEntities: any[] = [] + let cursor: string | undefined + let page = 0 + + do { + const result = await storage.getNounsWithPagination({ + limit: 3, + cursor + }) + + page++ + console.log(`๐Ÿ“„ Page ${page}: ${result.items.length} entities, hasMore: ${result.hasMore}`) + + allEntities.push(...result.items) + cursor = result.nextCursor + + // Safety check to prevent infinite loops + expect(page).toBeLessThan(20) + } while (cursor) + + console.log(`โœ… Loaded ${allEntities.length} total entities across ${page} pages`) + + // Verify all entities were loaded + expect(allEntities.length).toBe(10) + }) + + it('should return correct totalCount on first call', async () => { + console.log('\n๐Ÿ“Š Test: Correct totalCount...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Write 5 entities + for (let i = 0; i < 5; i++) { + await storage.saveNoun({ + id: randomUUID(), + vector: [0.1, 0.2, 0.3], + connections: new Map(), + layer: 0 + }) + } + + // Get first page + const result = await storage.getNounsWithPagination({ limit: 2 }) + + console.log(`๐Ÿ“Š First page: ${result.items.length} items`) + + expect(result.items.length).toBeLessThanOrEqual(2) + }) + + it('should handle verb operations with UUID sharding', async () => { + console.log('\n๐Ÿ”— Test: Verb operations with UUID sharding...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Create verb + const verbId = randomUUID() + const verb = { + id: verbId, + vector: [0.1, 0.2, 0.3], + connections: new Map() + } + + await storage.saveVerb(verb) + + // Verify verb was saved with UUID sharding + const shardedKeys = Array.from(mockGcsObjects.keys()).filter(k => + k.match(/entities\/verbs\/vectors\/[0-9a-f]{2}\//) + ) + expect(shardedKeys.length).toBeGreaterThan(0) + + // Retrieve verb + const retrieved = await storage.getVerb(verbId) + expect(retrieved).toBeTruthy() + expect(retrieved!.id).toBe(verbId) + + console.log('โœ… Verb operations successful') + }) + + it('should handle metadata sharding correctly', async () => { + console.log('\n๐Ÿ—‚๏ธ Test: Metadata sharding...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Create noun with metadata + const nounId = randomUUID() + const noun = { + id: nounId, + vector: [0.1, 0.2, 0.3], + connections: new Map(), + layer: 0 + } + const metadata = { + type: 'user', + name: 'Test User', + email: 'test@example.com' + } + + await storage.saveNoun(noun) + await (storage as any).saveNounMetadata_internal(nounId, metadata) + + // Verify metadata was saved with UUID sharding + const metadataKeys = Array.from(mockGcsObjects.keys()).filter(k => + k.match(/entities\/nouns\/metadata\/[0-9a-f]{2}\//) + ) + expect(metadataKeys.length).toBeGreaterThan(0) + + // Retrieve metadata + const retrieved = await storage.getNounMetadata(nounId) + expect(retrieved).toBeTruthy() + expect(retrieved.name).toBe('Test User') + + console.log('โœ… Metadata sharding successful') + }) + + it('should clear all data correctly', async () => { + console.log('\n๐Ÿงน Test: Clear all data...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Write some data + for (let i = 0; i < 3; i++) { + await storage.saveNoun({ + id: randomUUID(), + vector: [0.1, 0.2, 0.3], + connections: new Map(), + layer: 0 + }) + } + + console.log(`๐Ÿ“Š Objects before clear: ${mockGcsObjects.size}`) + + // Clear all data + await storage.clear() + + console.log(`๐Ÿ“Š Objects after clear: ${mockGcsObjects.size}`) + + expect(mockGcsObjects.size).toBe(0) + console.log('โœ… Clear successful') + }) + + it('should handle throttling errors correctly', async () => { + console.log('\n๐Ÿšฆ Test: Throttling detection...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Test throttling error detection + const throttlingError = { code: 429, message: 'Too Many Requests' } + expect((storage as any).isThrottlingError(throttlingError)).toBe(true) + + const quotaError = { code: 403, message: 'Quota exceeded' } + expect((storage as any).isThrottlingError(quotaError)).toBe(true) + + const normalError = { code: 500, message: 'Internal Server Error' } + expect((storage as any).isThrottlingError(normalError)).toBe(false) + + console.log('โœ… Throttling detection working') + }) + + it('should manage statistics correctly', async () => { + console.log('\n๐Ÿ“Š Test: Statistics management...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + // Create statistics + const stats = { + nounCount: { 'test-service': 5 }, + verbCount: { 'test-service': 3 }, + metadataCount: {}, + hnswIndexSize: 100, + lastUpdated: new Date().toISOString() + } + + // Save statistics + await (storage as any).saveStatisticsData(stats) + + // Verify statistics key was created + const statsKeys = Array.from(mockGcsObjects.keys()).filter(k => + k.includes('_system/statistics.json') + ) + expect(statsKeys.length).toBe(1) + + // Retrieve statistics + const retrieved = await (storage as any).getStatisticsData() + expect(retrieved).toBeTruthy() + expect(retrieved.nounCount['test-service']).toBe(5) + + console.log('โœ… Statistics management successful') + }) + + it('should get storage status', async () => { + console.log('\n๐Ÿ“Š Test: Storage status...') + + // Create storage + storage = new GcsStorage({ + bucketName: 'test-bucket' + }) + + // Mock the GCS client + ;(storage as any).storage = createMockGcsClient() + ;(storage as any).bucket = (storage as any).storage.bucket('test-bucket') + ;(storage as any).isInitialized = true + + const status = await storage.getStorageStatus() + + expect(status.type).toBe('gcs-native') + expect(status.details).toBeTruthy() + expect(status.details!.bucket).toBe('test-bucket') + + console.log('โœ… Storage status retrieved:', status) + }) +}) + +console.log('\nโœ… GCS Native Storage Tests Complete')