fix: resolve critical GCS storage bugs preventing production use

Fixes three critical bugs in GCS storage adapter that prevented the Soulcraft
Studio team from using GCS in production:

1. Missing metadata in getNode() - entities returned without metadata fields,
   causing brain.get() to return null and preventing entity reconstruction
2. Index rebuild failure - cascading effect from missing metadata prevented
   proper HNSW index rebuilding after container restarts
3. Invalid GCS API parameters - passing Number.MAX_SAFE_INTEGER to maxResults
   parameter exceeded GCS API limit of 5000, causing "Invalid unsigned integer" errors

Changes:
- Add metadata field to getNode() return value (gcsStorage.ts:520)
- Add MAX_GCS_PAGE_SIZE constant (5000) for GCS API compliance
- Cap maxResults in getNodesWithPagination to prevent API errors
- Cap maxResults in getVerbsWithPagination to prevent API errors
- Update storage type from 'gcs-native' to 'gcs' for consistency

These fixes make GCS storage production-ready and fully compatible with
filesystem storage behavior.
This commit is contained in:
David Snelling 2025-10-10 14:49:53 -07:00
parent 8ba8336fca
commit 3cd0b9affa

View file

@ -38,6 +38,11 @@ type Storage = any
type Bucket = any
type File = any
// GCS API limits
// Maximum value for maxResults parameter in GCS API calls
// Values above this cause "Invalid unsigned integer" errors
const MAX_GCS_PAGE_SIZE = 5000
/**
* Native Google Cloud Storage adapter for server environments
* Uses the @google-cloud/storage library with Application Default Credentials
@ -516,7 +521,8 @@ export class GcsStorage extends BaseStorage {
id: data.id,
vector: data.vector,
connections,
level: data.level || 0
level: data.level || 0,
metadata: data.metadata // CRITICAL: Include metadata for entity reconstruction
}
// Update cache
@ -995,9 +1001,13 @@ export class GcsStorage extends BaseStorage {
const shardPrefix = `${this.nounPrefix}${shardId}/`
// List objects in this shard
// Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
const requestedPageSize = limit - nodes.length
const cappedPageSize = Math.min(requestedPageSize, MAX_GCS_PAGE_SIZE)
const [files, , response] = await this.bucket!.getFiles({
prefix: shardPrefix,
maxResults: limit - nodes.length,
maxResults: cappedPageSize,
pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined
})
@ -1149,9 +1159,12 @@ export class GcsStorage extends BaseStorage {
try {
// List verbs (simplified - not sharded yet in original implementation)
// Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
const cappedLimit = Math.min(limit, MAX_GCS_PAGE_SIZE)
const [files, , response] = await this.bucket!.getFiles({
prefix: this.verbPrefix,
maxResults: limit,
maxResults: cappedLimit,
pageToken: options.cursor
})
@ -1370,7 +1383,7 @@ export class GcsStorage extends BaseStorage {
const [metadata] = await this.bucket!.getMetadata()
return {
type: 'gcs-native',
type: 'gcs',
used: 0, // GCS doesn't provide usage info easily
quota: null, // No quota in GCS
details: {
@ -1383,7 +1396,7 @@ export class GcsStorage extends BaseStorage {
} catch (error) {
this.logger.error('Failed to get storage status:', error)
return {
type: 'gcs-native',
type: 'gcs',
used: 0,
quota: null
}