fix: resolve critical GCS storage bugs preventing production use

Fixes three critical bugs in GCS storage adapter that prevented the Soulcraft
Studio team from using GCS in production:

1. Missing metadata in getNode() - entities returned without metadata fields,
   causing brain.get() to return null and preventing entity reconstruction
2. Index rebuild failure - cascading effect from missing metadata prevented
   proper HNSW index rebuilding after container restarts
3. Invalid GCS API parameters - passing Number.MAX_SAFE_INTEGER to maxResults
   parameter exceeded GCS API limit of 5000, causing "Invalid unsigned integer" errors

Changes:
- Add metadata field to getNode() return value (gcsStorage.ts:520)
- Add MAX_GCS_PAGE_SIZE constant (5000) for GCS API compliance
- Cap maxResults in getNodesWithPagination to prevent API errors
- Cap maxResults in getVerbsWithPagination to prevent API errors
- Update storage type from 'gcs-native' to 'gcs' for consistency

These fixes make GCS storage production-ready and fully compatible with
filesystem storage behavior.
This commit is contained in:
David Snelling 2025-10-10 14:49:53 -07:00
parent 8ba8336fca
commit 3cd0b9affa

View file

@ -38,6 +38,11 @@ type Storage = any
type Bucket = any type Bucket = any
type File = any type File = any
// GCS API limits
// Maximum value for maxResults parameter in GCS API calls
// Values above this cause "Invalid unsigned integer" errors
const MAX_GCS_PAGE_SIZE = 5000
/** /**
* Native Google Cloud Storage adapter for server environments * Native Google Cloud Storage adapter for server environments
* Uses the @google-cloud/storage library with Application Default Credentials * Uses the @google-cloud/storage library with Application Default Credentials
@ -516,7 +521,8 @@ export class GcsStorage extends BaseStorage {
id: data.id, id: data.id,
vector: data.vector, vector: data.vector,
connections, connections,
level: data.level || 0 level: data.level || 0,
metadata: data.metadata // CRITICAL: Include metadata for entity reconstruction
} }
// Update cache // Update cache
@ -995,9 +1001,13 @@ export class GcsStorage extends BaseStorage {
const shardPrefix = `${this.nounPrefix}${shardId}/` const shardPrefix = `${this.nounPrefix}${shardId}/`
// List objects in this shard // List objects in this shard
// Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
const requestedPageSize = limit - nodes.length
const cappedPageSize = Math.min(requestedPageSize, MAX_GCS_PAGE_SIZE)
const [files, , response] = await this.bucket!.getFiles({ const [files, , response] = await this.bucket!.getFiles({
prefix: shardPrefix, prefix: shardPrefix,
maxResults: limit - nodes.length, maxResults: cappedPageSize,
pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined
}) })
@ -1149,9 +1159,12 @@ export class GcsStorage extends BaseStorage {
try { try {
// List verbs (simplified - not sharded yet in original implementation) // List verbs (simplified - not sharded yet in original implementation)
// Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
const cappedLimit = Math.min(limit, MAX_GCS_PAGE_SIZE)
const [files, , response] = await this.bucket!.getFiles({ const [files, , response] = await this.bucket!.getFiles({
prefix: this.verbPrefix, prefix: this.verbPrefix,
maxResults: limit, maxResults: cappedLimit,
pageToken: options.cursor pageToken: options.cursor
}) })
@ -1370,7 +1383,7 @@ export class GcsStorage extends BaseStorage {
const [metadata] = await this.bucket!.getMetadata() const [metadata] = await this.bucket!.getMetadata()
return { return {
type: 'gcs-native', type: 'gcs',
used: 0, // GCS doesn't provide usage info easily used: 0, // GCS doesn't provide usage info easily
quota: null, // No quota in GCS quota: null, // No quota in GCS
details: { details: {
@ -1383,7 +1396,7 @@ export class GcsStorage extends BaseStorage {
} catch (error) { } catch (error) {
this.logger.error('Failed to get storage status:', error) this.logger.error('Failed to get storage status:', error)
return { return {
type: 'gcs-native', type: 'gcs',
used: 0, used: 0,
quota: null quota: null
} }