fix: resolve critical GCS storage bugs preventing production use
Fixes three critical bugs in GCS storage adapter that prevented the Soulcraft Studio team from using GCS in production: 1. Missing metadata in getNode() - entities returned without metadata fields, causing brain.get() to return null and preventing entity reconstruction 2. Index rebuild failure - cascading effect from missing metadata prevented proper HNSW index rebuilding after container restarts 3. Invalid GCS API parameters - passing Number.MAX_SAFE_INTEGER to maxResults parameter exceeded GCS API limit of 5000, causing "Invalid unsigned integer" errors Changes: - Add metadata field to getNode() return value (gcsStorage.ts:520) - Add MAX_GCS_PAGE_SIZE constant (5000) for GCS API compliance - Cap maxResults in getNodesWithPagination to prevent API errors - Cap maxResults in getVerbsWithPagination to prevent API errors - Update storage type from 'gcs-native' to 'gcs' for consistency These fixes make GCS storage production-ready and fully compatible with filesystem storage behavior.
This commit is contained in:
parent
8ba8336fca
commit
3cd0b9affa
1 changed files with 18 additions and 5 deletions
|
|
@ -38,6 +38,11 @@ type Storage = any
|
||||||
type Bucket = any
|
type Bucket = any
|
||||||
type File = any
|
type File = any
|
||||||
|
|
||||||
|
// GCS API limits
|
||||||
|
// Maximum value for maxResults parameter in GCS API calls
|
||||||
|
// Values above this cause "Invalid unsigned integer" errors
|
||||||
|
const MAX_GCS_PAGE_SIZE = 5000
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Native Google Cloud Storage adapter for server environments
|
* Native Google Cloud Storage adapter for server environments
|
||||||
* Uses the @google-cloud/storage library with Application Default Credentials
|
* Uses the @google-cloud/storage library with Application Default Credentials
|
||||||
|
|
@ -516,7 +521,8 @@ export class GcsStorage extends BaseStorage {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
connections,
|
connections,
|
||||||
level: data.level || 0
|
level: data.level || 0,
|
||||||
|
metadata: data.metadata // CRITICAL: Include metadata for entity reconstruction
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
|
|
@ -995,9 +1001,13 @@ export class GcsStorage extends BaseStorage {
|
||||||
const shardPrefix = `${this.nounPrefix}${shardId}/`
|
const shardPrefix = `${this.nounPrefix}${shardId}/`
|
||||||
|
|
||||||
// List objects in this shard
|
// List objects in this shard
|
||||||
|
// Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
|
||||||
|
const requestedPageSize = limit - nodes.length
|
||||||
|
const cappedPageSize = Math.min(requestedPageSize, MAX_GCS_PAGE_SIZE)
|
||||||
|
|
||||||
const [files, , response] = await this.bucket!.getFiles({
|
const [files, , response] = await this.bucket!.getFiles({
|
||||||
prefix: shardPrefix,
|
prefix: shardPrefix,
|
||||||
maxResults: limit - nodes.length,
|
maxResults: cappedPageSize,
|
||||||
pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined
|
pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1149,9 +1159,12 @@ export class GcsStorage extends BaseStorage {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// List verbs (simplified - not sharded yet in original implementation)
|
// List verbs (simplified - not sharded yet in original implementation)
|
||||||
|
// Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
|
||||||
|
const cappedLimit = Math.min(limit, MAX_GCS_PAGE_SIZE)
|
||||||
|
|
||||||
const [files, , response] = await this.bucket!.getFiles({
|
const [files, , response] = await this.bucket!.getFiles({
|
||||||
prefix: this.verbPrefix,
|
prefix: this.verbPrefix,
|
||||||
maxResults: limit,
|
maxResults: cappedLimit,
|
||||||
pageToken: options.cursor
|
pageToken: options.cursor
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1370,7 +1383,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
const [metadata] = await this.bucket!.getMetadata()
|
const [metadata] = await this.bucket!.getMetadata()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
type: 'gcs-native',
|
type: 'gcs',
|
||||||
used: 0, // GCS doesn't provide usage info easily
|
used: 0, // GCS doesn't provide usage info easily
|
||||||
quota: null, // No quota in GCS
|
quota: null, // No quota in GCS
|
||||||
details: {
|
details: {
|
||||||
|
|
@ -1383,7 +1396,7 @@ export class GcsStorage extends BaseStorage {
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error('Failed to get storage status:', error)
|
this.logger.error('Failed to get storage status:', error)
|
||||||
return {
|
return {
|
||||||
type: 'gcs-native',
|
type: 'gcs',
|
||||||
used: 0,
|
used: 0,
|
||||||
quota: null
|
quota: null
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue