soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix(storage): CRITICAL - preserve vectors when updating HNSW connections (v4.7.3)

Browse source 
CRITICAL DATA CORRUPTION FIX affecting ALL storage adapters

## Root Cause
When HNSW index updated node connections (adding new neighbors), saveHNSWData()
overwrote the entire node file with ONLY {level, connections}, destroying vector data.

## Impact
- v4.7.2: Broke ALL imports - relate() crashed with "Cannot read properties of undefined"
- Affected ALL storage adapters: FileSystem, GCS, Azure, R2, OPFS, S3Compatible
- VFS imports completely non-functional
- Any multi-entity operation would corrupt existing entity vectors

## The Bug
```typescript
// OLD CODE (v4.7.2) - DESTROYED VECTORS:
async saveHNSWData(id, hnswData) {
  await writeFile(path, JSON.stringify(hnswData))  // Only {level, connections}!
}
```

When entity2 was added to HNSW:
1. HNSW found entity1 as neighbor
2. Updated entity1's connections
3. Called saveHNSWData(entity1.id, {level, connections})
4. Overwrote entity1.json with ONLY {level, connections}
5. **entity1.vector and entity1.id were DESTROYED**

## The Fix
```typescript
// NEW CODE (v4.7.3) - PRESERVES ALL DATA:
async saveHNSWData(id, hnswData) {
  const existing = await readFile(path)
  const updated = {...existing, level: hnswData.level, connections: hnswData.connections}
  await writeFile(path, JSON.stringify(updated))  // Preserves id, vector, etc.
}
```

Now READ existing node, UPDATE only HNSW fields, WRITE complete node.

## Files Changed
- src/storage/adapters/fileSystemStorage.ts (line 2590-2626)
- src/storage/adapters/gcsStorage.ts (line 1863-1911)
- src/storage/adapters/azureBlobStorage.ts (line 1638-1682)
- src/storage/adapters/r2Storage.ts (line 999-1029)
- src/storage/adapters/opfsStorage.ts (line 2012-2051)
- src/storage/adapters/s3CompatibleStorage.ts (line 3903-3961)

## Testing
✅ FileSystemStorage: Verified with test-relate-crash.js
✅ All adapters: Compilation successful
✅ Imports: VFS directory creation and relate() working

## Breaking Changes
NONE - This is a critical bug fix

## Migration
Workshop team: Delete brainy-data and reimport with v4.7.3

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-10-27 13:07:00 -07:00
parent fc307bc215
commit 46e74827c4
8 changed files with 180 additions and 349 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/storage/adapters/azureBlobStorage.ts
View file

@ -1642,14 +1642,39 @@ export class AzureBlobStorage extends BaseStorage {
await this.ensureInitialized()
try {
// CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
const shard = getShardIdFromUuid(nounId)
const key = `entities/nouns/hnsw/${shard}/${nounId}.json`
const blockBlobClient = this.containerClient!.getBlockBlobClient(key)
const content = JSON.stringify(hnswData, null, 2)
await blockBlobClient.upload(content, content.length, {
blobHTTPHeaders: { blobContentType: 'application/json' }
})
try {
// Read existing node data
const downloadResponse = await blockBlobClient.download(0)
const existingData = await this.streamToBuffer(downloadResponse.readableStreamBody!)
const existingNode = JSON.parse(existingData.toString())
// Preserve id and vector, update only HNSW graph metadata
const updatedNode = {
...existingNode,
level: hnswData.level,
connections: hnswData.connections
}
const content = JSON.stringify(updatedNode, null, 2)
await blockBlobClient.upload(content, content.length, {
blobHTTPHeaders: { blobContentType: 'application/json' }
})
} catch (error: any) {
// If node doesn't exist yet, create it with just HNSW data
if (error.statusCode === 404 || error.code === 'BlobNotFound') {
const content = JSON.stringify(hnswData, null, 2)
await blockBlobClient.upload(content, content.length, {
blobHTTPHeaders: { blobContentType: 'application/json' }
})
} else {
throw error
}
}
} catch (error) {
this.logger.error(`Failed to save HNSW data for ${nounId}:`, error)
throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`)

35
src/storage/adapters/fileSystemStorage.ts
View file

@ -2579,13 +2579,36 @@ export class FileSystemStorage extends BaseStorage {
}): Promise<void> {
await this.ensureInitialized()
// Use sharded path for HNSW data
const shard = nounId.substring(0, 2).toLowerCase()
const hnswDir = path.join(this.rootDir, 'entities', 'nouns', 'hnsw', shard)
await this.ensureDirectoryExists(hnswDir)
// CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
// Previous implementation overwrote the entire file, destroying vector data
// Now we READ the existing node, UPDATE only connections/level, then WRITE back the complete node
const filePath = path.join(hnswDir, `${nounId}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(hnswData, null, 2))
const filePath = this.getNodePath(nounId)
try {
// Read existing node data
const existingData = await fs.promises.readFile(filePath, 'utf-8')
const existingNode = JSON.parse(existingData)
// Preserve id and vector, update only HNSW graph metadata
const updatedNode = {
...existingNode, // Preserve all existing fields (id, vector, etc.)
level: hnswData.level,
connections: hnswData.connections
}
// Write back the COMPLETE node with updated HNSW data
await fs.promises.writeFile(filePath, JSON.stringify(updatedNode, null, 2))
} catch (error: any) {
// If node doesn't exist yet, create it with just HNSW data
// This should only happen during initial node creation
if (error.code === 'ENOENT') {
await this.ensureDirectoryExists(path.dirname(filePath))
await fs.promises.writeFile(filePath, JSON.stringify(hnswData, null, 2))
} else {
throw error
}
}
}
/**

40
src/storage/adapters/gcsStorage.ts
View file

@ -1867,15 +1867,43 @@ export class GcsStorage extends BaseStorage {
await this.ensureInitialized()
try {
// Use sharded path for HNSW data
// CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
// Previous implementation overwrote the entire file, destroying vector data
// Now we READ the existing node, UPDATE only connections/level, then WRITE back the complete node
const shard = getShardIdFromUuid(nounId)
const key = `entities/nouns/hnsw/${shard}/${nounId}.json`
const file = this.bucket!.file(key)
await file.save(JSON.stringify(hnswData, null, 2), {
contentType: 'application/json',
resumable: false
})
try {
// Read existing node data
const [existingData] = await file.download()
const existingNode = JSON.parse(existingData.toString())
// Preserve id and vector, update only HNSW graph metadata
const updatedNode = {
...existingNode, // Preserve all existing fields (id, vector, etc.)
level: hnswData.level,
connections: hnswData.connections
}
// Write back the COMPLETE node with updated HNSW data
await file.save(JSON.stringify(updatedNode, null, 2), {
contentType: 'application/json',
resumable: false
})
} catch (error: any) {
// If node doesn't exist yet, create it with just HNSW data
// This should only happen during initial node creation
if (error.code === 404) {
await file.save(JSON.stringify(hnswData, null, 2), {
contentType: 'application/json',
resumable: false
})
} else {
throw error
}
}
} catch (error) {
this.logger.error(`Failed to save HNSW data for ${nounId}:`, error)
throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`)

32
src/storage/adapters/opfsStorage.ts
View file

@ -2016,20 +2016,34 @@ export class OPFSStorage extends BaseStorage {
await this.ensureInitialized()
try {
// Get or create the hnsw directory under nouns
// CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
const hnswDir = await this.nounsDir!.getDirectoryHandle('hnsw', { create: true })
// Use sharded path for HNSW data
const shard = getShardIdFromUuid(nounId)
const shardDir = await hnswDir.getDirectoryHandle(shard, { create: true })
// Create or get the file in the shard directory
const fileHandle = await shardDir.getFileHandle(`${nounId}.json`, { create: true })
// Write the HNSW data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(hnswData, null, 2))
await writable.close()
try {
// Read existing node data
const file = await fileHandle.getFile()
const existingData = await file.text()
const existingNode = JSON.parse(existingData)
// Preserve id and vector, update only HNSW graph metadata
const updatedNode = {
...existingNode,
level: hnswData.level,
connections: hnswData.connections
}
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(updatedNode, null, 2))
await writable.close()
} catch (error) {
// If node doesn't exist or read fails, create with just HNSW data
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(hnswData, null, 2))
await writable.close()
}
} catch (error) {
console.error(`Failed to save HNSW data for ${nounId}:`, error)
throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`)

22
src/storage/adapters/r2Storage.ts
View file

@ -1002,10 +1002,30 @@ export class R2Storage extends BaseStorage {
}): Promise<void> {
await this.ensureInitialized()
// CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
const shard = getShardIdFromUuid(nounId)
const key = `entities/nouns/hnsw/${shard}/${nounId}.json`
await this.writeObjectToPath(key, hnswData)
try {
// Read existing node data
const existingNode = await this.readObjectFromPath(key)
if (existingNode) {
// Preserve id and vector, update only HNSW graph metadata
const updatedNode = {
...existingNode,
level: hnswData.level,
connections: hnswData.connections
}
await this.writeObjectToPath(key, updatedNode)
} else {
// Node doesn't exist yet, create with just HNSW data
await this.writeObjectToPath(key, hnswData)
}
} catch (error) {
// If read fails, create with just HNSW data
await this.writeObjectToPath(key, hnswData)
}
}
public async getHNSWData(nounId: string): Promise<{

53
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -3907,20 +3907,53 @@ export class S3CompatibleStorage extends BaseStorage {
await this.ensureInitialized()
try {
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const { PutObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3')
// Use sharded path for HNSW data
// CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
const shard = getShardIdFromUuid(nounId)
const key = `entities/nouns/hnsw/${shard}/${nounId}.json`
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: key,
Body: JSON.stringify(hnswData, null, 2),
ContentType: 'application/json'
})
)
try {
// Read existing node data
const getResponse = await this.s3Client!.send(
new GetObjectCommand({
Bucket: this.bucketName,
Key: key
})
)
const existingData = await getResponse.Body!.transformToString()
const existingNode = JSON.parse(existingData)
// Preserve id and vector, update only HNSW graph metadata
const updatedNode = {
...existingNode,
level: hnswData.level,
connections: hnswData.connections
}
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: key,
Body: JSON.stringify(updatedNode, null, 2),
ContentType: 'application/json'
})
)
} catch (error: any) {
// If node doesn't exist yet, create it with just HNSW data
if (error.name === 'NoSuchKey' || error.Code === 'NoSuchKey') {
await this.s3Client!.send(
new PutObjectCommand({
Bucket: this.bucketName,
Key: key,
Body: JSON.stringify(hnswData, null, 2),
ContentType: 'application/json'
})
)
} else {
throw error
}
}
} catch (error) {
this.logger.error(`Failed to save HNSW data for ${nounId}:`, error)
throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`)