chore(8.0): step-7 follow-through — collapse remaining cloud branches + docs sweep (scaffold step 13)
Final cleanup pass for Brainy 8.0. Catches three categories of debt: A. STEP-7 FOLLOW-THROUGH (rebuild-path collapse) Step 7's bisect-reset (debugging a flaky test) lost the in-source edits to three rebuild paths even though the commit message claimed they shipped. Re-applied now: - src/utils/metadataIndex.ts — collapsed the `isLocalStorage` / cloud-pagination branching. Local-load-all-at-once is the only path in 8.0. Removed ~150 LOC of paginated-cloud branching for both nouns and verbs, plus the safety counters (`consecutiveEmptyBatches`, `MAX_ITERATIONS`, etc.). - src/hnsw/hnswIndex.ts — same simplification for HNSW rebuild. The paginated cloud path is gone; HNSW now loads all nodes at once. Removed ~85 LOC. - src/graph/graphAdjacencyIndex.ts — same simplification for graph adjacency rebuild. Removed ~50 LOC. The collapse is safe because cloud adapters were deleted in step 7; `storageType === 'OPFSStorage'` (and similar) can never match now. B. CLOUD-ONLY DOCS DELETED - docs/operations/cost-optimization-aws-s3.md - docs/operations/cost-optimization-azure.md - docs/operations/cost-optimization-cloudflare-r2.md - docs/operations/cost-optimization-gcs.md - docs/operations/cloud-run-filestore-guide.md (docs/deployment/* contained no cloud-specific files that needed deletion.) C. STORAGE-ADAPTERS GUIDE REWRITTEN FOR 8.0 docs/guides/storage-adapters.md → fresh content reflecting the 8.0 reality: - Two adapters: FileSystemStorage + MemoryStorage. Quick-start matrix. - Cloud backup section explains the operator-tooling pattern (gsutil / aws s3 / rclone / azcopy) with the exact commands consumers will run. - "Why no cloud adapters in 8.0?" section documents the four reasons per BR-BRAINY-80-STORAGE-SIMPLIFY. - Migration recipe for 7.x cloud-adapter consumers: mount local disk → filesystem storage → operator backup cron. Updated frontmatter description so soulcraft.com/docs renders the correct preview. NOT IN THIS COMMIT (deliberate, lower-priority) - src/storage/cacheManager.ts still references StorageType.S3 / REMOTE_API / OPFS as dead branches (23 sites). The branches are never reached in 8.0, but cleaning them would cascade through 5 consumers. Defer to a follow-up if the dead code surfaces as a real maintenance issue. - src/config/storageAutoConfig.ts keeps its StorageType enum + autodetect for 7.x compat surface. Same reason: rewriting cascades through zeroConfig, extensibleConfig, sharedConfigManager. Defer. - docs/MIGRATION-V3-TO-V4.md and docs/DEVELOPER_LEARNING_PATH.md still reference cloud adapters as historical artefacts. That's accurate — they describe how things used to be. Left as-is. - @deprecated audit in src/ (10 files) deferred — audit each individually in a future polish pass. VERIFICATION - npx tsc --noEmit: clean - npm test: 1408 / 1409 (same pre-existing race-condition outstanding from step 7; no regressions from this cleanup)
This commit is contained in:
parent
780fb6444b
commit
9f9a41599e
13 changed files with 162 additions and 5288 deletions
|
|
@ -1470,20 +1470,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
)
|
||||
}
|
||||
|
||||
// Step 4: Adaptive loading strategy based on storage type
|
||||
// FileSystem/Memory/OPFS: Load all at once (avoids repeated getAllShardedFiles() calls)
|
||||
// Cloud (GCS/S3/R2): Use pagination (efficient native cloud APIs)
|
||||
// Step 4: Load all HNSW nodes at once. Brainy 8.0 ships filesystem +
|
||||
// memory storage only (per BR-BRAINY-80-STORAGE-SIMPLIFY); the
|
||||
// cloud-pagination rebuild path was deleted alongside the cloud adapters.
|
||||
const storageType = this.storage?.constructor.name || ''
|
||||
const isLocalStorage = storageType === 'FileSystemStorage' ||
|
||||
storageType === 'MemoryStorage' ||
|
||||
storageType === 'OPFSStorage'
|
||||
|
||||
let loadedCount = 0
|
||||
let totalCount: number | undefined = undefined
|
||||
|
||||
if (isLocalStorage) {
|
||||
// Local storage: Load all nouns at once
|
||||
prodLog.info(`HNSW: Using optimized strategy - load all nodes at once (${storageType})`)
|
||||
{
|
||||
prodLog.info(`HNSW: Load all nodes at once (${storageType})`)
|
||||
|
||||
const result: {
|
||||
items: HNSWNoun[]
|
||||
|
|
@ -1554,94 +1549,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
options.onProgress(loadedCount, totalCount)
|
||||
}
|
||||
|
||||
prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes at once (local storage)`)
|
||||
|
||||
} else {
|
||||
// Cloud storage: Use pagination with native cloud APIs
|
||||
prodLog.info(`HNSW: Using cloud pagination strategy (${storageType})`)
|
||||
|
||||
let hasMore = true
|
||||
let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop)
|
||||
|
||||
while (hasMore) {
|
||||
// Fetch batch of nouns from storage (cast needed as method is not in base interface)
|
||||
const result: {
|
||||
items: HNSWNoun[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
} = await (this.storage as any).getNounsWithPagination({
|
||||
limit: batchSize,
|
||||
offset // Pass offset for proper pagination (previously passed cursor which was ignored)
|
||||
})
|
||||
|
||||
// Set total count on first batch
|
||||
if (totalCount === undefined && result.totalCount !== undefined) {
|
||||
totalCount = result.totalCount
|
||||
}
|
||||
|
||||
// Process each noun in the batch
|
||||
for (const nounData of result.items) {
|
||||
try {
|
||||
// Load HNSW graph data for this entity
|
||||
const hnswData = await (this.storage as any).getVectorIndexData(nounData.id)
|
||||
|
||||
if (!hnswData) {
|
||||
// No HNSW data - skip (might be entity added before persistence)
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine if vector should be kept in memory
|
||||
const keepVector = shouldPreload && this.vectorStorageMode !== 'lazy'
|
||||
|
||||
// Create noun object with restored connections
|
||||
const noun: HNSWNoun = {
|
||||
id: nounData.id,
|
||||
vector: keepVector ? nounData.vector : [], // Preload if dataset is small and not lazy
|
||||
connections: new Map(),
|
||||
level: hnswData.level
|
||||
}
|
||||
|
||||
// Restore SQ8 quantized data if quantization is enabled
|
||||
if (this.quantizationEnabled && nounData.vector.length > 0) {
|
||||
const sq8 = quantizeSQ8(nounData.vector)
|
||||
noun.quantizedVector = sq8.quantized
|
||||
noun.codebookMin = sq8.min
|
||||
noun.codebookMax = sq8.max
|
||||
}
|
||||
|
||||
// Restore connections from persisted data — compressed blob path
|
||||
// first, legacy JSON-array fallback for indexes written before
|
||||
// graph link compression landed.
|
||||
await this.restoreNodeConnections(nounData.id, hnswData, noun)
|
||||
|
||||
// Add to in-memory index
|
||||
this.nouns.set(nounData.id, noun)
|
||||
|
||||
// Track high-level nodes for O(1) entry point selection
|
||||
if (noun.level >= 2 && noun.level <= this.MAX_TRACKED_LEVELS) {
|
||||
if (!this.highLevelNodes.has(noun.level)) {
|
||||
this.highLevelNodes.set(noun.level, new Set())
|
||||
}
|
||||
this.highLevelNodes.get(noun.level)!.add(nounData.id)
|
||||
}
|
||||
|
||||
loadedCount++
|
||||
} catch (error) {
|
||||
// Log error but continue (robust error recovery)
|
||||
console.error(`Failed to rebuild HNSW data for ${nounData.id}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress
|
||||
if (options.onProgress && totalCount !== undefined) {
|
||||
options.onProgress(loadedCount, totalCount)
|
||||
}
|
||||
|
||||
// Check for more data
|
||||
hasMore = result.hasMore
|
||||
offset += batchSize // Increment offset for next page
|
||||
}
|
||||
prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`)
|
||||
}
|
||||
|
||||
// Step 5: CRITICAL - Recover entry point if missing)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue