David Snelling
8ff382ca3b
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
71 lines
2.1 KiB
TypeScript
71 lines
2.1 KiB
TypeScript
/**
|
|
* Enhanced Batch S3 Operations for High-Performance Vector Retrieval
|
|
* Implements optimized batch operations to reduce S3 API calls and latency
|
|
*/
|
|
import { HNSWNoun } from '../../coreTypes.js';
|
|
type S3Client = any;
|
|
export interface BatchRetrievalOptions {
|
|
maxConcurrency?: number;
|
|
prefetchSize?: number;
|
|
useS3Select?: boolean;
|
|
compressionEnabled?: boolean;
|
|
}
|
|
export interface BatchResult<T> {
|
|
items: Map<string, T>;
|
|
errors: Map<string, Error>;
|
|
statistics: {
|
|
totalRequested: number;
|
|
totalRetrieved: number;
|
|
totalErrors: number;
|
|
duration: number;
|
|
apiCalls: number;
|
|
};
|
|
}
|
|
/**
|
|
* High-performance batch operations for S3-compatible storage
|
|
* Optimizes retrieval patterns for HNSW search operations
|
|
*/
|
|
export declare class BatchS3Operations {
|
|
private s3Client;
|
|
private bucketName;
|
|
private options;
|
|
constructor(s3Client: S3Client, bucketName: string, options?: BatchRetrievalOptions);
|
|
/**
|
|
* Batch retrieve HNSW nodes with intelligent prefetching
|
|
*/
|
|
batchGetNodes(nodeIds: string[], prefix?: string): Promise<BatchResult<HNSWNoun>>;
|
|
/**
|
|
* Parallel GetObject operations for small batches
|
|
*/
|
|
private parallelGetObjects;
|
|
/**
|
|
* Chunked parallel retrieval with intelligent batching
|
|
*/
|
|
private chunkedParallelGet;
|
|
/**
|
|
* List-based batch retrieval for large datasets
|
|
* Uses S3 ListObjects to reduce API calls
|
|
*/
|
|
private listBasedBatchGet;
|
|
/**
|
|
* Intelligent prefetch based on HNSW graph connectivity
|
|
*/
|
|
prefetchConnectedNodes(currentNodeIds: string[], connectionMap: Map<string, Set<string>>, prefix?: string): Promise<BatchResult<HNSWNoun>>;
|
|
/**
|
|
* S3 Select-based retrieval for filtered queries
|
|
*/
|
|
selectiveRetrieve(prefix: string, filter: {
|
|
vectorDimension?: number;
|
|
metadataKey?: string;
|
|
metadataValue?: any;
|
|
}): Promise<BatchResult<HNSWNoun>>;
|
|
/**
|
|
* Parse stored object from JSON string
|
|
*/
|
|
private parseStoredObject;
|
|
/**
|
|
* Utility function to chunk arrays
|
|
*/
|
|
private chunkArray;
|
|
}
|
|
export {};
|