chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,429 @@
|
|||
/**
|
||||
* Read-Only Storage Optimizations for Production Deployments
|
||||
* Implements compression, memory-mapping, and pre-built index segments
|
||||
*/
|
||||
// Compression types supported
|
||||
var CompressionType;
|
||||
(function (CompressionType) {
|
||||
CompressionType["NONE"] = "none";
|
||||
CompressionType["GZIP"] = "gzip";
|
||||
CompressionType["BROTLI"] = "brotli";
|
||||
CompressionType["QUANTIZATION"] = "quantization";
|
||||
CompressionType["HYBRID"] = "hybrid";
|
||||
})(CompressionType || (CompressionType = {}));
|
||||
// Vector quantization methods
|
||||
var QuantizationType;
|
||||
(function (QuantizationType) {
|
||||
QuantizationType["SCALAR"] = "scalar";
|
||||
QuantizationType["PRODUCT"] = "product";
|
||||
QuantizationType["BINARY"] = "binary"; // Binary quantization
|
||||
})(QuantizationType || (QuantizationType = {}));
|
||||
/**
|
||||
* Read-only storage optimizations for high-performance production deployments
|
||||
*/
|
||||
export class ReadOnlyOptimizations {
|
||||
constructor(config = {}) {
|
||||
this.segments = new Map();
|
||||
this.compressionStats = {
|
||||
originalSize: 0,
|
||||
compressedSize: 0,
|
||||
compressionRatio: 0,
|
||||
decompressionTime: 0
|
||||
};
|
||||
// Quantization codebooks for vector compression
|
||||
this.quantizationCodebooks = new Map();
|
||||
// Memory-mapped buffers for large datasets
|
||||
this.memoryMappedBuffers = new Map();
|
||||
this.config = {
|
||||
prebuiltIndexPath: '',
|
||||
memoryMapped: true,
|
||||
compression: {
|
||||
vectorCompression: CompressionType.QUANTIZATION,
|
||||
metadataCompression: CompressionType.GZIP,
|
||||
quantizationType: QuantizationType.SCALAR,
|
||||
quantizationBits: 8,
|
||||
compressionLevel: 6
|
||||
},
|
||||
segmentSize: 10000, // 10k nodes per segment
|
||||
prefetchSegments: 3,
|
||||
cacheIndexInMemory: false,
|
||||
...config
|
||||
};
|
||||
if (config.compression) {
|
||||
this.config.compression = { ...this.config.compression, ...config.compression };
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Compress vector data using specified compression method
|
||||
*/
|
||||
async compressVector(vector, segmentId) {
|
||||
const startTime = Date.now();
|
||||
let compressedData;
|
||||
switch (this.config.compression.vectorCompression) {
|
||||
case CompressionType.QUANTIZATION:
|
||||
compressedData = await this.quantizeVector(vector, segmentId);
|
||||
break;
|
||||
case CompressionType.GZIP:
|
||||
const gzipBuffer = new Float32Array(vector).buffer;
|
||||
compressedData = await this.gzipCompress(gzipBuffer.slice(0));
|
||||
break;
|
||||
case CompressionType.BROTLI:
|
||||
const brotliBuffer = new Float32Array(vector).buffer;
|
||||
compressedData = await this.brotliCompress(brotliBuffer.slice(0));
|
||||
break;
|
||||
case CompressionType.HYBRID:
|
||||
// First quantize, then compress
|
||||
const quantized = await this.quantizeVector(vector, segmentId);
|
||||
compressedData = await this.gzipCompress(quantized);
|
||||
break;
|
||||
default:
|
||||
const defaultBuffer = new Float32Array(vector).buffer;
|
||||
compressedData = defaultBuffer.slice(0);
|
||||
break;
|
||||
}
|
||||
// Update compression statistics
|
||||
const originalSize = vector.length * 4; // 4 bytes per float32
|
||||
this.compressionStats.originalSize += originalSize;
|
||||
this.compressionStats.compressedSize += compressedData.byteLength;
|
||||
this.compressionStats.decompressionTime += Date.now() - startTime;
|
||||
this.updateCompressionRatio();
|
||||
return compressedData;
|
||||
}
|
||||
/**
|
||||
* Decompress vector data
|
||||
*/
|
||||
async decompressVector(compressedData, segmentId, originalDimension) {
|
||||
switch (this.config.compression.vectorCompression) {
|
||||
case CompressionType.QUANTIZATION:
|
||||
return this.dequantizeVector(compressedData, segmentId, originalDimension);
|
||||
case CompressionType.GZIP:
|
||||
const gzipDecompressed = await this.gzipDecompress(compressedData);
|
||||
return Array.from(new Float32Array(gzipDecompressed));
|
||||
case CompressionType.BROTLI:
|
||||
const brotliDecompressed = await this.brotliDecompress(compressedData);
|
||||
return Array.from(new Float32Array(brotliDecompressed));
|
||||
case CompressionType.HYBRID:
|
||||
const gzipStage = await this.gzipDecompress(compressedData);
|
||||
return this.dequantizeVector(gzipStage, segmentId, originalDimension);
|
||||
default:
|
||||
return Array.from(new Float32Array(compressedData));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Scalar quantization of vectors to 8-bit integers
|
||||
*/
|
||||
async quantizeVector(vector, segmentId) {
|
||||
let codebook = this.quantizationCodebooks.get(segmentId);
|
||||
if (!codebook) {
|
||||
// Create codebook (min/max values for scaling)
|
||||
const min = Math.min(...vector);
|
||||
const max = Math.max(...vector);
|
||||
codebook = new Float32Array([min, max]);
|
||||
this.quantizationCodebooks.set(segmentId, codebook);
|
||||
}
|
||||
const [min, max] = codebook;
|
||||
const scale = (max - min) / 255; // 8-bit quantization
|
||||
const quantized = new Uint8Array(vector.length);
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
quantized[i] = Math.round((vector[i] - min) / scale);
|
||||
}
|
||||
// Store codebook with quantized data
|
||||
const result = new ArrayBuffer(quantized.byteLength + codebook.byteLength);
|
||||
const resultView = new Uint8Array(result);
|
||||
// First 8 bytes: codebook (min, max as float32)
|
||||
resultView.set(new Uint8Array(codebook.buffer), 0);
|
||||
// Remaining bytes: quantized vector
|
||||
resultView.set(quantized, codebook.byteLength);
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Dequantize 8-bit vectors back to float32
|
||||
*/
|
||||
dequantizeVector(quantizedData, segmentId, dimension) {
|
||||
const dataView = new Uint8Array(quantizedData);
|
||||
// Extract codebook (first 8 bytes)
|
||||
const codebookBytes = dataView.slice(0, 8);
|
||||
const codebook = new Float32Array(codebookBytes.buffer);
|
||||
const [min, max] = codebook;
|
||||
// Extract quantized vector
|
||||
const quantized = dataView.slice(8);
|
||||
const scale = (max - min) / 255;
|
||||
const result = [];
|
||||
for (let i = 0; i < dimension; i++) {
|
||||
result[i] = min + quantized[i] * scale;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* GZIP compression using browser/Node.js APIs
|
||||
*/
|
||||
async gzipCompress(data) {
|
||||
if (typeof CompressionStream !== 'undefined') {
|
||||
// Browser environment
|
||||
const stream = new CompressionStream('gzip');
|
||||
const writer = stream.writable.getWriter();
|
||||
const reader = stream.readable.getReader();
|
||||
writer.write(new Uint8Array(data));
|
||||
writer.close();
|
||||
const chunks = [];
|
||||
let result = await reader.read();
|
||||
while (!result.done) {
|
||||
chunks.push(result.value);
|
||||
result = await reader.read();
|
||||
}
|
||||
// Combine chunks
|
||||
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return combined.buffer;
|
||||
}
|
||||
else {
|
||||
// Node.js environment - would use zlib
|
||||
console.warn('GZIP compression not available, returning original data');
|
||||
return data;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* GZIP decompression
|
||||
*/
|
||||
async gzipDecompress(compressedData) {
|
||||
if (typeof DecompressionStream !== 'undefined') {
|
||||
// Browser environment
|
||||
const stream = new DecompressionStream('gzip');
|
||||
const writer = stream.writable.getWriter();
|
||||
const reader = stream.readable.getReader();
|
||||
writer.write(new Uint8Array(compressedData));
|
||||
writer.close();
|
||||
const chunks = [];
|
||||
let result = await reader.read();
|
||||
while (!result.done) {
|
||||
chunks.push(result.value);
|
||||
result = await reader.read();
|
||||
}
|
||||
// Combine chunks
|
||||
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return combined.buffer;
|
||||
}
|
||||
else {
|
||||
console.warn('GZIP decompression not available, returning original data');
|
||||
return compressedData;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Brotli compression (placeholder - similar to GZIP)
|
||||
*/
|
||||
async brotliCompress(data) {
|
||||
// Would implement Brotli compression here
|
||||
console.warn('Brotli compression not implemented, falling back to GZIP');
|
||||
return this.gzipCompress(data);
|
||||
}
|
||||
/**
|
||||
* Brotli decompression (placeholder)
|
||||
*/
|
||||
async brotliDecompress(compressedData) {
|
||||
console.warn('Brotli decompression not implemented, falling back to GZIP');
|
||||
return this.gzipDecompress(compressedData);
|
||||
}
|
||||
/**
|
||||
* Create prebuilt index segments for faster loading
|
||||
*/
|
||||
async createPrebuiltSegments(nodes, outputPath) {
|
||||
const segments = [];
|
||||
const segmentSize = this.config.segmentSize;
|
||||
console.log(`Creating ${Math.ceil(nodes.length / segmentSize)} prebuilt segments`);
|
||||
for (let i = 0; i < nodes.length; i += segmentSize) {
|
||||
const segmentNodes = nodes.slice(i, i + segmentSize);
|
||||
const segmentId = `segment_${Math.floor(i / segmentSize)}`;
|
||||
const segment = {
|
||||
id: segmentId,
|
||||
nodeCount: segmentNodes.length,
|
||||
vectorDimension: segmentNodes[0]?.vector.length || 0,
|
||||
compression: this.config.compression.vectorCompression,
|
||||
localPath: `${outputPath}/${segmentId}.dat`,
|
||||
loadedInMemory: false,
|
||||
lastAccessed: 0
|
||||
};
|
||||
// Compress and serialize segment data
|
||||
const compressedData = await this.compressSegment(segmentNodes);
|
||||
// In a real implementation, you would write this to disk/S3
|
||||
console.log(`Created segment ${segmentId} with ${compressedData.byteLength} bytes`);
|
||||
segments.push(segment);
|
||||
this.segments.set(segmentId, segment);
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
/**
|
||||
* Compress an entire segment of nodes
|
||||
*/
|
||||
async compressSegment(nodes) {
|
||||
const serialized = JSON.stringify(nodes.map(node => ({
|
||||
id: node.id,
|
||||
vector: node.vector,
|
||||
connections: this.serializeConnections(node.connections)
|
||||
})));
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(serialized);
|
||||
// Apply metadata compression
|
||||
switch (this.config.compression.metadataCompression) {
|
||||
case CompressionType.GZIP:
|
||||
return this.gzipCompress(data.buffer.slice(0));
|
||||
case CompressionType.BROTLI:
|
||||
return this.brotliCompress(data.buffer.slice(0));
|
||||
default:
|
||||
return data.buffer.slice(0);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Load a segment from storage with caching
|
||||
*/
|
||||
async loadSegment(segmentId) {
|
||||
const segment = this.segments.get(segmentId);
|
||||
if (!segment) {
|
||||
throw new Error(`Segment ${segmentId} not found`);
|
||||
}
|
||||
segment.lastAccessed = Date.now();
|
||||
// Check if segment is already loaded in memory
|
||||
if (segment.loadedInMemory && this.memoryMappedBuffers.has(segmentId)) {
|
||||
return this.deserializeSegment(this.memoryMappedBuffers.get(segmentId));
|
||||
}
|
||||
// Load from storage (S3, disk, etc.)
|
||||
const compressedData = await this.loadSegmentFromStorage(segment);
|
||||
// Cache in memory if configured
|
||||
if (this.config.cacheIndexInMemory) {
|
||||
this.memoryMappedBuffers.set(segmentId, compressedData);
|
||||
segment.loadedInMemory = true;
|
||||
}
|
||||
return this.deserializeSegment(compressedData);
|
||||
}
|
||||
/**
|
||||
* Load segment data from storage
|
||||
*/
|
||||
async loadSegmentFromStorage(segment) {
|
||||
// Load segment from memory-mapped buffer if available
|
||||
const cached = this.memoryMappedBuffers.get(segment.id);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
// In production, this would load from actual storage (S3, file system, etc)
|
||||
// For now, throw an error to indicate missing implementation
|
||||
throw new Error(`Segment loading not implemented. Segment ${segment.id} requires storage integration.`);
|
||||
}
|
||||
/**
|
||||
* Deserialize and decompress segment data
|
||||
*/
|
||||
async deserializeSegment(compressedData) {
|
||||
// Decompress metadata
|
||||
let decompressed;
|
||||
switch (this.config.compression.metadataCompression) {
|
||||
case CompressionType.GZIP:
|
||||
decompressed = await this.gzipDecompress(compressedData);
|
||||
break;
|
||||
case CompressionType.BROTLI:
|
||||
decompressed = await this.brotliDecompress(compressedData);
|
||||
break;
|
||||
default:
|
||||
decompressed = compressedData;
|
||||
break;
|
||||
}
|
||||
// Parse JSON
|
||||
const decoder = new TextDecoder();
|
||||
const jsonStr = decoder.decode(decompressed);
|
||||
const parsed = JSON.parse(jsonStr);
|
||||
// Reconstruct HNSWNoun objects
|
||||
return parsed.map((item) => ({
|
||||
id: item.id,
|
||||
vector: item.vector,
|
||||
connections: this.deserializeConnections(item.connections)
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Serialize connections Map for storage
|
||||
*/
|
||||
serializeConnections(connections) {
|
||||
const result = {};
|
||||
for (const [level, nodeIds] of connections.entries()) {
|
||||
result[level.toString()] = Array.from(nodeIds);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Deserialize connections from storage format
|
||||
*/
|
||||
deserializeConnections(serialized) {
|
||||
const result = new Map();
|
||||
for (const [levelStr, nodeIds] of Object.entries(serialized)) {
|
||||
result.set(parseInt(levelStr), new Set(nodeIds));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Prefetch segments based on access patterns
|
||||
*/
|
||||
async prefetchSegments(currentSegmentId) {
|
||||
const segment = this.segments.get(currentSegmentId);
|
||||
if (!segment)
|
||||
return;
|
||||
// Simple prefetching strategy - load adjacent segments
|
||||
const segmentNumber = parseInt(currentSegmentId.split('_')[1]);
|
||||
const toPrefetch = [];
|
||||
for (let i = 1; i <= this.config.prefetchSegments; i++) {
|
||||
const nextId = `segment_${segmentNumber + i}`;
|
||||
const prevId = `segment_${segmentNumber - i}`;
|
||||
if (this.segments.has(nextId) && !this.memoryMappedBuffers.has(nextId)) {
|
||||
toPrefetch.push(nextId);
|
||||
}
|
||||
if (this.segments.has(prevId) && !this.memoryMappedBuffers.has(prevId)) {
|
||||
toPrefetch.push(prevId);
|
||||
}
|
||||
}
|
||||
// Prefetch in background
|
||||
for (const segmentId of toPrefetch) {
|
||||
this.loadSegment(segmentId).catch(error => {
|
||||
console.warn(`Failed to prefetch segment ${segmentId}:`, error);
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update compression statistics
|
||||
*/
|
||||
updateCompressionRatio() {
|
||||
if (this.compressionStats.originalSize > 0) {
|
||||
this.compressionStats.compressionRatio =
|
||||
this.compressionStats.compressedSize / this.compressionStats.originalSize;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get compression statistics
|
||||
*/
|
||||
getCompressionStats() {
|
||||
const memoryUsage = Array.from(this.memoryMappedBuffers.values())
|
||||
.reduce((sum, buffer) => sum + buffer.byteLength, 0);
|
||||
return {
|
||||
...this.compressionStats,
|
||||
segmentCount: this.segments.size,
|
||||
memoryUsage
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Cleanup memory-mapped buffers
|
||||
*/
|
||||
cleanup() {
|
||||
this.memoryMappedBuffers.clear();
|
||||
this.quantizationCodebooks.clear();
|
||||
// Mark all segments as not loaded
|
||||
for (const segment of this.segments.values()) {
|
||||
segment.loadedInMemory = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=readOnlyOptimizations.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue