Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
425
dist/storage/readOnlyOptimizations.js
vendored
Normal file
425
dist/storage/readOnlyOptimizations.js
vendored
Normal file
|
|
@ -0,0 +1,425 @@
|
|||
/**
|
||||
* Read-Only Storage Optimizations for Production Deployments
|
||||
* Implements compression, memory-mapping, and pre-built index segments
|
||||
*/
|
||||
// Compression types supported
|
||||
var CompressionType;
|
||||
(function (CompressionType) {
|
||||
CompressionType["NONE"] = "none";
|
||||
CompressionType["GZIP"] = "gzip";
|
||||
CompressionType["BROTLI"] = "brotli";
|
||||
CompressionType["QUANTIZATION"] = "quantization";
|
||||
CompressionType["HYBRID"] = "hybrid";
|
||||
})(CompressionType || (CompressionType = {}));
|
||||
// Vector quantization methods
|
||||
var QuantizationType;
|
||||
(function (QuantizationType) {
|
||||
QuantizationType["SCALAR"] = "scalar";
|
||||
QuantizationType["PRODUCT"] = "product";
|
||||
QuantizationType["BINARY"] = "binary"; // Binary quantization
|
||||
})(QuantizationType || (QuantizationType = {}));
|
||||
/**
|
||||
* Read-only storage optimizations for high-performance production deployments
|
||||
*/
|
||||
export class ReadOnlyOptimizations {
|
||||
constructor(config = {}) {
|
||||
this.segments = new Map();
|
||||
this.compressionStats = {
|
||||
originalSize: 0,
|
||||
compressedSize: 0,
|
||||
compressionRatio: 0,
|
||||
decompressionTime: 0
|
||||
};
|
||||
// Quantization codebooks for vector compression
|
||||
this.quantizationCodebooks = new Map();
|
||||
// Memory-mapped buffers for large datasets
|
||||
this.memoryMappedBuffers = new Map();
|
||||
this.config = {
|
||||
prebuiltIndexPath: '',
|
||||
memoryMapped: true,
|
||||
compression: {
|
||||
vectorCompression: CompressionType.QUANTIZATION,
|
||||
metadataCompression: CompressionType.GZIP,
|
||||
quantizationType: QuantizationType.SCALAR,
|
||||
quantizationBits: 8,
|
||||
compressionLevel: 6
|
||||
},
|
||||
segmentSize: 10000, // 10k nodes per segment
|
||||
prefetchSegments: 3,
|
||||
cacheIndexInMemory: false,
|
||||
...config
|
||||
};
|
||||
if (config.compression) {
|
||||
this.config.compression = { ...this.config.compression, ...config.compression };
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Compress vector data using specified compression method
|
||||
*/
|
||||
async compressVector(vector, segmentId) {
|
||||
const startTime = Date.now();
|
||||
let compressedData;
|
||||
switch (this.config.compression.vectorCompression) {
|
||||
case CompressionType.QUANTIZATION:
|
||||
compressedData = await this.quantizeVector(vector, segmentId);
|
||||
break;
|
||||
case CompressionType.GZIP:
|
||||
const gzipBuffer = new Float32Array(vector).buffer;
|
||||
compressedData = await this.gzipCompress(gzipBuffer.slice(0));
|
||||
break;
|
||||
case CompressionType.BROTLI:
|
||||
const brotliBuffer = new Float32Array(vector).buffer;
|
||||
compressedData = await this.brotliCompress(brotliBuffer.slice(0));
|
||||
break;
|
||||
case CompressionType.HYBRID:
|
||||
// First quantize, then compress
|
||||
const quantized = await this.quantizeVector(vector, segmentId);
|
||||
compressedData = await this.gzipCompress(quantized);
|
||||
break;
|
||||
default:
|
||||
const defaultBuffer = new Float32Array(vector).buffer;
|
||||
compressedData = defaultBuffer.slice(0);
|
||||
break;
|
||||
}
|
||||
// Update compression statistics
|
||||
const originalSize = vector.length * 4; // 4 bytes per float32
|
||||
this.compressionStats.originalSize += originalSize;
|
||||
this.compressionStats.compressedSize += compressedData.byteLength;
|
||||
this.compressionStats.decompressionTime += Date.now() - startTime;
|
||||
this.updateCompressionRatio();
|
||||
return compressedData;
|
||||
}
|
||||
/**
|
||||
* Decompress vector data
|
||||
*/
|
||||
async decompressVector(compressedData, segmentId, originalDimension) {
|
||||
switch (this.config.compression.vectorCompression) {
|
||||
case CompressionType.QUANTIZATION:
|
||||
return this.dequantizeVector(compressedData, segmentId, originalDimension);
|
||||
case CompressionType.GZIP:
|
||||
const gzipDecompressed = await this.gzipDecompress(compressedData);
|
||||
return Array.from(new Float32Array(gzipDecompressed));
|
||||
case CompressionType.BROTLI:
|
||||
const brotliDecompressed = await this.brotliDecompress(compressedData);
|
||||
return Array.from(new Float32Array(brotliDecompressed));
|
||||
case CompressionType.HYBRID:
|
||||
const gzipStage = await this.gzipDecompress(compressedData);
|
||||
return this.dequantizeVector(gzipStage, segmentId, originalDimension);
|
||||
default:
|
||||
return Array.from(new Float32Array(compressedData));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Scalar quantization of vectors to 8-bit integers
|
||||
*/
|
||||
async quantizeVector(vector, segmentId) {
|
||||
let codebook = this.quantizationCodebooks.get(segmentId);
|
||||
if (!codebook) {
|
||||
// Create codebook (min/max values for scaling)
|
||||
const min = Math.min(...vector);
|
||||
const max = Math.max(...vector);
|
||||
codebook = new Float32Array([min, max]);
|
||||
this.quantizationCodebooks.set(segmentId, codebook);
|
||||
}
|
||||
const [min, max] = codebook;
|
||||
const scale = (max - min) / 255; // 8-bit quantization
|
||||
const quantized = new Uint8Array(vector.length);
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
quantized[i] = Math.round((vector[i] - min) / scale);
|
||||
}
|
||||
// Store codebook with quantized data
|
||||
const result = new ArrayBuffer(quantized.byteLength + codebook.byteLength);
|
||||
const resultView = new Uint8Array(result);
|
||||
// First 8 bytes: codebook (min, max as float32)
|
||||
resultView.set(new Uint8Array(codebook.buffer), 0);
|
||||
// Remaining bytes: quantized vector
|
||||
resultView.set(quantized, codebook.byteLength);
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Dequantize 8-bit vectors back to float32
|
||||
*/
|
||||
dequantizeVector(quantizedData, segmentId, dimension) {
|
||||
const dataView = new Uint8Array(quantizedData);
|
||||
// Extract codebook (first 8 bytes)
|
||||
const codebookBytes = dataView.slice(0, 8);
|
||||
const codebook = new Float32Array(codebookBytes.buffer);
|
||||
const [min, max] = codebook;
|
||||
// Extract quantized vector
|
||||
const quantized = dataView.slice(8);
|
||||
const scale = (max - min) / 255;
|
||||
const result = [];
|
||||
for (let i = 0; i < dimension; i++) {
|
||||
result[i] = min + quantized[i] * scale;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* GZIP compression using browser/Node.js APIs
|
||||
*/
|
||||
async gzipCompress(data) {
|
||||
if (typeof CompressionStream !== 'undefined') {
|
||||
// Browser environment
|
||||
const stream = new CompressionStream('gzip');
|
||||
const writer = stream.writable.getWriter();
|
||||
const reader = stream.readable.getReader();
|
||||
writer.write(new Uint8Array(data));
|
||||
writer.close();
|
||||
const chunks = [];
|
||||
let result = await reader.read();
|
||||
while (!result.done) {
|
||||
chunks.push(result.value);
|
||||
result = await reader.read();
|
||||
}
|
||||
// Combine chunks
|
||||
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return combined.buffer;
|
||||
}
|
||||
else {
|
||||
// Node.js environment - would use zlib
|
||||
console.warn('GZIP compression not available, returning original data');
|
||||
return data;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* GZIP decompression
|
||||
*/
|
||||
async gzipDecompress(compressedData) {
|
||||
if (typeof DecompressionStream !== 'undefined') {
|
||||
// Browser environment
|
||||
const stream = new DecompressionStream('gzip');
|
||||
const writer = stream.writable.getWriter();
|
||||
const reader = stream.readable.getReader();
|
||||
writer.write(new Uint8Array(compressedData));
|
||||
writer.close();
|
||||
const chunks = [];
|
||||
let result = await reader.read();
|
||||
while (!result.done) {
|
||||
chunks.push(result.value);
|
||||
result = await reader.read();
|
||||
}
|
||||
// Combine chunks
|
||||
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return combined.buffer;
|
||||
}
|
||||
else {
|
||||
console.warn('GZIP decompression not available, returning original data');
|
||||
return compressedData;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Brotli compression (placeholder - similar to GZIP)
|
||||
*/
|
||||
async brotliCompress(data) {
|
||||
// Would implement Brotli compression here
|
||||
console.warn('Brotli compression not implemented, falling back to GZIP');
|
||||
return this.gzipCompress(data);
|
||||
}
|
||||
/**
|
||||
* Brotli decompression (placeholder)
|
||||
*/
|
||||
async brotliDecompress(compressedData) {
|
||||
console.warn('Brotli decompression not implemented, falling back to GZIP');
|
||||
return this.gzipDecompress(compressedData);
|
||||
}
|
||||
/**
|
||||
* Create prebuilt index segments for faster loading
|
||||
*/
|
||||
async createPrebuiltSegments(nodes, outputPath) {
|
||||
const segments = [];
|
||||
const segmentSize = this.config.segmentSize;
|
||||
console.log(`Creating ${Math.ceil(nodes.length / segmentSize)} prebuilt segments`);
|
||||
for (let i = 0; i < nodes.length; i += segmentSize) {
|
||||
const segmentNodes = nodes.slice(i, i + segmentSize);
|
||||
const segmentId = `segment_${Math.floor(i / segmentSize)}`;
|
||||
const segment = {
|
||||
id: segmentId,
|
||||
nodeCount: segmentNodes.length,
|
||||
vectorDimension: segmentNodes[0]?.vector.length || 0,
|
||||
compression: this.config.compression.vectorCompression,
|
||||
localPath: `${outputPath}/${segmentId}.dat`,
|
||||
loadedInMemory: false,
|
||||
lastAccessed: 0
|
||||
};
|
||||
// Compress and serialize segment data
|
||||
const compressedData = await this.compressSegment(segmentNodes);
|
||||
// In a real implementation, you would write this to disk/S3
|
||||
console.log(`Created segment ${segmentId} with ${compressedData.byteLength} bytes`);
|
||||
segments.push(segment);
|
||||
this.segments.set(segmentId, segment);
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
/**
|
||||
* Compress an entire segment of nodes
|
||||
*/
|
||||
async compressSegment(nodes) {
|
||||
const serialized = JSON.stringify(nodes.map(node => ({
|
||||
id: node.id,
|
||||
vector: node.vector,
|
||||
connections: this.serializeConnections(node.connections)
|
||||
})));
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(serialized);
|
||||
// Apply metadata compression
|
||||
switch (this.config.compression.metadataCompression) {
|
||||
case CompressionType.GZIP:
|
||||
return this.gzipCompress(data.buffer.slice(0));
|
||||
case CompressionType.BROTLI:
|
||||
return this.brotliCompress(data.buffer.slice(0));
|
||||
default:
|
||||
return data.buffer.slice(0);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Load a segment from storage with caching
|
||||
*/
|
||||
async loadSegment(segmentId) {
|
||||
const segment = this.segments.get(segmentId);
|
||||
if (!segment) {
|
||||
throw new Error(`Segment ${segmentId} not found`);
|
||||
}
|
||||
segment.lastAccessed = Date.now();
|
||||
// Check if segment is already loaded in memory
|
||||
if (segment.loadedInMemory && this.memoryMappedBuffers.has(segmentId)) {
|
||||
return this.deserializeSegment(this.memoryMappedBuffers.get(segmentId));
|
||||
}
|
||||
// Load from storage (S3, disk, etc.)
|
||||
const compressedData = await this.loadSegmentFromStorage(segment);
|
||||
// Cache in memory if configured
|
||||
if (this.config.cacheIndexInMemory) {
|
||||
this.memoryMappedBuffers.set(segmentId, compressedData);
|
||||
segment.loadedInMemory = true;
|
||||
}
|
||||
return this.deserializeSegment(compressedData);
|
||||
}
|
||||
/**
|
||||
* Load segment data from storage
|
||||
*/
|
||||
async loadSegmentFromStorage(segment) {
|
||||
// This would integrate with your S3 storage adapter
|
||||
// For now, return a placeholder
|
||||
console.log(`Loading segment ${segment.id} from storage`);
|
||||
return new ArrayBuffer(0);
|
||||
}
|
||||
/**
|
||||
* Deserialize and decompress segment data
|
||||
*/
|
||||
async deserializeSegment(compressedData) {
|
||||
// Decompress metadata
|
||||
let decompressed;
|
||||
switch (this.config.compression.metadataCompression) {
|
||||
case CompressionType.GZIP:
|
||||
decompressed = await this.gzipDecompress(compressedData);
|
||||
break;
|
||||
case CompressionType.BROTLI:
|
||||
decompressed = await this.brotliDecompress(compressedData);
|
||||
break;
|
||||
default:
|
||||
decompressed = compressedData;
|
||||
break;
|
||||
}
|
||||
// Parse JSON
|
||||
const decoder = new TextDecoder();
|
||||
const jsonStr = decoder.decode(decompressed);
|
||||
const parsed = JSON.parse(jsonStr);
|
||||
// Reconstruct HNSWNoun objects
|
||||
return parsed.map((item) => ({
|
||||
id: item.id,
|
||||
vector: item.vector,
|
||||
connections: this.deserializeConnections(item.connections)
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Serialize connections Map for storage
|
||||
*/
|
||||
serializeConnections(connections) {
|
||||
const result = {};
|
||||
for (const [level, nodeIds] of connections.entries()) {
|
||||
result[level.toString()] = Array.from(nodeIds);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Deserialize connections from storage format
|
||||
*/
|
||||
deserializeConnections(serialized) {
|
||||
const result = new Map();
|
||||
for (const [levelStr, nodeIds] of Object.entries(serialized)) {
|
||||
result.set(parseInt(levelStr), new Set(nodeIds));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Prefetch segments based on access patterns
|
||||
*/
|
||||
async prefetchSegments(currentSegmentId) {
|
||||
const segment = this.segments.get(currentSegmentId);
|
||||
if (!segment)
|
||||
return;
|
||||
// Simple prefetching strategy - load adjacent segments
|
||||
const segmentNumber = parseInt(currentSegmentId.split('_')[1]);
|
||||
const toPrefetch = [];
|
||||
for (let i = 1; i <= this.config.prefetchSegments; i++) {
|
||||
const nextId = `segment_${segmentNumber + i}`;
|
||||
const prevId = `segment_${segmentNumber - i}`;
|
||||
if (this.segments.has(nextId) && !this.memoryMappedBuffers.has(nextId)) {
|
||||
toPrefetch.push(nextId);
|
||||
}
|
||||
if (this.segments.has(prevId) && !this.memoryMappedBuffers.has(prevId)) {
|
||||
toPrefetch.push(prevId);
|
||||
}
|
||||
}
|
||||
// Prefetch in background
|
||||
for (const segmentId of toPrefetch) {
|
||||
this.loadSegment(segmentId).catch(error => {
|
||||
console.warn(`Failed to prefetch segment ${segmentId}:`, error);
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update compression statistics
|
||||
*/
|
||||
updateCompressionRatio() {
|
||||
if (this.compressionStats.originalSize > 0) {
|
||||
this.compressionStats.compressionRatio =
|
||||
this.compressionStats.compressedSize / this.compressionStats.originalSize;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get compression statistics
|
||||
*/
|
||||
getCompressionStats() {
|
||||
const memoryUsage = Array.from(this.memoryMappedBuffers.values())
|
||||
.reduce((sum, buffer) => sum + buffer.byteLength, 0);
|
||||
return {
|
||||
...this.compressionStats,
|
||||
segmentCount: this.segments.size,
|
||||
memoryUsage
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Cleanup memory-mapped buffers
|
||||
*/
|
||||
cleanup() {
|
||||
this.memoryMappedBuffers.clear();
|
||||
this.quantizationCodebooks.clear();
|
||||
// Mark all segments as not loaded
|
||||
for (const segment of this.segments.values()) {
|
||||
segment.loadedInMemory = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=readOnlyOptimizations.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue