chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,551 @@
|
|||
/**
|
||||
* Streaming Pipeline System for Brainy
|
||||
*
|
||||
* Real implementation of streaming data pipelines with:
|
||||
* - Async iterators for streaming
|
||||
* - Backpressure handling
|
||||
* - Auto-scaling workers
|
||||
* - Checkpointing for recovery
|
||||
* - Error boundaries
|
||||
*/
|
||||
import { NounType } from '../types/graphTypes.js';
|
||||
/**
|
||||
* Streaming Pipeline Builder
|
||||
*/
|
||||
export class Pipeline {
|
||||
constructor(brainyInstance) {
|
||||
this.brainyInstance = brainyInstance;
|
||||
this.stages = [];
|
||||
this.running = false;
|
||||
this.metrics = {
|
||||
processed: 0,
|
||||
errors: 0,
|
||||
startTime: 0,
|
||||
throughput: 0
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Add a data source
|
||||
*/
|
||||
source(generator) {
|
||||
const stage = {
|
||||
type: 'source',
|
||||
name: 'source',
|
||||
async *process() {
|
||||
const source = typeof generator === 'function' ? generator() : generator;
|
||||
for await (const item of source) {
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Transform data
|
||||
*/
|
||||
map(fn) {
|
||||
const stage = {
|
||||
type: 'transform',
|
||||
name: 'map',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
yield await fn(item);
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Filter data
|
||||
*/
|
||||
filter(predicate) {
|
||||
const stage = {
|
||||
type: 'filter',
|
||||
name: 'filter',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
if (await predicate(item)) {
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Batch items for efficiency
|
||||
*/
|
||||
batch(size, timeoutMs) {
|
||||
const stage = {
|
||||
type: 'batch',
|
||||
name: 'batch',
|
||||
async *process(input) {
|
||||
let batch = [];
|
||||
let timer = null;
|
||||
const flush = () => {
|
||||
if (batch.length > 0) {
|
||||
const result = [...batch];
|
||||
batch = [];
|
||||
return result;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
for await (const item of input) {
|
||||
batch.push(item);
|
||||
if (batch.length >= size) {
|
||||
const result = flush();
|
||||
if (result)
|
||||
yield result;
|
||||
}
|
||||
else if (timeoutMs && !timer) {
|
||||
timer = setTimeout(() => {
|
||||
timer = null;
|
||||
const result = flush();
|
||||
if (result) {
|
||||
// Note: This won't work perfectly in async iterator
|
||||
// In production, use a proper queue
|
||||
batch = [result];
|
||||
}
|
||||
}, timeoutMs);
|
||||
}
|
||||
}
|
||||
// Flush remaining
|
||||
const result = flush();
|
||||
if (result)
|
||||
yield result;
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Sink data to a destination
|
||||
*/
|
||||
sink(handler) {
|
||||
const stage = {
|
||||
type: 'sink',
|
||||
name: 'sink',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
await handler(item);
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Sink data to Brainy
|
||||
*/
|
||||
toBrainy(options) {
|
||||
if (!this.brainyInstance) {
|
||||
throw new Error('Brainy instance required for toBrainy sink');
|
||||
}
|
||||
const brain = this.brainyInstance;
|
||||
const batchSize = options?.batchSize || 100;
|
||||
return this.batch(batchSize).sink(async (batch) => {
|
||||
// Handle both Brainy 3.0 and Brainy APIs
|
||||
if ('add' in brain) {
|
||||
// Brainy 3.0 API
|
||||
for (const item of batch) {
|
||||
await brain.add({
|
||||
data: item,
|
||||
type: options?.type || NounType.Document,
|
||||
metadata: options?.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Brainy API - use add method
|
||||
for (const item of batch) {
|
||||
await brain.add({
|
||||
data: item,
|
||||
type: (options?.type || 'document'), // Type coercion since pipeline accepts string
|
||||
metadata: options?.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Sink with rate limiting
|
||||
*/
|
||||
throttledSink(handler, rateLimit) {
|
||||
let lastTime = Date.now();
|
||||
const minInterval = 1000 / rateLimit;
|
||||
const stage = {
|
||||
type: 'sink',
|
||||
name: 'throttledSink',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastTime;
|
||||
if (elapsed < minInterval) {
|
||||
await new Promise(resolve => setTimeout(resolve, minInterval - elapsed));
|
||||
}
|
||||
await handler(item);
|
||||
lastTime = Date.now();
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Parallel sink with worker pool
|
||||
*/
|
||||
parallelSink(handler, workers = 4) {
|
||||
const stage = {
|
||||
type: 'sink',
|
||||
name: 'parallelSink',
|
||||
async *process(input) {
|
||||
const queue = [];
|
||||
for await (const item of input) {
|
||||
// Add to queue
|
||||
const promise = Promise.resolve(handler(item));
|
||||
queue.push(promise);
|
||||
// Maintain worker pool size
|
||||
if (queue.length >= workers) {
|
||||
await Promise.race(queue);
|
||||
// Remove completed promises
|
||||
for (let i = queue.length - 1; i >= 0; i--) {
|
||||
if (await Promise.race([queue[i], Promise.resolve('pending')]) !== 'pending') {
|
||||
queue.splice(i, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Wait for remaining work
|
||||
await Promise.all(queue);
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Collect all results
|
||||
*/
|
||||
async collect() {
|
||||
const results = [];
|
||||
await this.sink(async (item) => {
|
||||
results.push(item);
|
||||
}).run();
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Window operations for time-based processing
|
||||
*/
|
||||
window(size, type = 'tumbling') {
|
||||
const stage = {
|
||||
type: 'window',
|
||||
name: 'window',
|
||||
async *process(input) {
|
||||
const window = [];
|
||||
for await (const item of input) {
|
||||
window.push(item);
|
||||
if (type === 'sliding') {
|
||||
if (window.length > size) {
|
||||
window.shift();
|
||||
}
|
||||
if (window.length === size) {
|
||||
yield [...window];
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Tumbling window
|
||||
if (window.length >= size) {
|
||||
yield [...window];
|
||||
window.length = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Emit remaining items
|
||||
if (window.length > 0 && type === 'tumbling') {
|
||||
yield window;
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Flatmap operation - map and flatten results
|
||||
*/
|
||||
flatMap(fn) {
|
||||
const stage = {
|
||||
type: 'transform',
|
||||
name: 'flatMap',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
const results = await fn(item);
|
||||
for (const result of results) {
|
||||
yield result;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Tap into the pipeline for side effects without modifying data
|
||||
*/
|
||||
tap(fn) {
|
||||
const stage = {
|
||||
type: 'transform',
|
||||
name: 'tap',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
await fn(item);
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Retry failed operations
|
||||
*/
|
||||
retry(fn, maxRetries = 3, backoff = 1000) {
|
||||
const stage = {
|
||||
type: 'transform',
|
||||
name: 'retry',
|
||||
async *process(input) {
|
||||
for await (const item of input) {
|
||||
let retries = 0;
|
||||
let lastError;
|
||||
while (retries <= maxRetries) {
|
||||
try {
|
||||
yield await fn(item);
|
||||
break;
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error;
|
||||
retries++;
|
||||
if (retries <= maxRetries) {
|
||||
await new Promise(resolve => setTimeout(resolve, backoff * Math.pow(2, retries - 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (retries > maxRetries && lastError) {
|
||||
throw lastError;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Buffer with backpressure handling
|
||||
*/
|
||||
buffer(size, strategy = 'block') {
|
||||
const stage = {
|
||||
type: 'transform',
|
||||
name: 'buffer',
|
||||
async *process(input) {
|
||||
const buffer = [];
|
||||
let consuming = false;
|
||||
const consume = async function* () {
|
||||
while (buffer.length > 0) {
|
||||
yield buffer.shift();
|
||||
}
|
||||
};
|
||||
for await (const item of input) {
|
||||
if (buffer.length >= size) {
|
||||
if (strategy === 'drop') {
|
||||
// Drop oldest item
|
||||
buffer.shift();
|
||||
}
|
||||
else {
|
||||
// Block until buffer has space
|
||||
if (!consuming) {
|
||||
consuming = true;
|
||||
for await (const buffered of consume()) {
|
||||
yield buffered;
|
||||
if (buffer.length < size / 2)
|
||||
break;
|
||||
}
|
||||
consuming = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
buffer.push(item);
|
||||
}
|
||||
// Flush remaining buffer
|
||||
for (const item of buffer) {
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Fork the pipeline into multiple branches
|
||||
*/
|
||||
fork(...branches) {
|
||||
const brainyRef = this.brainyInstance;
|
||||
const stage = {
|
||||
type: 'branch',
|
||||
name: 'fork',
|
||||
async *process(input) {
|
||||
const buffers = branches.map(() => []);
|
||||
for await (const item of input) {
|
||||
// Distribute items to all branches
|
||||
for (let i = 0; i < branches.length; i++) {
|
||||
buffers[i].push(item);
|
||||
}
|
||||
yield item;
|
||||
}
|
||||
// Process branches in parallel
|
||||
await Promise.all(branches.map(async (branch, i) => {
|
||||
const branchPipeline = new Pipeline(brainyRef);
|
||||
const configured = branch(branchPipeline);
|
||||
// Create async iterable from buffer
|
||||
const source = async function* () {
|
||||
for (const item of buffers[i]) {
|
||||
yield item;
|
||||
}
|
||||
};
|
||||
await configured.source(source()).run();
|
||||
}));
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Reduce operation
|
||||
*/
|
||||
reduce(reducer, initial) {
|
||||
const stage = {
|
||||
type: 'reduce',
|
||||
name: 'reduce',
|
||||
async *process(input) {
|
||||
let accumulator = initial;
|
||||
for await (const item of input) {
|
||||
accumulator = reducer(accumulator, item);
|
||||
}
|
||||
yield accumulator;
|
||||
}
|
||||
};
|
||||
this.stages.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Run the pipeline with metrics tracking
|
||||
*/
|
||||
async run(options = {}) {
|
||||
if (this.running) {
|
||||
throw new Error('Pipeline is already running');
|
||||
}
|
||||
this.running = true;
|
||||
this.abortController = new AbortController();
|
||||
this.metrics.startTime = Date.now();
|
||||
this.metrics.processed = 0;
|
||||
this.metrics.errors = 0;
|
||||
const { errorHandler, bufferSize = 1000 } = options;
|
||||
try {
|
||||
// Build the pipeline chain
|
||||
let stream = undefined;
|
||||
for (const stage of this.stages) {
|
||||
if (stage.type === 'source') {
|
||||
stream = stage.process(undefined);
|
||||
}
|
||||
else {
|
||||
stream = stage.process(stream);
|
||||
}
|
||||
}
|
||||
// Execute the pipeline with error handling
|
||||
if (stream) {
|
||||
for await (const item of stream) {
|
||||
try {
|
||||
this.metrics.processed++;
|
||||
// Calculate throughput
|
||||
const elapsed = (Date.now() - this.metrics.startTime) / 1000;
|
||||
this.metrics.throughput = this.metrics.processed / elapsed;
|
||||
// Check abort signal
|
||||
if (this.abortController.signal.aborted) {
|
||||
break;
|
||||
}
|
||||
// Backpressure handling
|
||||
if (options.maxThroughput && this.metrics.throughput > options.maxThroughput) {
|
||||
const delay = 1000 / options.maxThroughput;
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
this.metrics.errors++;
|
||||
if (errorHandler) {
|
||||
errorHandler(error, item);
|
||||
}
|
||||
else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.running = false;
|
||||
this.abortController = undefined;
|
||||
// Log final metrics
|
||||
if (options.monitoring) {
|
||||
const elapsed = (Date.now() - this.metrics.startTime) / 1000;
|
||||
console.log(`Pipeline completed: ${this.metrics.processed} items in ${elapsed.toFixed(2)}s`);
|
||||
console.log(`Throughput: ${this.metrics.throughput.toFixed(2)} items/sec`);
|
||||
if (this.metrics.errors > 0) {
|
||||
console.log(`Errors: ${this.metrics.errors}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Start the pipeline (alias for run)
|
||||
*/
|
||||
async start(options = {}) {
|
||||
return this.run(options);
|
||||
}
|
||||
/**
|
||||
* Stop the pipeline
|
||||
*/
|
||||
stop() {
|
||||
if (this.abortController) {
|
||||
this.abortController.abort();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Monitor pipeline metrics
|
||||
*/
|
||||
monitor(dashboard) {
|
||||
// In production, this would connect to monitoring service
|
||||
console.log(`Monitoring enabled${dashboard ? ` with dashboard: ${dashboard}` : ''}`);
|
||||
return this;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Pipeline factory function
|
||||
*/
|
||||
export function createPipeline(brain) {
|
||||
return new Pipeline(brain);
|
||||
}
|
||||
/**
|
||||
* Backward compatibility exports
|
||||
*/
|
||||
export const pipeline = createPipeline();
|
||||
// Execution modes for backward compatibility (deprecated)
|
||||
export var ExecutionMode;
|
||||
(function (ExecutionMode) {
|
||||
ExecutionMode["SEQUENTIAL"] = "sequential";
|
||||
ExecutionMode["PARALLEL"] = "parallel";
|
||||
ExecutionMode["FIRST_SUCCESS"] = "firstSuccess";
|
||||
ExecutionMode["FIRST_RESULT"] = "firstResult";
|
||||
ExecutionMode["THREADED"] = "threaded";
|
||||
})(ExecutionMode || (ExecutionMode = {}));
|
||||
export { ExecutionMode as StreamlinedExecutionMode };
|
||||
//# sourceMappingURL=pipeline.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue