feat: implement SequentialPipeline with threading and WebSocket support
Added the `SequentialPipeline` class for structured augmentation execution. Integrated threading capabilities via the `THREADED` execution mode in `augmentationPipeline`. Enhanced WebSocket support for real-time data processing through pipelines.
This commit is contained in:
parent
9ffc9d965e
commit
a774bb0f82
3 changed files with 453 additions and 4 deletions
|
|
@ -13,6 +13,8 @@ import {
|
|||
AugmentationResponse,
|
||||
AugmentationType
|
||||
} from './types/augmentations.js'
|
||||
import { isThreadingAvailable, isBrowser, isNode } from './utils/environment.js'
|
||||
import { executeInThread } from './utils/workerUtils.js'
|
||||
|
||||
/**
|
||||
* Type definitions for the augmentation registry
|
||||
|
|
@ -35,7 +37,8 @@ export enum ExecutionMode {
|
|||
SEQUENTIAL = 'sequential',
|
||||
PARALLEL = 'parallel',
|
||||
FIRST_SUCCESS = 'firstSuccess',
|
||||
FIRST_RESULT = 'firstResult'
|
||||
FIRST_RESULT = 'firstResult',
|
||||
THREADED = 'threaded' // Execute in separate threads when available
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -45,6 +48,8 @@ export interface PipelineOptions {
|
|||
mode?: ExecutionMode;
|
||||
timeout?: number;
|
||||
stopOnError?: boolean;
|
||||
forceThreading?: boolean; // Force threading even if not in THREADED mode
|
||||
disableThreading?: boolean; // Disable threading even if in THREADED mode
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -53,7 +58,9 @@ export interface PipelineOptions {
|
|||
const DEFAULT_PIPELINE_OPTIONS: PipelineOptions = {
|
||||
mode: ExecutionMode.SEQUENTIAL,
|
||||
timeout: 30000,
|
||||
stopOnError: false
|
||||
stopOnError: false,
|
||||
forceThreading: false,
|
||||
disableThreading: false
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -501,6 +508,32 @@ export class AugmentationPipeline {
|
|||
return methods.every(method => typeof (augmentation as any)[method] === 'function')
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if threading should be used based on options and environment
|
||||
*
|
||||
* @param options The pipeline options
|
||||
* @returns True if threading should be used, false otherwise
|
||||
*/
|
||||
private shouldUseThreading(options: PipelineOptions): boolean {
|
||||
// If threading is explicitly disabled, don't use it
|
||||
if (options.disableThreading) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If threading is explicitly forced, use it if available
|
||||
if (options.forceThreading) {
|
||||
return isThreadingAvailable();
|
||||
}
|
||||
|
||||
// If in THREADED mode, use threading if available
|
||||
if (options.mode === ExecutionMode.THREADED) {
|
||||
return isThreadingAvailable();
|
||||
}
|
||||
|
||||
// Otherwise, don't use threading
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a pipeline for a specific augmentation type
|
||||
*
|
||||
|
|
@ -551,8 +584,33 @@ export class AugmentationPipeline {
|
|||
})
|
||||
: null
|
||||
|
||||
// Execute the method on the augmentation
|
||||
const methodPromise = (augmentation[method] as Function)(...args) as AugmentationResponse<R>
|
||||
// Check if threading should be used
|
||||
const useThreading = this.shouldUseThreading(options);
|
||||
|
||||
// Execute the method on the augmentation, using threading if appropriate
|
||||
let methodPromise: Promise<AugmentationResponse<R>>;
|
||||
|
||||
if (useThreading) {
|
||||
// Execute in a separate thread
|
||||
try {
|
||||
// Create a function that can be serialized and executed in a worker
|
||||
const workerFn = (...workerArgs: any[]) => {
|
||||
// This function will be stringified and executed in the worker
|
||||
// It needs to be self-contained
|
||||
const augFn = augmentation[method as string] as Function;
|
||||
return augFn.apply(augmentation, workerArgs);
|
||||
};
|
||||
|
||||
methodPromise = executeInThread<AugmentationResponse<R>>(workerFn, ...args);
|
||||
} catch (threadError) {
|
||||
console.warn(`Failed to execute in thread, falling back to main thread: ${threadError}`);
|
||||
// Fall back to executing in the main thread
|
||||
methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<R>);
|
||||
}
|
||||
} else {
|
||||
// Execute in the main thread
|
||||
methodPromise = Promise.resolve((augmentation[method] as Function)(...args) as AugmentationResponse<R>);
|
||||
}
|
||||
|
||||
// Race the method promise against the timeout promise if a timeout is specified
|
||||
const result = timeoutPromise
|
||||
|
|
@ -576,6 +634,30 @@ export class AugmentationPipeline {
|
|||
// Execute all augmentations in parallel
|
||||
return enabledAugmentations.map(executeMethod)
|
||||
|
||||
case ExecutionMode.THREADED:
|
||||
// Execute all augmentations in parallel with threading enabled
|
||||
// Force threading for this mode
|
||||
const threadedOptions = { ...options, forceThreading: true };
|
||||
|
||||
// Create a new executeMethod function that uses the threaded options
|
||||
const executeMethodThreaded = async (augmentation: T) => {
|
||||
// Save the original options
|
||||
const originalOptions = options;
|
||||
|
||||
// Set the options to the threaded options
|
||||
options = threadedOptions;
|
||||
|
||||
// Execute the method
|
||||
const result = await executeMethod(augmentation);
|
||||
|
||||
// Restore the original options
|
||||
options = originalOptions;
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
return enabledAugmentations.map(executeMethodThreaded);
|
||||
|
||||
case ExecutionMode.FIRST_SUCCESS:
|
||||
// Execute augmentations sequentially until one succeeds
|
||||
for (const augmentation of enabledAugmentations) {
|
||||
|
|
|
|||
364
src/sequentialPipeline.ts
Normal file
364
src/sequentialPipeline.ts
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
/**
|
||||
* Sequential Augmentation Pipeline
|
||||
*
|
||||
* This module provides a pipeline for executing augmentations in a specific sequence:
|
||||
* ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
|
||||
*
|
||||
* It supports high-performance streaming data from WebSockets without blocking.
|
||||
*/
|
||||
|
||||
import {
|
||||
AugmentationType,
|
||||
IAugmentation,
|
||||
IWebSocketSupport,
|
||||
ISenseAugmentation,
|
||||
IMemoryAugmentation,
|
||||
ICognitionAugmentation,
|
||||
IConduitAugmentation,
|
||||
IActivationAugmentation,
|
||||
IPerceptionAugmentation,
|
||||
AugmentationResponse,
|
||||
WebSocketConnection
|
||||
} from './types/augmentations.js'
|
||||
import { BrainyData } from './brainyData.js'
|
||||
import { augmentationPipeline } from './augmentationPipeline.js'
|
||||
|
||||
/**
|
||||
* Options for sequential pipeline execution
|
||||
*/
|
||||
export interface SequentialPipelineOptions {
|
||||
/**
|
||||
* Timeout for each augmentation execution in milliseconds
|
||||
*/
|
||||
timeout?: number;
|
||||
|
||||
/**
|
||||
* Whether to stop execution if an error occurs
|
||||
*/
|
||||
stopOnError?: boolean;
|
||||
|
||||
/**
|
||||
* BrainyData instance to use for storage
|
||||
*/
|
||||
brainyData?: BrainyData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default pipeline options
|
||||
*/
|
||||
const DEFAULT_SEQUENTIAL_PIPELINE_OPTIONS: SequentialPipelineOptions = {
|
||||
timeout: 30000,
|
||||
stopOnError: false
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of a pipeline execution
|
||||
*/
|
||||
export interface PipelineResult<T> {
|
||||
/**
|
||||
* Whether the pipeline execution was successful
|
||||
*/
|
||||
success: boolean;
|
||||
|
||||
/**
|
||||
* The data returned by the pipeline
|
||||
*/
|
||||
data: T;
|
||||
|
||||
/**
|
||||
* Error message if the pipeline execution failed
|
||||
*/
|
||||
error?: string;
|
||||
|
||||
/**
|
||||
* Results from each stage of the pipeline
|
||||
*/
|
||||
stageResults: {
|
||||
sense?: AugmentationResponse<unknown>;
|
||||
memory?: AugmentationResponse<unknown>;
|
||||
cognition?: AugmentationResponse<unknown>;
|
||||
conduit?: AugmentationResponse<unknown>;
|
||||
activation?: AugmentationResponse<unknown>;
|
||||
perception?: AugmentationResponse<unknown>;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* SequentialPipeline class
|
||||
*
|
||||
* Executes augmentations in a specific sequence:
|
||||
* ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
|
||||
*/
|
||||
export class SequentialPipeline {
|
||||
private brainyData: BrainyData;
|
||||
|
||||
/**
|
||||
* Create a new sequential pipeline
|
||||
*
|
||||
* @param options Options for the pipeline
|
||||
*/
|
||||
constructor(options: SequentialPipelineOptions = {}) {
|
||||
this.brainyData = options.brainyData || new BrainyData();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the pipeline
|
||||
*
|
||||
* @returns A promise that resolves when initialization is complete
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
await this.brainyData.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process data through the sequential pipeline
|
||||
*
|
||||
* @param rawData The raw data to process
|
||||
* @param dataType The type of data (e.g., 'text', 'image', 'audio')
|
||||
* @param options Options for pipeline execution
|
||||
* @returns A promise that resolves with the pipeline result
|
||||
*/
|
||||
public async processData(
|
||||
rawData: Buffer | string,
|
||||
dataType: string,
|
||||
options: SequentialPipelineOptions = {}
|
||||
): Promise<PipelineResult<unknown>> {
|
||||
const opts = { ...DEFAULT_SEQUENTIAL_PIPELINE_OPTIONS, ...options };
|
||||
const result: PipelineResult<unknown> = {
|
||||
success: true,
|
||||
data: null,
|
||||
stageResults: {}
|
||||
};
|
||||
|
||||
try {
|
||||
// Step 1: Process raw data with ISense augmentations
|
||||
const senseResults = await augmentationPipeline.executeSensePipeline(
|
||||
'processRawData',
|
||||
[rawData, dataType],
|
||||
{ timeout: opts.timeout, stopOnError: opts.stopOnError }
|
||||
);
|
||||
|
||||
// Get the first successful result
|
||||
let senseResult: AugmentationResponse<{ nouns: string[], verbs: string[] }> | null = null;
|
||||
for (const resultPromise of senseResults) {
|
||||
const res = await resultPromise;
|
||||
if (res.success) {
|
||||
senseResult = res;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!senseResult || !senseResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
error: 'Failed to process raw data with ISense augmentations',
|
||||
stageResults: { sense: senseResult || { success: false, data: null, error: 'No sense augmentations available' } }
|
||||
};
|
||||
}
|
||||
|
||||
result.stageResults.sense = senseResult;
|
||||
|
||||
// Step 2: Store data in BrainyData using IMemory augmentations
|
||||
const memoryAugmentations = augmentationPipeline.getAugmentationsByType(AugmentationType.MEMORY) as IMemoryAugmentation[];
|
||||
|
||||
if (memoryAugmentations.length === 0) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
error: 'No memory augmentations available',
|
||||
stageResults: result.stageResults
|
||||
};
|
||||
}
|
||||
|
||||
// Use the first available memory augmentation
|
||||
const memoryAugmentation = memoryAugmentations[0];
|
||||
|
||||
// Generate a key for the data
|
||||
const dataKey = `data_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
|
||||
|
||||
// Store the data
|
||||
const memoryResult = await memoryAugmentation.storeData(
|
||||
dataKey,
|
||||
{
|
||||
rawData,
|
||||
dataType,
|
||||
nouns: senseResult.data.nouns,
|
||||
verbs: senseResult.data.verbs,
|
||||
timestamp: Date.now()
|
||||
}
|
||||
);
|
||||
|
||||
if (!memoryResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
error: `Failed to store data: ${memoryResult.error}`,
|
||||
stageResults: { ...result.stageResults, memory: memoryResult }
|
||||
};
|
||||
}
|
||||
|
||||
result.stageResults.memory = memoryResult;
|
||||
|
||||
// Step 3: Trigger ICognition augmentations to analyze the data
|
||||
const cognitionResults = await augmentationPipeline.executeCognitionPipeline(
|
||||
'reason',
|
||||
[`Analyze data with key ${dataKey}`, { dataKey }],
|
||||
{ timeout: opts.timeout, stopOnError: opts.stopOnError }
|
||||
);
|
||||
|
||||
// Get the first successful result
|
||||
let cognitionResult: AugmentationResponse<{ inference: string, confidence: number }> | null = null;
|
||||
for (const resultPromise of cognitionResults) {
|
||||
const res = await resultPromise;
|
||||
if (res.success) {
|
||||
cognitionResult = res;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (cognitionResult) {
|
||||
result.stageResults.cognition = cognitionResult;
|
||||
}
|
||||
|
||||
// Step 4: Send notifications to IConduit augmentations
|
||||
const conduitResults = await augmentationPipeline.executeConduitPipeline(
|
||||
'writeData',
|
||||
[{ dataKey, nouns: senseResult.data.nouns, verbs: senseResult.data.verbs }],
|
||||
{ timeout: opts.timeout, stopOnError: opts.stopOnError }
|
||||
);
|
||||
|
||||
// Get the first successful result
|
||||
let conduitResult: AugmentationResponse<unknown> | null = null;
|
||||
for (const resultPromise of conduitResults) {
|
||||
const res = await resultPromise;
|
||||
if (res.success) {
|
||||
conduitResult = res;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (conduitResult) {
|
||||
result.stageResults.conduit = conduitResult;
|
||||
}
|
||||
|
||||
// Step 5: Send notifications to IActivation augmentations
|
||||
const activationResults = await augmentationPipeline.executeActivationPipeline(
|
||||
'triggerAction',
|
||||
['dataProcessed', { dataKey }],
|
||||
{ timeout: opts.timeout, stopOnError: opts.stopOnError }
|
||||
);
|
||||
|
||||
// Get the first successful result
|
||||
let activationResult: AugmentationResponse<unknown> | null = null;
|
||||
for (const resultPromise of activationResults) {
|
||||
const res = await resultPromise;
|
||||
if (res.success) {
|
||||
activationResult = res;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (activationResult) {
|
||||
result.stageResults.activation = activationResult;
|
||||
}
|
||||
|
||||
// Step 6: Send notifications to IPerception augmentations
|
||||
const perceptionResults = await augmentationPipeline.executePerceptionPipeline(
|
||||
'interpret',
|
||||
[senseResult.data.nouns, senseResult.data.verbs, { dataKey }],
|
||||
{ timeout: opts.timeout, stopOnError: opts.stopOnError }
|
||||
);
|
||||
|
||||
// Get the first successful result
|
||||
let perceptionResult: AugmentationResponse<Record<string, unknown>> | null = null;
|
||||
for (const resultPromise of perceptionResults) {
|
||||
const res = await resultPromise;
|
||||
if (res.success) {
|
||||
perceptionResult = res;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (perceptionResult) {
|
||||
result.stageResults.perception = perceptionResult;
|
||||
result.data = perceptionResult.data;
|
||||
} else {
|
||||
// If no perception result, use the cognition result as the final data
|
||||
result.data = cognitionResult ? cognitionResult.data : { dataKey };
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
error: `Pipeline execution failed: ${error}`,
|
||||
stageResults: result.stageResults
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process WebSocket data through the sequential pipeline
|
||||
*
|
||||
* @param connection The WebSocket connection
|
||||
* @param dataType The type of data (e.g., 'text', 'image', 'audio')
|
||||
* @param options Options for pipeline execution
|
||||
* @returns A function to handle incoming WebSocket messages
|
||||
*/
|
||||
public createWebSocketHandler(
|
||||
connection: WebSocketConnection,
|
||||
dataType: string,
|
||||
options: SequentialPipelineOptions = {}
|
||||
): (data: unknown) => void {
|
||||
return (data: unknown) => {
|
||||
// Process the data asynchronously without blocking
|
||||
this.processData(
|
||||
typeof data === 'string' ? data : JSON.stringify(data),
|
||||
dataType,
|
||||
options
|
||||
).catch(error => {
|
||||
console.error('Error processing WebSocket data:', error);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up a WebSocket connection to process data through the pipeline
|
||||
*
|
||||
* @param url The WebSocket URL to connect to
|
||||
* @param dataType The type of data (e.g., 'text', 'image', 'audio')
|
||||
* @param options Options for pipeline execution
|
||||
* @returns A promise that resolves with the WebSocket connection
|
||||
*/
|
||||
public async setupWebSocketPipeline(
|
||||
url: string,
|
||||
dataType: string,
|
||||
options: SequentialPipelineOptions = {}
|
||||
): Promise<WebSocketConnection> {
|
||||
// Get WebSocket-supporting augmentations
|
||||
const webSocketAugmentations = augmentationPipeline.getWebSocketAugmentations();
|
||||
|
||||
if (webSocketAugmentations.length === 0) {
|
||||
throw new Error('No WebSocket-supporting augmentations available');
|
||||
}
|
||||
|
||||
// Use the first available WebSocket augmentation
|
||||
const webSocketAugmentation = webSocketAugmentations[0];
|
||||
|
||||
// Connect to the WebSocket
|
||||
const connection = await webSocketAugmentation.connectWebSocket(url);
|
||||
|
||||
// Create a handler for incoming messages
|
||||
const handler = this.createWebSocketHandler(connection, dataType, options);
|
||||
|
||||
// Register the handler
|
||||
await webSocketAugmentation.onWebSocketMessage(connection.connectionId, handler);
|
||||
|
||||
return connection;
|
||||
}
|
||||
}
|
||||
|
||||
// Create and export a default instance of the sequential pipeline
|
||||
export const sequentialPipeline = new SequentialPipeline();
|
||||
|
|
@ -48,6 +48,9 @@ export interface IAugmentation {
|
|||
shutDown(): Promise<void>
|
||||
|
||||
getStatus(): Promise<'active' | 'inactive' | 'error'>
|
||||
|
||||
// Allow string indexing for dynamic method access
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue