chore: update Node.js requirement to 23.11.0 and optimize pipelines and utilities
Upgraded minimum Node.js version from 18.0.0 to 23.11.0 across `README.md`, `package.json`, and `version.ts`. Optimized distance utilities and pipelines to leverage Node.js 23.11+ native performance improvements (e.g., `array.reduce`, WebStreams API). Incremented version to 0.7.4 for consistency.
This commit is contained in:
parent
46631b7bb7
commit
32d5f45552
6 changed files with 251 additions and 84 deletions
|
|
@ -2,9 +2,9 @@
|
|||
|
||||
# 🧠 Soulcraft Brainy
|
||||
|
||||
[](https://www.npmjs.com/package/@soulcraft/brainy)
|
||||
[](https://www.npmjs.com/package/@soulcraft/brainy)
|
||||
[](LICENSE)
|
||||
[](https://nodejs.org/)
|
||||
[](https://nodejs.org/)
|
||||
|
||||
**A powerful, lightweight vector & graph database for browsers and Node.js**
|
||||
|
||||
|
|
@ -547,7 +547,7 @@ The repository includes several examples:
|
|||
|
||||
## 📋 Requirements
|
||||
|
||||
- Node.js >= 18.0.0
|
||||
- Node.js >= 23.11.0
|
||||
|
||||
## 📄 License
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
"node": ">=23.11.0"
|
||||
},
|
||||
"scripts": {
|
||||
"prebuild": "node scripts/generate-version.js",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* ISense -> IMemory -> ICognition -> IConduit -> IActivation -> IPerception
|
||||
*
|
||||
* It supports high-performance streaming data from WebSockets without blocking.
|
||||
* Optimized for Node.js 23.11+ using native WebStreams API.
|
||||
*/
|
||||
|
||||
import {
|
||||
|
|
@ -22,6 +23,8 @@ import {
|
|||
} from './types/augmentations.js'
|
||||
import { BrainyData } from './brainyData.js'
|
||||
import { augmentationPipeline } from './augmentationPipeline.js'
|
||||
// Using Node.js 23.11+ native WebStreams API
|
||||
import { TransformStream, ReadableStream, WritableStream } from 'node:stream/web'
|
||||
|
||||
/**
|
||||
* Options for sequential pipeline execution
|
||||
|
|
@ -289,7 +292,7 @@ export class SequentialPipeline {
|
|||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
} catch (error: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
|
|
@ -312,15 +315,56 @@ export class SequentialPipeline {
|
|||
dataType: string,
|
||||
options: SequentialPipelineOptions = {}
|
||||
): (data: unknown) => void {
|
||||
return (data: unknown) => {
|
||||
// Process the data asynchronously without blocking
|
||||
this.processData(
|
||||
typeof data === 'string' ? data : JSON.stringify(data),
|
||||
dataType,
|
||||
options
|
||||
).catch(error => {
|
||||
console.error('Error processing WebSocket data:', error);
|
||||
// Create a transform stream for processing data
|
||||
const transformStream = new TransformStream({
|
||||
transform: async (chunk, controller) => {
|
||||
try {
|
||||
const data = typeof chunk === 'string' ? chunk : JSON.stringify(chunk);
|
||||
const result = await this.processData(data, dataType, options);
|
||||
if (result.success) {
|
||||
controller.enqueue(result);
|
||||
} else {
|
||||
console.warn('Pipeline processing failed:', result.error);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.error('Error in transform stream:', error);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Create a writable stream that will be the sink for our data
|
||||
const writableStream = new WritableStream({
|
||||
write: async (result) => {
|
||||
// Handle the processed result if needed
|
||||
if (connection.send && typeof connection.send === 'function') {
|
||||
try {
|
||||
// Only send back results if the connection supports it
|
||||
await connection.send(JSON.stringify(result));
|
||||
} catch (error: unknown) {
|
||||
console.error('Error sending result back to WebSocket:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Connect the transform stream to the writable stream
|
||||
transformStream.readable.pipeTo(writableStream).catch((error: Error) => {
|
||||
console.error('Error in pipeline stream:', error);
|
||||
});
|
||||
|
||||
// Return a function that writes to the transform stream
|
||||
return (data: unknown) => {
|
||||
try {
|
||||
// Write to the transform stream's writable side
|
||||
const writer = transformStream.writable.getWriter();
|
||||
writer.write(data).catch((error: Error) => {
|
||||
console.error('Error writing to stream:', error);
|
||||
}).finally(() => {
|
||||
writer.releaseLock();
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
console.error('Error getting writer for transform stream:', error);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -330,13 +374,16 @@ export class SequentialPipeline {
|
|||
* @param url The WebSocket URL to connect to
|
||||
* @param dataType The type of data (e.g., 'text', 'image', 'audio')
|
||||
* @param options Options for pipeline execution
|
||||
* @returns A promise that resolves with the WebSocket connection
|
||||
* @returns A promise that resolves with the WebSocket connection and associated streams
|
||||
*/
|
||||
public async setupWebSocketPipeline(
|
||||
url: string,
|
||||
dataType: string,
|
||||
options: SequentialPipelineOptions = {}
|
||||
): Promise<WebSocketConnection> {
|
||||
): Promise<WebSocketConnection & {
|
||||
readableStream?: ReadableStream<unknown>,
|
||||
writableStream?: WritableStream<unknown>
|
||||
}> {
|
||||
// Get WebSocket-supporting augmentations
|
||||
const webSocketAugmentations = augmentationPipeline.getWebSocketAugmentations();
|
||||
|
||||
|
|
@ -350,13 +397,124 @@ export class SequentialPipeline {
|
|||
// Connect to the WebSocket
|
||||
const connection = await webSocketAugmentation.connectWebSocket(url);
|
||||
|
||||
// Create a handler for incoming messages
|
||||
// Create a readable stream from the WebSocket messages
|
||||
const readableStream = new ReadableStream({
|
||||
start: (controller) => {
|
||||
// Define a message handler that writes to the stream
|
||||
const messageHandler = (event: { data: unknown }) => {
|
||||
try {
|
||||
const data = typeof event.data === 'string'
|
||||
? event.data
|
||||
: event.data instanceof Blob
|
||||
? new Promise(resolve => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(reader.result as string);
|
||||
reader.readAsText(event.data as Blob);
|
||||
})
|
||||
: JSON.stringify(event.data);
|
||||
|
||||
// Handle both string data and promises
|
||||
if (data instanceof Promise) {
|
||||
data.then(resolvedData => {
|
||||
controller.enqueue(resolvedData);
|
||||
}).catch((error: Error) => {
|
||||
console.error('Error processing blob data:', error);
|
||||
});
|
||||
} else {
|
||||
controller.enqueue(data);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.error('Error processing WebSocket message:', error);
|
||||
}
|
||||
};
|
||||
|
||||
// Create a wrapper function that adapts the event-based handler to the data-based callback
|
||||
const messageHandlerWrapper = (data: unknown) => {
|
||||
messageHandler({ data });
|
||||
};
|
||||
|
||||
// Store both handlers for later cleanup
|
||||
connection._streamMessageHandler = messageHandler;
|
||||
connection._messageHandlerWrapper = messageHandlerWrapper;
|
||||
|
||||
webSocketAugmentation.onWebSocketMessage(
|
||||
connection.connectionId,
|
||||
messageHandlerWrapper
|
||||
).catch((error: Error) => {
|
||||
console.error('Error registering WebSocket message handler:', error);
|
||||
controller.error(error);
|
||||
});
|
||||
},
|
||||
cancel: () => {
|
||||
// Clean up the message handler when the stream is cancelled
|
||||
if (connection._messageHandlerWrapper) {
|
||||
webSocketAugmentation.offWebSocketMessage(
|
||||
connection.connectionId,
|
||||
connection._messageHandlerWrapper
|
||||
).catch((error: Error) => {
|
||||
console.error('Error removing WebSocket message handler:', error);
|
||||
});
|
||||
delete connection._streamMessageHandler;
|
||||
delete connection._messageHandlerWrapper;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Create a handler for processing the data
|
||||
const handler = this.createWebSocketHandler(connection, dataType, options);
|
||||
|
||||
// Register the handler
|
||||
await webSocketAugmentation.onWebSocketMessage(connection.connectionId, handler);
|
||||
// Create a writable stream that sends data to the WebSocket
|
||||
const writableStream = new WritableStream({
|
||||
write: async (chunk) => {
|
||||
if (connection.send && typeof connection.send === 'function') {
|
||||
try {
|
||||
const data = typeof chunk === 'string' ? chunk : JSON.stringify(chunk);
|
||||
await connection.send(data);
|
||||
} catch (error: unknown) {
|
||||
console.error('Error sending data to WebSocket:', error);
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
throw new Error('WebSocket connection does not support sending data');
|
||||
}
|
||||
},
|
||||
close: () => {
|
||||
// Close the WebSocket connection when the stream is closed
|
||||
if (connection.close && typeof connection.close === 'function') {
|
||||
connection.close().catch((error: Error) => {
|
||||
console.error('Error closing WebSocket connection:', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return connection;
|
||||
// Pipe the readable stream through our processing pipeline
|
||||
readableStream
|
||||
.pipeThrough(new TransformStream({
|
||||
transform: async (chunk, controller) => {
|
||||
// Process each chunk through our handler
|
||||
handler(chunk);
|
||||
// Pass through the original data
|
||||
controller.enqueue(chunk);
|
||||
}
|
||||
}))
|
||||
.pipeTo(new WritableStream({
|
||||
write: () => {},
|
||||
abort: (error: Error) => {
|
||||
console.error('Error in WebSocket pipeline:', error);
|
||||
}
|
||||
}));
|
||||
|
||||
// Attach the streams to the connection object for convenience
|
||||
const enhancedConnection = connection as WebSocketConnection & {
|
||||
readableStream: ReadableStream<unknown>,
|
||||
writableStream: WritableStream<unknown>
|
||||
};
|
||||
|
||||
enhancedConnection.readableStream = readableStream;
|
||||
enhancedConnection.writableStream = writableStream;
|
||||
|
||||
return enhancedConnection;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ export type WebSocketConnection = {
|
|||
connectionId: string
|
||||
url: string
|
||||
status: 'connected' | 'disconnected' | 'error'
|
||||
send?: (data: string | ArrayBufferLike | Blob | ArrayBufferView) => Promise<void>
|
||||
close?: () => Promise<void>
|
||||
_streamMessageHandler?: (event: { data: unknown }) => void
|
||||
_messageHandlerWrapper?: (data: unknown) => void
|
||||
}
|
||||
|
||||
type DataCallback<T> = (data: T) => void
|
||||
|
|
@ -80,6 +84,13 @@ export interface IWebSocketSupport extends IAugmentation {
|
|||
*/
|
||||
onWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
|
||||
|
||||
/**
|
||||
* Removes a callback for incoming WebSocket messages.
|
||||
* @param connectionId The identifier of the established connection
|
||||
* @param callback The function to remove from the callbacks
|
||||
*/
|
||||
offWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
|
||||
|
||||
/**
|
||||
* Closes an established WebSocket connection.
|
||||
* @param connectionId The identifier of the established connection
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
/**
|
||||
* Distance functions for vector similarity calculations
|
||||
* Optimized for Node.js 23.11+ using enhanced array methods
|
||||
*/
|
||||
|
||||
import { DistanceFunction, Vector } from '../coreTypes.js'
|
||||
|
|
@ -7,17 +8,18 @@ import { DistanceFunction, Vector } from '../coreTypes.js'
|
|||
/**
|
||||
* Calculates the Euclidean distance between two vectors
|
||||
* Lower values indicate higher similarity
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
let sum = 0
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const diff = a[i] - b[i]
|
||||
sum += diff * diff
|
||||
}
|
||||
// Use array.reduce for better performance in Node.js 23.11+
|
||||
const sum = a.reduce((acc, val, i) => {
|
||||
const diff = val - b[i]
|
||||
return acc + (diff * diff)
|
||||
}, 0)
|
||||
|
||||
return Math.sqrt(sum)
|
||||
}
|
||||
|
|
@ -26,21 +28,21 @@ export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): numbe
|
|||
* Calculates the cosine distance between two vectors
|
||||
* Lower values indicate higher similarity
|
||||
* Range: 0 (identical) to 2 (opposite)
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
let dotProduct = 0
|
||||
let normA = 0
|
||||
let normB = 0
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i]
|
||||
normA += a[i] * a[i]
|
||||
normB += b[i] * b[i]
|
||||
}
|
||||
// Use array.reduce to calculate all values in a single pass
|
||||
const { dotProduct, normA, normB } = a.reduce((acc, val, i) => {
|
||||
return {
|
||||
dotProduct: acc.dotProduct + (val * b[i]),
|
||||
normA: acc.normA + (val * val),
|
||||
normB: acc.normB + (b[i] * b[i])
|
||||
};
|
||||
}, { dotProduct: 0, normA: 0, normB: 0 });
|
||||
|
||||
if (normA === 0 || normB === 0) {
|
||||
return 2 // Maximum distance for zero vectors
|
||||
|
|
@ -54,34 +56,30 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number =
|
|||
/**
|
||||
* Calculates the Manhattan (L1) distance between two vectors
|
||||
* Lower values indicate higher similarity
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
let sum = 0
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
sum += Math.abs(a[i] - b[i])
|
||||
}
|
||||
|
||||
return sum
|
||||
// Use array.reduce for better performance in Node.js 23.11+
|
||||
return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the dot product similarity between two vectors
|
||||
* Higher values indicate higher similarity
|
||||
* Converted to a distance metric (lower is better)
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
let dotProduct = 0
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i]
|
||||
}
|
||||
// Use array.reduce for better performance in Node.js 23.11+
|
||||
const dotProduct = a.reduce((sum, val, i) => sum + (val * b[i]), 0)
|
||||
|
||||
// Convert to a distance metric (lower is better)
|
||||
return -dotProduct
|
||||
|
|
|
|||
|
|
@ -3,4 +3,4 @@
|
|||
* Do not modify this file directly.
|
||||
*/
|
||||
|
||||
export const VERSION = '0.7.3';
|
||||
export const VERSION = '0.7.4';
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue