diff --git a/README.md b/README.md
index c189ea4a..05999ed4 100644
--- a/README.md
+++ b/README.md
@@ -1,211 +1,438 @@
-# Soulcraft Brainy
+# π§ Soulcraft Brainy
-A combined Graph and Vector database that runs in browsers and Node.js environments. Brainy provides efficient vector
-search capabilities with persistent storage options.
+
-## What is Brainy?
+[](https://www.npmjs.com/package/@soulcraft/brainy)
+[](LICENSE)
+[](https://nodejs.org/)
-Brainy is a lightweight database that combines:
+**A powerful, lightweight vector & graph database for browsers and Node.js**
-- Vector search (for similarity-based retrieval)
-- Graph relationships (for structured data connections)
-- Persistent storage (works across sessions)
+
-It's designed to work seamlessly in both browser and server environments.
+## β¨ Overview
-## How It Works
+Brainy combines the power of vector search with graph relationships in a lightweight, cross-platform database. Whether you're building AI applications, recommendation systems, or knowledge graphs, Brainy provides the tools you need to store, connect, and retrieve your data intelligently.
-Brainy combines three key technologies:
+### π Key Features
-1. **Vector Embeddings**: Converts data (text, images, etc.) into numerical vectors that capture semantic meaning
-2. **HNSW Algorithm**: Enables fast similarity search through a hierarchical graph structure
-3. **Persistent Storage**: Uses the best available storage option for your environment:
- - Browser: Origin Private File System (OPFS)
- - Node.js: File system
- - Server: S3-compatible storage (optional)
- - Fallback: In-memory storage
+- **Vector Search** - Find semantically similar content using embeddings
+- **Graph Relationships** - Connect data with meaningful relationships
+- **Cross-Platform** - Works in browsers, Node.js, and server environments
+- **Persistent Storage** - Data persists across sessions
+- **TypeScript Support** - Fully typed API with generics
+- **CLI Tools** - Powerful command-line interface for data management
-## Data Model
+## π What Can You Build?
-Brainy uses a graph-based data model with two primary concepts:
+- **Semantic Search Engines** - Find content based on meaning, not just keywords
+- **Recommendation Systems** - Suggest similar items based on vector similarity
+- **Knowledge Graphs** - Build connected data structures with relationships
+- **AI Applications** - Store and retrieve embeddings for machine learning models
+- **Data Organization Tools** - Automatically categorize and connect related information
-1. **Nouns**: The main entities in your data (nodes in the graph)
- - Each noun has a unique ID, vector representation, and metadata
- - Nouns can be categorized by type (Person, Place, Thing, Event, Concept, etc.)
- - Nouns are automatically vectorized for similarity search
+## π§ Installation
-2. **Verbs**: Relationships between nouns (edges in the graph)
- - Each verb connects a source noun to a target noun
- - Verbs have types that define the relationship (RelatedTo, Controls, Contains, etc.)
- - Verbs can have their own metadata to describe the relationship
-
-## What Can You Use It For?
-
-- **Semantic Search**: Find content based on meaning, not just keywords
-- **Recommendation Systems**: Suggest similar items based on vector similarity
-- **Knowledge Graphs**: Build connected data structures with relationships
-- **Data Organization**: Automatically categorize and connect related information
-- **AI Applications**: Store and retrieve embeddings for machine learning models
-
-## Installation
-
-Due to a dependency conflict between TensorFlow.js packages, you need to use the `--legacy-peer-deps` flag when
-installing:
+Due to a dependency conflict between TensorFlow.js packages, use the `--legacy-peer-deps` flag when installing:
```bash
npm install @soulcraft/brainy --legacy-peer-deps
```
-If you encounter an error like this:
-
-```
-npm error ERESOLVE unable to resolve dependency tree
-npm error Found: @tensorflow/tfjs-core@4.22.0
-npm error Could not resolve dependency: peer @tensorflow/tfjs-core@"3.21.0" from @tensorflow/tfjs-converter@3.21.0
-```
-
-Use the `--legacy-peer-deps` flag as shown above to resolve it.
-
-## Command Line Interface
-
-Brainy includes a command-line interface (CLI) that allows you to experiment with the API and data directly from the
-terminal.
-
-### Using the CLI during development
-
-```bash
-# Run the CLI directly from the source
-npm run cli help
-
-# Initialize a database
-npm run cli init
-
-# Add some data
-npm run cli add "Cats are independent pets" '{"noun":"Thing","category":"animal"}'
-npm run cli add "Dogs are loyal companions" '{"noun":"Thing","category":"animal"}'
-
-# Search for similar items
-npm run cli search "feline pets" 5
-
-# Add relationships between items
-npm run cli addVerb RelatedTo '{"description":"Both are pets"}'
-
-# View database status
-npm run cli status
-```
-
-### Installing the CLI globally
-
-```bash
-# Install the package globally (with --legacy-peer-deps to resolve TensorFlow.js dependency conflicts)
-npm install -g @soulcraft/brainy --legacy-peer-deps
-
-# Now you can use the 'brainy' command directly
-brainy help
-brainy init
-brainy add "Some text" '{"noun":"Thing"}'
-```
-
-### Available Commands
-
-- `init` - Initialize a new database
-- `add [metadata]` - Add a new noun with the given text and optional metadata
-- `search [limit]` - Search for nouns similar to the query
-- `get ` - Get a noun by ID
-- `delete ` - Delete a noun by ID
-- `addVerb [metadata]` - Add a relationship between nouns
-- `getVerbs ` - Get all relationships for a noun
-- `status` - Show database status
-- `completion-setup` - Setup shell autocomplete for the Brainy CLI
-
-### Autocomplete
-
-The CLI supports autocomplete for commands, noun types, and verb types. To enable autocomplete:
-
-```bash
-# Set up autocomplete for your shell
-brainy completion-setup
-
-# Restart your shell or source your shell configuration file
-# For bash:
-source ~/.bashrc
-
-# For zsh:
-source ~/.zshrc
-```
-
-Once enabled, you can use tab completion for:
-- Commands (e.g., `brainy a` completes to `brainy add`)
-- Noun types in metadata (e.g., when adding nouns)
-- Verb types (e.g., when adding relationships)
-
-## Basic Usage
+## π Quick Start
```typescript
-import {BrainyData, NounType, VerbType} from '@soulcraft/brainy';
+import { BrainyData, NounType, VerbType } from '@soulcraft/brainy';
// Create and initialize the database
const db = new BrainyData();
await db.init();
-// Add nouns (automatically converted to vectors)
+// Add data (automatically converted to vectors)
const catId = await db.add("Cats are independent pets", {
- noun: NounType.Thing,
- category: 'animal'
+ noun: NounType.Thing,
+ category: 'animal'
});
const dogId = await db.add("Dogs are loyal companions", {
- noun: NounType.Thing,
- category: 'animal'
+ noun: NounType.Thing,
+ category: 'animal'
});
-// Search for similar nouns
+// Search for similar items
const results = await db.searchText("feline pets", 2);
console.log(results);
-// Returns nouns similar to "feline pets" with their similarity scores
+// Returns items similar to "feline pets" with similarity scores
-// Add a verb (relationship) between nouns
+// Add a relationship between items
await db.addVerb(catId, dogId, {
- verb: VerbType.RelatedTo,
- description: 'Both are common household pets'
+ verb: VerbType.RelatedTo,
+ description: 'Both are common household pets'
+});
+```
+
+## 𧩠How It Works
+
+Brainy combines three key technologies:
+
+1. **Vector Embeddings** - Converts data (text, images, etc.) into numerical vectors that capture semantic meaning
+2. **HNSW Algorithm** - Enables fast similarity search through a hierarchical graph structure
+3. **Persistent Storage** - Uses the best available storage option for your environment:
+ - Browser: Origin Private File System (OPFS)
+ - Node.js: File system
+ - Server: S3-compatible storage (optional)
+ - Fallback: In-memory storage
+
+## π The Brainy Pipeline
+
+Brainy's data processing pipeline transforms raw data into searchable, connected knowledge. Here's how the magic happens:
+
+```
+Raw Data β Embedding β Vector Storage β Graph Connections β Query & Retrieval
+```
+
+### π Pipeline Stages
+
+1. **Data Ingestion** π½οΈ
+ - Raw text or pre-computed vectors enter the pipeline
+ - Data is validated and prepared for processing
+
+2. **Embedding Generation** π§
+ - Text is transformed into numerical vectors using embedding models
+ - Choose between TensorFlow Universal Sentence Encoder (high quality) or Simple Embedding (faster)
+ - Custom embedding functions can be plugged in for specialized domains
+
+3. **Vector Indexing** π
+ - Vectors are indexed using the HNSW algorithm
+ - Hierarchical structure enables lightning-fast similarity search
+ - Configurable parameters for precision vs. performance tradeoffs
+
+4. **Graph Construction** πΈοΈ
+ - Nouns (entities) become nodes in the knowledge graph
+ - Verbs (relationships) connect related entities
+ - Typed relationships add semantic meaning to connections
+
+5. **Persistent Storage** πΎ
+ - Data is saved using the optimal storage for your environment
+ - Automatic selection between OPFS, filesystem, S3, or memory
+ - Configurable storage adapters for custom persistence needs
+
+### 𧩠Augmentation Types
+
+Brainy uses a powerful augmentation system to extend functionality. Augmentations are processed in the following order:
+
+1. **SENSE** ποΈ
+ - Ingests and processes raw, unstructured data into nouns and verbs
+ - Handles text, images, audio streams, and other input formats
+ - Example: Converting raw text into structured entities
+
+2. **MEMORY** πΎ
+ - Provides storage capabilities for data in different formats
+ - Manages persistence across sessions
+ - Example: Storing vectors in OPFS or filesystem
+
+3. **COGNITION** π§
+ - Enables advanced reasoning, inference, and logical operations
+ - Analyzes relationships between entities
+ - Example: Inferring new connections between existing data
+
+4. **CONDUIT** π
+ - Establishes high-bandwidth channels for structured data exchange
+ - Connects with external systems
+ - Example: Integrating with third-party APIs
+
+5. **ACTIVATION** β‘
+ - Initiates actions, responses, or data manipulations
+ - Triggers events based on data changes
+ - Example: Sending notifications when new data is processed
+
+6. **PERCEPTION** π
+ - Interprets, contextualizes, and visualizes identified nouns and verbs
+ - Creates meaningful representations of data
+ - Example: Generating visualizations of graph relationships
+
+7. **DIALOG** π¬
+ - Facilitates natural language understanding and generation
+ - Enables conversational interactions
+ - Example: Processing user queries and generating responses
+
+8. **WEBSOCKET** π
+ - Enables real-time communication via WebSockets
+ - Can be combined with other augmentation types
+ - Example: Streaming data processing in real-time
+
+### π Streaming Data Support
+
+Brainy's pipeline is designed to handle streaming data efficiently:
+
+1. **WebSocket Integration** π
+ - Built-in support for WebSocket connections
+ - Process data as it arrives without blocking
+ - Example: `setupWebSocketPipeline(url, dataType, options)`
+
+2. **Asynchronous Processing** β‘
+ - Non-blocking architecture for real-time data handling
+ - Parallel processing of incoming streams
+ - Example: `createWebSocketHandler(connection, dataType, options)`
+
+3. **Event-Based Architecture** π‘
+ - Augmentations can listen to data feeds and streams
+ - Real-time updates propagate through the pipeline
+ - Example: `listenToFeed(feedUrl, callback)`
+
+4. **Threaded Execution** π§΅
+ - Optional multi-threading for high-performance streaming
+ - Configurable execution modes (SEQUENTIAL, PARALLEL, THREADED)
+ - Example: `executeTypedPipeline(augmentations, method, args, { mode: ExecutionMode.THREADED })`
+
+### πββοΈ Running the Pipeline
+
+The pipeline runs automatically when you:
+
+```typescript
+// Add data (runs embedding β indexing β storage)
+const id = await db.add("Your text data here", { metadata });
+
+// Search (runs embedding β similarity search)
+const results = await db.searchText("Your query here", 5);
+
+// Connect entities (runs graph construction β storage)
+await db.addVerb(sourceId, targetId, { verb: VerbType.RelatedTo });
+```
+
+Using the CLI:
+
+```bash
+# Add data through the CLI pipeline
+brainy add "Your text data here" '{"noun":"Thing"}'
+
+# Search through the CLI pipeline
+brainy search "Your query here" --limit 5
+
+# Connect entities through the CLI
+brainy addVerb RelatedTo
+```
+
+### π§ Extending the Pipeline
+
+Brainy's pipeline is designed for extensibility at every stage:
+
+1. **Custom Embedding** π§©
+ ```typescript
+ // Create your own embedding function
+ const myEmbedder = async (text) => {
+ // Your custom embedding logic here
+ return [0.1, 0.2, 0.3, ...]; // Return a vector
+ };
+
+ // Use it in Brainy
+ const db = new BrainyData({
+ embeddingFunction: myEmbedder
+ });
+ ```
+
+2. **Custom Distance Functions** π
+ ```typescript
+ // Define your own distance function
+ const myDistance = (a, b) => {
+ // Your custom distance calculation
+ return Math.sqrt(a.reduce((sum, val, i) => sum + Math.pow(val - b[i], 2), 0));
+ };
+
+ // Use it in Brainy
+ const db = new BrainyData({
+ distanceFunction: myDistance
+ });
+ ```
+
+3. **Custom Storage Adapters** π¦
+ ```typescript
+ // Implement the StorageAdapter interface
+ class MyStorage implements StorageAdapter {
+ // Your storage implementation
+ }
+
+ // Use it in Brainy
+ const db = new BrainyData({
+ storageAdapter: new MyStorage()
+ });
+ ```
+
+4. **Augmentations System** π§
+ ```typescript
+ // Create custom augmentations to extend functionality
+ const myAugmentation = {
+ type: 'memory',
+ name: 'my-custom-storage',
+ // Implementation details
+ };
+
+ // Register with Brainy
+ db.registerAugmentation(myAugmentation);
+ ```
+
+## π Data Model
+
+Brainy uses a graph-based data model with two primary concepts:
+
+### Nouns (Entities)
+
+The main entities in your data (nodes in the graph):
+- Each noun has a unique ID, vector representation, and metadata
+- Nouns can be categorized by type (Person, Place, Thing, Event, Concept, etc.)
+- Nouns are automatically vectorized for similarity search
+
+### Verbs (Relationships)
+
+Connections between nouns (edges in the graph):
+- Each verb connects a source noun to a target noun
+- Verbs have types that define the relationship (RelatedTo, Controls, Contains, etc.)
+- Verbs can have their own metadata to describe the relationship
+
+## π₯οΈ Command Line Interface
+
+Brainy includes a powerful CLI for managing your data:
+
+```bash
+# Install globally
+npm install -g @soulcraft/brainy --legacy-peer-deps
+
+# Initialize a database
+brainy init
+
+# Add some data
+brainy add "Cats are independent pets" '{"noun":"Thing","category":"animal"}'
+brainy add "Dogs are loyal companions" '{"noun":"Thing","category":"animal"}'
+
+# Search for similar items
+brainy search "feline pets" 5
+
+# Add relationships between items
+brainy addVerb RelatedTo '{"description":"Both are pets"}'
+
+# Visualize the graph structure
+brainy visualize
+brainy visualize --root --depth 3
+```
+
+### π Using During Development
+
+```bash
+# Run the CLI directly from the source
+npm run cli help
+
+# Generate a random graph for testing
+npm run cli generate-random-graph --noun-count 20 --verb-count 40
+```
+
+### π Available Commands
+
+#### Basic Database Operations:
+- `init` - Initialize a new database
+- `add [metadata]` - Add a new noun with text and optional metadata
+- `search [limit]` - Search for nouns similar to the query
+- `get ` - Get a noun by ID
+- `delete ` - Delete a noun by ID
+- `addVerb [metadata]` - Add a relationship
+- `getVerbs ` - Get all relationships for a noun
+- `status` - Show database status
+- `clear` - Clear all data from the database
+- `generate-random-graph` - Generate test data
+- `visualize` - Visualize the graph structure
+- `completion-setup` - Setup shell autocomplete
+
+#### Pipeline and Augmentation Commands:
+- `list-augmentations` - List all available augmentation types and registered augmentations
+- `augmentation-info ` - Get detailed information about a specific augmentation type
+- `test-pipeline [text]` - Test the sequential pipeline with sample data
+ - `-t, --data-type ` - Type of data to process (default: 'text')
+ - `-m, --mode ` - Execution mode: sequential, parallel, threaded (default: 'sequential')
+ - `-s, --stop-on-error` - Stop execution if an error occurs
+ - `-v, --verbose` - Show detailed output
+- `stream-test` - Test streaming data through the pipeline (simulated)
+ - `-c, --count ` - Number of data items to stream (default: 5)
+ - `-i, --interval ` - Interval between data items in milliseconds (default: 1000)
+ - `-t, --data-type ` - Type of data to process (default: 'text')
+ - `-v, --verbose` - Show detailed output
+
+## π API Reference
+
+### Database Management
+
+```typescript
+// Initialize the database
+await db.init();
+
+// Clear all data
+await db.clear();
+
+// Get database status
+const status = await db.status();
+```
+
+### Working with Nouns (Entities)
+
+```typescript
+// Add a noun (automatically vectorized)
+const id = await db.add(textOrVector, {
+ noun: NounType.Thing,
+ // other metadata...
});
// Retrieve a noun
-const cat = await db.get(catId);
-console.log(cat);
-
-// Get all relationships for a noun
-const catRelationships = await db.getVerbsBySource(catId);
-console.log(catRelationships);
+const noun = await db.get(id);
// Update noun metadata
-await db.updateMetadata(catId, {
- noun: NounType.Thing,
- category: 'animal',
- size: 'small'
+await db.updateMetadata(id, {
+ noun: NounType.Thing,
+ // updated metadata...
});
// Delete a noun
-await db.delete(dogId);
+await db.delete(id);
+
+// Search for similar nouns
+const results = await db.search(vectorOrText, numResults);
+const textResults = await db.searchText("query text", numResults);
+
+// Search by noun type
+const thingNouns = await db.searchByNounTypes([NounType.Thing], numResults);
```
-## Key Features
+### Working with Verbs (Relationships)
-- **Automatic Vectorization**: Converts text and data to vector embeddings
-- **Fast Similarity Search**: Uses HNSW algorithm for efficient retrieval
-- **Noun-Based Data Model**: Organize entities with typed categorization (Person, Place, Thing, etc.)
-- **Verb-Based Relationships**: Create meaningful connections between nouns with typed relationships
-- **Cross-Platform**: Works in browsers, Node.js, and server environments
-- **Persistent Storage**: Multiple storage options (OPFS, filesystem, cloud)
-- **TypeScript Support**: Fully typed API with generics
-- **Flexible Configuration**: Customize distance functions, embedding models, and more
+```typescript
+// Add a relationship between nouns
+await db.addVerb(sourceId, targetId, {
+ verb: VerbType.RelatedTo,
+ // other metadata...
+});
-## Advanced Usage
+// Get all relationships
+const verbs = await db.getAllVerbs();
+
+// Get relationships by source noun
+const outgoingVerbs = await db.getVerbsBySource(sourceId);
+
+// Get relationships by target noun
+const incomingVerbs = await db.getVerbsByTarget(targetId);
+
+// Get relationships by type
+const containsVerbs = await db.getVerbsByType(VerbType.Contains);
+
+// Get a specific relationship
+const verb = await db.getVerb(verbId);
+
+// Delete a relationship
+await db.deleteVerb(verbId);
+```
+
+## βοΈ Advanced Configuration
### Custom Embedding
```typescript
-import {BrainyData, createSimpleEmbeddingFunction} from '@soulcraft/brainy';
+import { BrainyData, createSimpleEmbeddingFunction } from '@soulcraft/brainy';
// Use a custom embedding function (faster but less accurate)
const db = new BrainyData({
@@ -217,10 +444,10 @@ await db.init();
const vector = await db.embed("Some text to convert to a vector");
```
-### Configuration Options
+### Performance Tuning
```typescript
-import {BrainyData, euclideanDistance} from '@soulcraft/brainy';
+import { BrainyData, euclideanDistance } from '@soulcraft/brainy';
// Configure with custom options
const db = new BrainyData({
@@ -254,90 +481,19 @@ const db = new BrainyData({
});
```
-## API Reference
-
-### Database Methods
-
-```typescript
-// Initialize the database
-await db.init();
-
-// Database management
-await db.clear();
-const size = db.size();
-const status = await db.status();
-```
-
-### Noun Methods (Entities)
-
-```typescript
-// Add a noun (automatically vectorized)
-const id = await db.add(textOrVector, {
- noun: NounType.Thing,
- // other metadata...
-});
-
-// Retrieve a noun
-const noun = await db.get(id);
-
-// Update noun metadata
-await db.updateMetadata(id, {
- noun: NounType.Thing,
- // updated metadata...
-});
-
-// Delete a noun
-await db.delete(id);
-
-// Search for similar nouns
-const results = await db.search(vectorOrText, numResults);
-const textResults = await db.searchText("query text", numResults);
-
-// Search by noun type
-const thingNouns = await db.searchByNounTypes([NounType.Thing], numResults);
-```
-
-### Verb Methods (Relationships)
-
-```typescript
-// Add a relationship between nouns
-await db.addVerb(sourceId, targetId, {
- verb: VerbType.RelatedTo,
- // other metadata...
-});
-
-// Get all relationships
-const verbs = await db.getAllVerbs();
-
-// Get relationships by source noun
-const outgoingVerbs = await db.getVerbsBySource(sourceId);
-
-// Get relationships by target noun
-const incomingVerbs = await db.getVerbsByTarget(targetId);
-
-// Get relationships by type
-const containsVerbs = await db.getVerbsByType(VerbType.Contains);
-
-// Get a specific relationship
-const verb = await db.getVerb(verbId);
-
-// Delete a relationship
-await db.deleteVerb(verbId);
-```
-
-### Distance Functions
+## π§ͺ Distance Functions
- `cosineDistance` (default)
- `euclideanDistance`
- `manhattanDistance`
- `dotProductDistance`
-### Embedding Options
+## π Embedding Options
- Default: TensorFlow Universal Sentence Encoder (high quality)
- Alternative: Simple character-based embedding (faster)
-## Extensions
+## π§° Extensions
Brainy includes an augmentation system for extending functionality:
@@ -345,21 +501,10 @@ Brainy includes an augmentation system for extending functionality:
- **Sense Augmentations**: Process raw data
- **Cognition Augmentations**: Reasoning and inference
- **Dialog Augmentations**: Natural language processing
-- **Perception Augmentations**: Data interpretation and/or visualization
+- **Perception Augmentations**: Data interpretation and visualization
- **Activation Augmentations**: Trigger actions
-For detailed documentation on extensions, see the [API docs](https://github.com/soulcraft-labs/brainy/docs).
-
-## Examples
-
-The repository includes several examples:
-
-- Web demo: `examples/demo.html`
-- Basic usage: `examples/basicUsage.js`
-- Custom storage: `examples/customStorage.js`
-- Memory augmentations: `examples/memoryAugmentationExample.js`
-
-## Browser Compatibility
+## π Browser Compatibility
Works in all modern browsers:
@@ -370,10 +515,19 @@ Works in all modern browsers:
For browsers without OPFS support, falls back to in-memory storage.
-## Requirements
+## π Examples
+
+The repository includes several examples:
+
+- Web demo: `examples/demo.html`
+- Basic usage: `examples/basicUsage.js`
+- Custom storage: `examples/customStorage.js`
+- Memory augmentations: `examples/memoryAugmentationExample.js`
+
+## π Requirements
- Node.js >= 18.0.0
-## License
+## π License
-MIT
+[MIT](LICENSE)
diff --git a/cli-wrapper.js b/cli-wrapper.js
new file mode 100755
index 00000000..169e2c96
--- /dev/null
+++ b/cli-wrapper.js
@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+
+/**
+ * CLI Wrapper Script
+ *
+ * This script serves as a wrapper for the Brainy CLI, ensuring that command-line arguments
+ * are properly passed to the CLI when invoked through npm scripts.
+ */
+
+import { spawn } from 'child_process'
+import { fileURLToPath } from 'url'
+import { dirname, join } from 'path'
+import fs from 'fs'
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+// Path to the actual CLI script
+const cliPath = join(__dirname, 'dist', 'cli.js')
+
+// Check if the CLI script exists
+if (!fs.existsSync(cliPath)) {
+ console.error(`Error: CLI script not found at ${cliPath}`)
+ console.error('Make sure you have built the project with "npm run build"')
+ process.exit(1)
+}
+
+// Special handling for version flags
+if (process.argv.includes('--version') || process.argv.includes('-V')) {
+ // Read version directly from package.json to ensure it's always correct
+ try {
+ const packageJsonPath = join(__dirname, 'package.json')
+ const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
+ console.log(packageJson.version)
+ process.exit(0)
+ } catch (error) {
+ console.error('Error loading version information:', error.message)
+ process.exit(1)
+ }
+}
+
+// Forward all arguments to the CLI script
+const args = process.argv.slice(2)
+
+// Check if npm is passing --force flag
+// When npm runs with --force, it sets the npm_config_force environment variable
+if (process.env.npm_config_force === 'true' && args.includes('clear') && !args.includes('--force') && !args.includes('-f')) {
+ args.push('--force')
+}
+
+const cli = spawn('node', [cliPath, ...args], { stdio: 'inherit' })
+
+cli.on('close', (code) => {
+ process.exit(code)
+})
diff --git a/package-lock.json b/package-lock.json
index dad3c8bb..95b9584c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -16,12 +16,13 @@
"@tensorflow/tfjs-backend-cpu": "^4.22.0",
"@tensorflow/tfjs-core": "^4.22.0",
"@tensorflow/tfjs-layers": "^4.22.0",
+ "ascii-tree": "^0.3.0",
"commander": "^14.0.0",
"omelette": "^0.4.17",
"uuid": "^9.0.0"
},
"bin": {
- "brainy": "dist/cli.js"
+ "brainy": "cli-wrapper.js"
},
"devDependencies": {
"@types/jest": "^29.5.3",
@@ -2876,12 +2877,13 @@
}
},
"node_modules/@tensorflow/tfjs-converter": {
- "version": "4.22.0",
- "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.22.0.tgz",
- "integrity": "sha512-PT43MGlnzIo+YfbsjM79Lxk9lOq6uUwZuCc8rrp0hfpLjF6Jv8jS84u2jFb+WpUeuF4K33ZDNx8CjiYrGQ2trQ==",
+ "version": "3.21.0",
+ "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz",
+ "integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==",
"license": "Apache-2.0",
+ "peer": true,
"peerDependencies": {
- "@tensorflow/tfjs-core": "4.22.0"
+ "@tensorflow/tfjs-core": "3.21.0"
}
},
"node_modules/@tensorflow/tfjs-core": {
@@ -2932,6 +2934,15 @@
"@tensorflow/tfjs-core": "4.22.0"
}
},
+ "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-converter": {
+ "version": "4.22.0",
+ "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.22.0.tgz",
+ "integrity": "sha512-PT43MGlnzIo+YfbsjM79Lxk9lOq6uUwZuCc8rrp0hfpLjF6Jv8jS84u2jFb+WpUeuF4K33ZDNx8CjiYrGQ2trQ==",
+ "license": "Apache-2.0",
+ "peerDependencies": {
+ "@tensorflow/tfjs-core": "4.22.0"
+ }
+ },
"node_modules/@types/babel__core": {
"version": "7.20.5",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -3451,6 +3462,15 @@
"node": ">=8"
}
},
+ "node_modules/ascii-tree": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/ascii-tree/-/ascii-tree-0.3.0.tgz",
+ "integrity": "sha512-eC3TvbRQLw+SY0HCVKPXv56JlZ6EeqLOIHT5oHGd0j98ngLDCjTEYbyHYHxrJcb4OwhqhiEo0zYr/FQpj1+Rpw==",
+ "license": "MIT",
+ "dependencies": {
+ "freetree": "0.2.2"
+ }
+ },
"node_modules/async": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz",
@@ -4596,6 +4616,12 @@
"node": ">= 6"
}
},
+ "node_modules/freetree": {
+ "version": "0.2.2",
+ "resolved": "https://registry.npmjs.org/freetree/-/freetree-0.2.2.tgz",
+ "integrity": "sha512-bZ0U0Qv2THYrnkrD7B4UMAhWnDm3+va6y2mNfReke9HY7PRdvLgEf5Vl1j70P5KffoioFvV0RFRfIBQMWOLpIw==",
+ "license": "MIT"
+ },
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
diff --git a/package.json b/package.json
index 4e0b4b99..19ba152d 100644
--- a/package.json
+++ b/package.json
@@ -25,10 +25,13 @@
"node": ">=18.0.0"
},
"scripts": {
+ "prebuild": "node scripts/generate-version.js",
"build": "tsc",
"test": "jest",
"start": "node dist/index.js",
- "cli": "node dist/cli.js",
+ "cli": "node ./cli-wrapper.js",
+ "cli:version": "node ./cli-wrapper.js --version",
+ "cli:v": "node ./cli-wrapper.js -V",
"version:patch": "npm version patch",
"version:minor": "npm version minor",
"version:major": "npm version major",
@@ -36,7 +39,7 @@
"postinstall": "echo 'Note: If you encounter dependency conflicts with TensorFlow.js packages, please use: npm install --legacy-peer-deps'"
},
"bin": {
- "brainy": "dist/cli.js"
+ "brainy": "cli-wrapper.js"
},
"keywords": [
"vector-database",
diff --git a/src/brainyData.ts b/src/brainyData.ts
index d97dfd89..246f0257 100644
--- a/src/brainyData.ts
+++ b/src/brainyData.ts
@@ -628,7 +628,23 @@ export class BrainyData {
// If metadata is provided and no vector is provided or forceEmbed is true, vectorize the metadata
if (options.metadata && (!vector || options.forceEmbed)) {
try {
- verbVector = await this.embeddingFunction(options.metadata)
+ // Extract a string representation from metadata for embedding
+ let textToEmbed: string;
+ if (typeof options.metadata === 'string') {
+ textToEmbed = options.metadata;
+ } else if (options.metadata.description && typeof options.metadata.description === 'string') {
+ textToEmbed = options.metadata.description;
+ } else {
+ // Convert to JSON string as fallback
+ textToEmbed = JSON.stringify(options.metadata);
+ }
+
+ // Ensure textToEmbed is a string
+ if (typeof textToEmbed !== 'string') {
+ textToEmbed = String(textToEmbed);
+ }
+
+ verbVector = await this.embeddingFunction(textToEmbed)
} catch (embedError) {
throw new Error(`Failed to vectorize verb metadata: ${embedError}`)
}
@@ -964,6 +980,144 @@ export class BrainyData {
}
}
}
+
+ /**
+ * Generate a random graph of data with typed nouns and verbs for testing and experimentation
+ * @param options Configuration options for the random graph
+ * @returns Object containing the IDs of the generated nouns and verbs
+ */
+ public async generateRandomGraph(options: {
+ nounCount?: number; // Number of nouns to generate (default: 10)
+ verbCount?: number; // Number of verbs to generate (default: 20)
+ nounTypes?: NounType[]; // Types of nouns to generate (default: all types)
+ verbTypes?: VerbType[]; // Types of verbs to generate (default: all types)
+ clearExisting?: boolean; // Whether to clear existing data before generating (default: false)
+ seed?: string; // Seed for random generation (default: random)
+ } = {}): Promise<{
+ nounIds: string[];
+ verbIds: string[];
+ }> {
+ await this.ensureInitialized();
+
+ // Check if database is in read-only mode
+ this.checkReadOnly();
+
+ // Set default options
+ const nounCount = options.nounCount || 10;
+ const verbCount = options.verbCount || 20;
+ const nounTypes = options.nounTypes || Object.values(NounType);
+ const verbTypes = options.verbTypes || Object.values(VerbType);
+ const clearExisting = options.clearExisting || false;
+
+ // Clear existing data if requested
+ if (clearExisting) {
+ await this.clear();
+ }
+
+ try {
+ // Generate random nouns
+ const nounIds: string[] = [];
+ const nounDescriptions: Record = {
+ [NounType.Person]: "A person with unique characteristics",
+ [NounType.Place]: "A location with specific attributes",
+ [NounType.Thing]: "An object with distinct properties",
+ [NounType.Event]: "An occurrence with temporal aspects",
+ [NounType.Concept]: "An abstract idea or notion",
+ [NounType.Content]: "A piece of content or information",
+ [NounType.Group]: "A collection of related entities",
+ [NounType.List]: "An ordered sequence of items",
+ [NounType.Category]: "A classification or grouping"
+ };
+
+ for (let i = 0; i < nounCount; i++) {
+ // Select a random noun type
+ const nounType = nounTypes[Math.floor(Math.random() * nounTypes.length)];
+
+ // Generate a random label
+ const label = `Random ${nounType} ${i + 1}`;
+
+ // Create metadata
+ const metadata = {
+ noun: nounType,
+ label,
+ description: nounDescriptions[nounType] || `A random ${nounType}`,
+ randomAttributes: {
+ value: Math.random() * 100,
+ priority: Math.floor(Math.random() * 5) + 1,
+ tags: [`tag-${i % 5}`, `category-${i % 3}`]
+ }
+ };
+
+ // Add the noun
+ const id = await this.add(metadata.description, metadata as T);
+ nounIds.push(id);
+ }
+
+ // Generate random verbs between nouns
+ const verbIds: string[] = [];
+ const verbDescriptions: Record = {
+ [VerbType.AttributedTo]: "Attribution relationship",
+ [VerbType.Controls]: "Control relationship",
+ [VerbType.Created]: "Creation relationship",
+ [VerbType.Earned]: "Achievement relationship",
+ [VerbType.Owns]: "Ownership relationship",
+ [VerbType.MemberOf]: "Membership relationship",
+ [VerbType.RelatedTo]: "General relationship",
+ [VerbType.WorksWith]: "Collaboration relationship",
+ [VerbType.FriendOf]: "Friendship relationship",
+ [VerbType.ReportsTo]: "Reporting relationship",
+ [VerbType.Supervises]: "Supervision relationship",
+ [VerbType.Mentors]: "Mentorship relationship"
+ };
+
+ for (let i = 0; i < verbCount; i++) {
+ // Select random source and target nouns
+ const sourceIndex = Math.floor(Math.random() * nounIds.length);
+ let targetIndex = Math.floor(Math.random() * nounIds.length);
+
+ // Ensure source and target are different
+ while (targetIndex === sourceIndex && nounIds.length > 1) {
+ targetIndex = Math.floor(Math.random() * nounIds.length);
+ }
+
+ const sourceId = nounIds[sourceIndex];
+ const targetId = nounIds[targetIndex];
+
+ // Select a random verb type
+ const verbType = verbTypes[Math.floor(Math.random() * verbTypes.length)];
+
+ // Create metadata
+ const metadata = {
+ verb: verbType,
+ description: verbDescriptions[verbType] || `A random ${verbType} relationship`,
+ weight: Math.random(),
+ confidence: Math.random(),
+ randomAttributes: {
+ strength: Math.random() * 100,
+ duration: Math.floor(Math.random() * 365) + 1,
+ tags: [`relation-${i % 5}`, `strength-${i % 3}`]
+ }
+ };
+
+ // Add the verb
+ const id = await this.addVerb(sourceId, targetId, undefined, {
+ type: verbType,
+ weight: metadata.weight,
+ metadata
+ });
+
+ verbIds.push(id);
+ }
+
+ return {
+ nounIds,
+ verbIds
+ };
+ } catch (error) {
+ console.error('Failed to generate random graph:', error);
+ throw new Error(`Failed to generate random graph: ${error}`);
+ }
+ }
}
// Export distance functions for convenience
diff --git a/src/cli.ts b/src/cli.ts
index 26dacfef..3deef7d6 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -11,18 +11,15 @@ import { dirname, join } from 'path'
import fs from 'fs'
import { Command } from 'commander'
import omelette from 'omelette'
+import { VERSION } from './utils/version.js'
+import { sequentialPipeline } from './sequentialPipeline.js'
+import { augmentationPipeline, ExecutionMode } from './augmentationPipeline.js'
+import { AugmentationType } from './types/augmentations.js'
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
-// Import package.json for version info
-const packageJson = {
- name: '@soulcraft/brainy',
- version: '0.6.0',
- description: 'A vector database using HNSW indexing with Origin Private File System storage'
-}
-
// Helper function to parse JSON safely
function parseJSON(str: string): any {
try {
@@ -70,9 +67,9 @@ const program = new Command()
// Configure the program
program
- .name(packageJson.name)
- .description(packageJson.description)
- .version(packageJson.version)
+ .name('@soulcraft/brainy')
+ .description('A vector database using HNSW indexing with Origin Private File System storage')
+ .version(VERSION, '-V, --version', 'Output the current version')
// Create data directory if it doesn't exist
const dataDir = join(__dirname, '..', 'data')
@@ -279,6 +276,323 @@ program
}
})
+program
+ .command('clear')
+ .description('Clear all data from the database')
+ .option('-f, --force', 'Skip confirmation prompt', false)
+ .action(async (options) => {
+ try {
+ // Confirm unless --force is used
+ if (!options.force) {
+ console.log('WARNING: This will permanently delete ALL data in the database.')
+ console.log('To proceed without confirmation, use the --force option.')
+
+ // Exit without doing anything
+ console.log('Operation cancelled. No data was deleted.')
+ console.log('To clear all data, use: brainy clear --force')
+ return
+ }
+
+ const db = createDb()
+ await db.init()
+
+ await db.clear()
+ console.log('Database cleared successfully. All data has been removed.')
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
+program
+ .command('visualize')
+ .description('Visualize the graph structure in ASCII format')
+ .option('-r, --root ', 'ID of the root noun to start visualization from')
+ .option('-d, --depth ', 'Maximum depth of the graph to visualize', '2')
+ .option('-t, --type ', 'Filter by noun type')
+ .option('-l, --limit ', 'Maximum number of nodes to display per level', '10')
+ .action(async (options) => {
+ try {
+ const db = createDb()
+ await db.init()
+
+ // Parse options
+ const depth = parseInt(options.depth, 10)
+ const limit = parseInt(options.limit, 10)
+ const rootId = options.root
+ const nounType = options.type ? resolveNounType(options.type) : undefined
+
+ // Get all nouns if no root is specified
+ if (!rootId && !nounType) {
+ // Get all nouns (limited by the limit option)
+ const allNouns = []
+ let count = 0
+
+ // Since there's no direct method to get all nouns, we'll use search with a high limit
+ const searchResults = await db.search("", 1000, {
+ forceEmbed: true
+ })
+
+ for (const result of searchResults) {
+ if (count >= limit) break
+ allNouns.push(result)
+ count++
+ }
+
+ if (allNouns.length === 0) {
+ console.log('No nouns found in the database.')
+ return
+ }
+
+ console.log(`Graph Overview (showing ${allNouns.length} nouns):\n`)
+
+ for (const noun of allNouns) {
+ // Get outgoing verbs
+ const outgoingVerbs = await db.getVerbsBySource(noun.id)
+ // Get incoming verbs
+ const incomingVerbs = await db.getVerbsByTarget(noun.id)
+
+ const nounType = noun.metadata?.noun || 'Unknown'
+ const label = noun.metadata?.label || noun.id.substring(0, 8)
+
+ console.log(`[${nounType}] ${label} (${noun.id})`)
+
+ if (outgoingVerbs.length > 0) {
+ console.log(' Outgoing:')
+ for (const verb of outgoingVerbs.slice(0, limit)) {
+ const targetNoun = await db.get(verb.targetId)
+ const targetLabel = targetNoun?.metadata?.label || verb.targetId.substring(0, 8)
+ console.log(` --(${verb.metadata?.verb || 'relates to'})--β [${targetNoun?.metadata?.noun || 'Unknown'}] ${targetLabel}`)
+ }
+ if (outgoingVerbs.length > limit) {
+ console.log(` ... and ${outgoingVerbs.length - limit} more`)
+ }
+ }
+
+ if (incomingVerbs.length > 0) {
+ console.log(' Incoming:')
+ for (const verb of incomingVerbs.slice(0, limit)) {
+ const sourceNoun = await db.get(verb.sourceId)
+ const sourceLabel = sourceNoun?.metadata?.label || verb.sourceId.substring(0, 8)
+ console.log(` β--(${verb.metadata?.verb || 'relates to'})-- [${sourceNoun?.metadata?.noun || 'Unknown'}] ${sourceLabel}`)
+ }
+ if (incomingVerbs.length > limit) {
+ console.log(` ... and ${incomingVerbs.length - limit} more`)
+ }
+ }
+
+ console.log('')
+ }
+
+ return
+ }
+
+ // If noun type is specified but no root, show all nouns of that type
+ if (!rootId && nounType) {
+ console.log(`Visualizing nouns of type: ${nounType}\n`)
+
+ // Search for nouns of the specified type
+ const searchResults = await db.search("", 1000, {
+ nounTypes: [nounType],
+ forceEmbed: true
+ })
+
+ const filteredNouns = searchResults.slice(0, limit)
+
+ if (filteredNouns.length === 0) {
+ console.log(`No nouns found with type: ${nounType}`)
+ return
+ }
+
+ for (const noun of filteredNouns) {
+ const label = noun.metadata?.label || noun.id.substring(0, 8)
+
+ console.log(`[${nounType}] ${label} (${noun.id})`)
+
+ // Get outgoing verbs
+ const outgoingVerbs = await db.getVerbsBySource(noun.id)
+ if (outgoingVerbs.length > 0) {
+ console.log(' Outgoing:')
+ for (const verb of outgoingVerbs.slice(0, limit)) {
+ const targetNoun = await db.get(verb.targetId)
+ const targetLabel = targetNoun?.metadata?.label || verb.targetId.substring(0, 8)
+ console.log(` --(${verb.metadata?.verb || 'relates to'})--β [${targetNoun?.metadata?.noun || 'Unknown'}] ${targetLabel}`)
+ }
+ if (outgoingVerbs.length > limit) {
+ console.log(` ... and ${outgoingVerbs.length - limit} more`)
+ }
+ }
+
+ console.log('')
+ }
+
+ return
+ }
+
+ // If root is specified, visualize the graph starting from that root
+ if (rootId) {
+ const rootNoun = await db.get(rootId)
+ if (!rootNoun) {
+ console.error(`Root noun with ID ${rootId} not found`)
+ return
+ }
+
+ console.log(`Visualizing graph from root: ${rootNoun.metadata?.label || rootId}\n`)
+
+ // Use a breadth-first search to visualize the graph
+ const visited = new Set()
+ const queue: Array<{ id: string; level: number; path: string }> = [
+ { id: rootId, level: 0, path: '' }
+ ]
+
+ while (queue.length > 0) {
+ const { id, level, path } = queue.shift()!
+
+ if (visited.has(id) || level > depth) {
+ continue
+ }
+
+ visited.add(id)
+
+ const noun = await db.get(id)
+ if (!noun) {
+ console.warn(`Noun with ID ${id} not found`)
+ continue
+ }
+
+ const nounType = noun.metadata?.noun || 'Unknown'
+ const label = noun.metadata?.label || id.substring(0, 8)
+
+ // Print the current noun with proper indentation
+ console.log(`${' '.repeat(level * 2)}${path}[${nounType}] ${label} (${id})`)
+
+ // Get outgoing verbs
+ const outgoingVerbs = await db.getVerbsBySource(id)
+
+ // Add target nouns to the queue for the next level
+ let verbCount = 0
+ for (const verb of outgoingVerbs) {
+ if (verbCount >= limit) {
+ console.log(`${' '.repeat((level + 1) * 2)}... and ${outgoingVerbs.length - limit} more`)
+ break
+ }
+
+ const verbType = verb.metadata?.verb || 'relates to'
+ queue.push({
+ id: verb.targetId,
+ level: level + 1,
+ path: `--(${verbType})--β `
+ })
+ verbCount++
+ }
+ }
+ }
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
+program
+ .command('generate-random-graph')
+ .description('Generate a random graph of data with typed nouns and verbs for testing')
+ .option('-n, --noun-count ', 'Number of nouns to generate', '10')
+ .option('-v, --verb-count ', 'Number of verbs to generate', '20')
+ .option('-c, --clear', 'Clear existing data before generating', false)
+ .option('-t, --noun-types ', 'Comma-separated list of noun types to use')
+ .option('-r, --verb-types ', 'Comma-separated list of verb types to use')
+ .action(async (options) => {
+ try {
+ const db = createDb()
+ await db.init()
+
+ // Parse options
+ const nounCount = parseInt(options.nounCount, 10)
+ const verbCount = parseInt(options.verbCount, 10)
+ const clearExisting = options.clear
+
+ // Parse noun types if provided
+ let nounTypes: NounType[] | undefined
+ if (options.nounTypes) {
+ const typeNames = options.nounTypes.split(',').map((t: string) => t.trim())
+ nounTypes = typeNames.map((name: string) => {
+ // Try to match by key name (case insensitive)
+ const key = Object.keys(NounType).find(
+ k => k.toLowerCase() === name.toLowerCase()
+ )
+ if (key) return NounType[key as keyof typeof NounType]
+
+ // If not found by key, check if it's a valid value
+ if (Object.values(NounType).includes(name as NounType)) {
+ return name as NounType
+ }
+
+ console.warn(`Warning: Unknown noun type "${name}", ignoring`)
+ return null
+ }).filter(Boolean) as NounType[]
+ }
+
+ // Parse verb types if provided
+ let verbTypes: VerbType[] | undefined
+ if (options.verbTypes) {
+ const typeNames = options.verbTypes.split(',').map((t: string) => t.trim())
+ verbTypes = typeNames.map((name: string) => {
+ // Try to match by key name (case insensitive)
+ const key = Object.keys(VerbType).find(
+ k => k.toLowerCase() === name.toLowerCase()
+ )
+ if (key) return VerbType[key as keyof typeof VerbType]
+
+ // If not found by key, check if it's a valid value
+ if (Object.values(VerbType).includes(name as VerbType)) {
+ return name as VerbType
+ }
+
+ console.warn(`Warning: Unknown verb type "${name}", ignoring`)
+ return null
+ }).filter(Boolean) as VerbType[]
+ }
+
+ console.log(`Generating random graph with ${nounCount} nouns and ${verbCount} verbs...`)
+ if (clearExisting) {
+ console.log('Clearing existing data first...')
+ }
+
+ const result = await db.generateRandomGraph({
+ nounCount,
+ verbCount,
+ nounTypes,
+ verbTypes,
+ clearExisting
+ })
+
+ console.log('Random graph generated successfully!')
+ console.log(`Created ${result.nounIds.length} nouns and ${result.verbIds.length} verbs`)
+
+ // Print some sample IDs
+ if (result.nounIds.length > 0) {
+ console.log('\nSample noun IDs:')
+ result.nounIds.slice(0, 3).forEach(id => console.log(`- ${id}`))
+ if (result.nounIds.length > 3) {
+ console.log(`... and ${result.nounIds.length - 3} more`)
+ }
+ }
+
+ if (result.verbIds.length > 0) {
+ console.log('\nSample verb IDs:')
+ result.verbIds.slice(0, 3).forEach(id => console.log(`- ${id}`))
+ if (result.verbIds.length > 3) {
+ console.log(`... and ${result.verbIds.length - 3} more`)
+ }
+ }
+
+ console.log('\nUse the search, get, or visualize commands to explore the generated graph')
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
// Add examples to help text
program.addHelpText('after', `
Examples:
@@ -286,6 +600,11 @@ Examples:
$ brainy add "Cats are independent pets" '{"noun":"Thing","category":"animal"}'
$ brainy search "feline pets" --limit 5
$ brainy addVerb id1 id2 RelatedTo '{"description":"Both are pets"}'
+ $ brainy clear --force
+ $ brainy generate-random-graph --noun-count 20 --verb-count 30 --clear
+ $ brainy generate-random-graph --noun-types Person,Thing --verb-types RelatedTo,Owns
+ $ brainy visualize --type Thing --limit 10
+ $ brainy visualize --root id1 --depth 3
`)
// Setup autocomplete
@@ -308,16 +627,23 @@ completion.tree({
'delete',
'getVerbs',
'status',
+ 'clear',
+ 'visualize',
+ 'generate-random-graph',
'completion-setup',
'init',
- 'help'
+ 'help',
+ 'list-augmentations',
+ 'augmentation-info',
+ 'test-pipeline',
+ 'stream-test'
],
// Command-specific completions
add: {
// For the second argument of 'add' command (metadata)
_: () => {
// Generate templates for each noun type
- return getNounTypes().map(type =>
+ return getNounTypes().map(type =>
`{"noun":"${type}","category":"example"}`
)
}
@@ -339,6 +665,51 @@ completion.tree({
delete: {},
getVerbs: {},
status: {},
+ clear: {
+ _: () => [
+ '--force'
+ ]
+ },
+ 'generate-random-graph': {
+ _: () => [
+ '--noun-count 10',
+ '--verb-count 20',
+ '--clear',
+ `--noun-types ${getNounTypes().join(',')}`,
+ `--verb-types ${getVerbTypes().join(',')}`
+ ]
+ },
+ 'list-augmentations': {},
+ 'augmentation-info': {
+ _: () => [
+ 'sense',
+ 'memory',
+ 'cognition',
+ 'conduit',
+ 'activation',
+ 'perception',
+ 'dialog',
+ 'websocket'
+ ]
+ },
+ 'test-pipeline': {
+ _: () => [
+ '--data-type text',
+ '--mode sequential',
+ '--mode parallel',
+ '--mode threaded',
+ '--stop-on-error',
+ '--verbose'
+ ]
+ },
+ 'stream-test': {
+ _: () => [
+ '--count 5',
+ '--interval 1000',
+ '--data-type text',
+ '--verbose'
+ ]
+ },
'completion-setup': {},
init: {},
help: {}
@@ -354,6 +725,326 @@ if (process.argv.includes('--completion-setup')) {
process.exit(0)
}
+// Pipeline and Augmentation Commands
+program
+ .command('list-augmentations')
+ .description('List all available augmentation types and registered augmentations')
+ .action(async () => {
+ try {
+ // Initialize the pipeline
+ await augmentationPipeline.initialize()
+
+ // Get available augmentation types
+ const availableTypes = augmentationPipeline.getAvailableAugmentationTypes()
+
+ console.log('Available Augmentation Types:')
+ if (availableTypes.length === 0) {
+ console.log(' No augmentation types available')
+ } else {
+ availableTypes.forEach(type => {
+ const augmentations = augmentationPipeline.getAugmentationsByType(type)
+ console.log(`\n${type.toUpperCase()} (${augmentations.length} registered):`)
+
+ if (augmentations.length === 0) {
+ console.log(' No augmentations registered for this type')
+ } else {
+ augmentations.forEach(aug => {
+ console.log(` - ${aug.name}: ${aug.description}`)
+ console.log(` Status: ${aug.enabled ? 'Enabled' : 'Disabled'}`)
+ })
+ }
+ })
+ }
+
+ // Show WebSocket augmentations separately
+ const webSocketAugs = augmentationPipeline.getWebSocketAugmentations()
+ console.log('\nWebSocket-Enabled Augmentations:')
+ if (webSocketAugs.length === 0) {
+ console.log(' No WebSocket-enabled augmentations available')
+ } else {
+ webSocketAugs.forEach(aug => {
+ console.log(` - ${aug.name}: ${aug.description}`)
+ console.log(` Status: ${aug.enabled ? 'Enabled' : 'Disabled'}`)
+ })
+ }
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
+program
+ .command('test-pipeline')
+ .description('Test the sequential pipeline with sample data')
+ .argument('[text]', 'Sample text to process through the pipeline', 'This is a test of the Brainy pipeline')
+ .option('-t, --data-type ', 'Type of data to process', 'text')
+ .option('-m, --mode ', 'Execution mode (sequential, parallel, threaded)', 'sequential')
+ .option('-s, --stop-on-error', 'Stop execution if an error occurs', false)
+ .option('-v, --verbose', 'Show detailed output', false)
+ .action(async (text, options) => {
+ try {
+ // Initialize the pipeline
+ await sequentialPipeline.initialize()
+
+ console.log(`Processing data: "${text}"`)
+ console.log(`Data type: ${options.dataType}`)
+ console.log(`Execution mode: ${options.mode}`)
+ console.log(`Stop on error: ${options.stopOnError}`)
+ console.log()
+
+ // Set execution mode
+ let executionMode = ExecutionMode.SEQUENTIAL
+ switch (options.mode.toLowerCase()) {
+ case 'parallel':
+ executionMode = ExecutionMode.PARALLEL
+ break
+ case 'threaded':
+ executionMode = ExecutionMode.THREADED
+ break
+ default:
+ executionMode = ExecutionMode.SEQUENTIAL
+ }
+
+ // Process the data
+ const result = await sequentialPipeline.processData(
+ text,
+ options.dataType,
+ {
+ stopOnError: options.stopOnError,
+ timeout: 30000
+ }
+ )
+
+ console.log('Pipeline Execution Result:')
+ console.log(`Success: ${result.success}`)
+
+ if (result.error) {
+ console.log(`Error: ${result.error}`)
+ }
+
+ console.log('\nStage Results:')
+
+ // Display stage results
+ Object.entries(result.stageResults).forEach(([stage, stageResult]) => {
+ console.log(`\n${stage.toUpperCase()}:`)
+ console.log(` Success: ${stageResult?.success}`)
+
+ if (stageResult?.error) {
+ console.log(` Error: ${stageResult.error}`)
+ }
+
+ if (stageResult?.data && options.verbose) {
+ console.log(' Data:')
+ console.log(JSON.stringify(stageResult.data, null, 2)
+ .split('\n')
+ .map(line => ` ${line}`)
+ .join('\n')
+ )
+ }
+ })
+
+ console.log('\nFinal Result Data:')
+ console.log(JSON.stringify(result.data, null, 2)
+ .split('\n')
+ .map(line => ` ${line}`)
+ .join('\n')
+ )
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
+program
+ .command('stream-test')
+ .description('Test streaming data through the pipeline (simulated)')
+ .option('-c, --count ', 'Number of data items to stream', '5')
+ .option('-i, --interval ', 'Interval between data items in milliseconds', '1000')
+ .option('-t, --data-type ', 'Type of data to process', 'text')
+ .option('-v, --verbose', 'Show detailed output', false)
+ .action(async (options) => {
+ try {
+ // Initialize the pipeline
+ await sequentialPipeline.initialize()
+
+ const count = parseInt(options.count, 10)
+ const interval = parseInt(options.interval, 10)
+
+ console.log(`Simulating stream of ${count} data items at ${interval}ms intervals`)
+ console.log(`Data type: ${options.dataType}`)
+ console.log()
+
+ // Create a handler function similar to what would be used with WebSockets
+ const handler = (data: string) => {
+ // Process the data asynchronously without blocking
+ sequentialPipeline.processData(
+ data,
+ options.dataType,
+ { stopOnError: false }
+ ).then(result => {
+ console.log(`\nProcessed: "${data}"`)
+ console.log(`Success: ${result.success}`)
+
+ if (options.verbose) {
+ console.log('Stage Results:')
+ Object.entries(result.stageResults).forEach(([stage, stageResult]) => {
+ if (stageResult?.success) {
+ console.log(` ${stage}: Success`)
+ } else {
+ console.log(` ${stage}: Failed - ${stageResult?.error || 'Unknown error'}`)
+ }
+ })
+ }
+
+ if (result.data) {
+ console.log('Result Data:')
+ console.log(JSON.stringify(result.data, null, 2)
+ .split('\n')
+ .map(line => ` ${line}`)
+ .join('\n')
+ )
+ }
+ }).catch(error => {
+ console.error(`Error processing "${data}":`, error.message)
+ })
+ }
+
+ // Generate sample data items
+ const sampleTexts = [
+ "The quick brown fox jumps over the lazy dog",
+ "Artificial intelligence is transforming how we interact with data",
+ "Vector databases enable semantic search capabilities",
+ "Graph relationships connect entities in meaningful ways",
+ "Streaming data requires efficient real-time processing",
+ "WebSockets provide bidirectional communication channels",
+ "Augmentations extend the functionality of the core system",
+ "Sequential pipelines process data in defined stages",
+ "Parallel execution improves throughput for large datasets",
+ "Threaded operations utilize multiple CPU cores efficiently"
+ ]
+
+ // Simulate streaming data
+ console.log('Starting simulated data stream...')
+
+ for (let i = 0; i < count; i++) {
+ // Use modulo to cycle through sample texts if count > samples
+ const text = sampleTexts[i % sampleTexts.length]
+
+ // Wait for the specified interval
+ if (i > 0) {
+ await new Promise(resolve => setTimeout(resolve, interval))
+ }
+
+ console.log(`\nStreaming item ${i + 1}/${count}: "${text}"`)
+
+ // Process the data
+ handler(text)
+ }
+
+ console.log('\nSimulated stream complete. Some processing may still be ongoing.')
+ console.log('In a real WebSocket scenario, the connection would remain open for continuous data.')
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
+program
+ .command('augmentation-info')
+ .description('Get detailed information about a specific augmentation type')
+ .argument('', 'Augmentation type (sense, memory, cognition, conduit, activation, perception, dialog, websocket)')
+ .action(async (typeArg) => {
+ try {
+ // Initialize the pipeline
+ await augmentationPipeline.initialize()
+
+ // Resolve the augmentation type
+ let augType: AugmentationType | undefined
+
+ // Convert input to proper enum value
+ const normalizedType = typeArg.toLowerCase()
+ switch (normalizedType) {
+ case 'sense':
+ augType = AugmentationType.SENSE
+ break
+ case 'memory':
+ augType = AugmentationType.MEMORY
+ break
+ case 'cognition':
+ augType = AugmentationType.COGNITION
+ break
+ case 'conduit':
+ augType = AugmentationType.CONDUIT
+ break
+ case 'activation':
+ augType = AugmentationType.ACTIVATION
+ break
+ case 'perception':
+ augType = AugmentationType.PERCEPTION
+ break
+ case 'dialog':
+ augType = AugmentationType.DIALOG
+ break
+ case 'websocket':
+ augType = AugmentationType.WEBSOCKET
+ break
+ default:
+ console.error(`Unknown augmentation type: ${typeArg}`)
+ console.log('Available types: sense, memory, cognition, conduit, activation, perception, dialog, websocket')
+ process.exit(1)
+ }
+
+ // Get augmentations of the specified type
+ const augmentations = augmentationPipeline.getAugmentationsByType(augType)
+
+ console.log(`\n${augType.toUpperCase()} Augmentation Details:`)
+
+ if (augmentations.length === 0) {
+ console.log(' No augmentations registered for this type')
+ } else {
+ // Display information about each augmentation
+ augmentations.forEach((aug, index) => {
+ console.log(`\n${index + 1}. ${aug.name}`)
+ console.log(` Description: ${aug.description}`)
+ console.log(` Status: ${aug.enabled ? 'Enabled' : 'Disabled'}`)
+
+ // List available methods
+ console.log(' Available Methods:')
+
+ // Get all methods that aren't from Object.prototype
+ const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(aug))
+ .filter(method =>
+ method !== 'constructor' &&
+ typeof (aug as any)[method] === 'function' &&
+ !['initialize', 'shutDown', 'getStatus'].includes(method)
+ )
+
+ if (methods.length === 0) {
+ console.log(' No custom methods available')
+ } else {
+ methods.forEach(method => {
+ console.log(` - ${method}`)
+ })
+ }
+ })
+ }
+
+ // Show pipeline order information
+ console.log('\nPipeline Execution Order:')
+ console.log(' 1. SENSE - Process raw data into structured nouns and verbs')
+ console.log(' 2. MEMORY - Store and retrieve data')
+ console.log(' 3. COGNITION - Analyze and reason about data')
+ console.log(' 4. CONDUIT - Exchange data with external systems')
+ console.log(' 5. ACTIVATION - Trigger actions based on data')
+ console.log(' 6. PERCEPTION - Interpret and visualize data')
+ console.log(' 7. DIALOG - Process natural language interactions')
+ console.log(' * WEBSOCKET - Enable real-time communication (can be combined with other types)')
+ } catch (error) {
+ console.error('Error:', (error as Error).message)
+ process.exit(1)
+ }
+ })
+
// Add a command for setting up autocomplete
program
.command('completion-setup')
diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts
index 3082ca49..ee4ff753 100644
--- a/src/hnsw/hnswIndex.ts
+++ b/src/hnsw/hnswIndex.ts
@@ -114,7 +114,7 @@ export class HNSWIndex {
for (const neighborId of connections) {
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
- console.error(`Neighbor with ID ${neighborId} not found in addItem traversal`)
+ // Skip neighbors that don't exist (expected during rapid additions/deletions)
continue
}
const distToNeighbor = this.distanceFunction(vector, neighbor.vector)
@@ -149,7 +149,7 @@ export class HNSWIndex {
for (const [neighborId, _] of neighbors) {
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
- console.error(`Neighbor with ID ${neighborId} not found`)
+ // Skip neighbors that don't exist (expected during rapid additions/deletions)
continue
}
@@ -240,7 +240,7 @@ export class HNSWIndex {
for (const neighborId of connections) {
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
- console.error(`Neighbor with ID ${neighborId} not found in search`)
+ // Skip neighbors that don't exist (expected during rapid additions/deletions)
continue
}
const distToNeighbor = this.distanceFunction(
@@ -284,7 +284,7 @@ export class HNSWIndex {
for (const neighborId of connections) {
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
- console.error(`Neighbor with ID ${neighborId} not found in removeItem`)
+ // Skip neighbors that don't exist (expected during rapid additions/deletions)
continue
}
if (neighbor.connections.has(level)) {
@@ -432,7 +432,7 @@ export class HNSWIndex {
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
- console.error(`Neighbor with ID ${neighborId} not found in searchLayer`)
+ // Skip neighbors that don't exist (expected during rapid additions/deletions)
continue
}
const distToNeighbor = this.distanceFunction(
@@ -501,7 +501,7 @@ export class HNSWIndex {
for (const neighborId of connections) {
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
- console.error(`Neighbor with ID ${neighborId} not found in pruneConnections`)
+ // Skip neighbors that don't exist (expected during rapid additions/deletions)
continue
}
diff --git a/src/storage/fileSystemStorage.ts b/src/storage/fileSystemStorage.ts
index bc8cd2c0..55810553 100644
--- a/src/storage/fileSystemStorage.ts
+++ b/src/storage/fileSystemStorage.ts
@@ -669,7 +669,7 @@ export class FileSystemStorage implements StorageAdapter {
}
/**
- * Delete a directory and all its contents
+ * Delete a directory and all its contents recursively
*/
private async deleteDirectory(dirPath: string): Promise {
try {
@@ -677,8 +677,19 @@ export class FileSystemStorage implements StorageAdapter {
for (const file of files) {
const filePath = path.join(dirPath, file)
- await fs.promises.unlink(filePath)
+ const stats = await fs.promises.stat(filePath)
+
+ if (stats.isDirectory()) {
+ // Recursively delete subdirectories
+ await this.deleteDirectory(filePath)
+ } else {
+ // Delete files
+ await fs.promises.unlink(filePath)
+ }
}
+
+ // After all contents are deleted, remove the directory itself
+ await fs.promises.rmdir(dirPath)
} catch (error) {
// If the directory doesn't exist, that's fine
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {