soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main'

Browse source
This commit is contained in:
David Snelling 2025-07-23 12:39:19 -07:00
commit b8f10ba39a
6 changed files with 399 additions and 313 deletions
Download patch file Download diff file Expand all files Collapse all files

610
README.md
View file

@ -61,7 +61,8 @@ GitHub Pages that showcases Brainy's main features.
npm install @soulcraft/brainy
```
TensorFlow.js packages are included as bundled dependencies and will be automatically installed without any additional configuration.
TensorFlow.js packages are included as bundled dependencies and will be automatically installed without any additional
configuration.
### Additional Packages
@ -84,7 +85,7 @@ REST API web service wrapper that provides HTTP endpoints for search operations
Brainy uses a unified build that automatically adapts to your environment (Node.js, browser, or serverless):
```typescript
import { BrainyData, NounType, VerbType } from '@soulcraft/brainy'
import {BrainyData, NounType, VerbType} from '@soulcraft/brainy'
// Create and initialize the database
const db = new BrainyData()
@ -92,13 +93,13 @@ await db.init()
// Add data (automatically converted to vectors)
const catId = await db.add("Cats are independent pets", {
noun: NounType.Thing,
category: 'animal'
noun: NounType.Thing,
category: 'animal'
})
const dogId = await db.add("Dogs are loyal companions", {
noun: NounType.Thing,
category: 'animal'
noun: NounType.Thing,
category: 'animal'
})
// Search for similar items
@ -107,8 +108,8 @@ console.log(results)
// Add a relationship between items
await db.addVerb(catId, dogId, {
verb: VerbType.RelatedTo,
description: 'Both are common household pets'
verb: VerbType.RelatedTo,
description: 'Both are common household pets'
})
```
@ -116,28 +117,30 @@ await db.addVerb(catId, dogId, {
```typescript
// Standard import - automatically adapts to any environment
import { BrainyData } from '@soulcraft/brainy'
import {BrainyData} from '@soulcraft/brainy'
// Minified version for production
import { BrainyData } from '@soulcraft/brainy/min'
import {BrainyData} from '@soulcraft/brainy/min'
```
> **Note**: The CLI functionality is available as a separate package `@soulcraft/brainy-cli` to reduce the bundle size of the main package. Install it globally with `npm install -g @soulcraft/brainy-cli` to use the command-line interface.
> **Note**: The CLI functionality is available as a separate package `@soulcraft/brainy-cli` to reduce the bundle size
> of the main package. Install it globally with `npm install -g @soulcraft/brainy-cli` to use the command-line
> interface.
### Browser Usage
```html
<script type="module">
// Use local files instead of CDN
import { BrainyData } from './dist/unified.js'
// Use local files instead of CDN
import {BrainyData} from './dist/unified.js'
// Or minified version
// import { BrainyData } from './dist/unified.min.js'
// Or minified version
// import { BrainyData } from './dist/unified.min.js'
const db = new BrainyData()
await db.init()
// ...
const db = new BrainyData()
await db.init()
// ...
</script>
```
@ -292,13 +295,13 @@ The pipeline runs automatically when you:
```typescript
// Add data (runs embedding → indexing → storage)
const id = await db.add("Your text data here", { metadata })
const id = await db.add("Your text data here", {metadata})
// Search (runs embedding → similarity search)
const results = await db.searchText("Your query here", 5)
// Connect entities (runs graph construction → storage)
await db.addVerb(sourceId, targetId, { verb: VerbType.RelatedTo })
await db.addVerb(sourceId, targetId, {verb: VerbType.RelatedTo})
```
Using the CLI:
@ -423,7 +426,8 @@ brainy visualize --root <id> --depth 3
### Using the CLI in Your Code
The CLI functionality is available as a separate package `@soulcraft/brainy-cli`. If you need CLI functionality in your application, install the CLI package:
The CLI functionality is available as a separate package `@soulcraft/brainy-cli`. If you need CLI functionality in your
application, install the CLI package:
```bash
npm install @soulcraft/brainy-cli
@ -481,7 +485,7 @@ const status = await db.status()
const backupData = await db.backup()
// Restore data into the database
const restoreResult = await db.restore(backupData, { clearExisting: true })
const restoreResult = await db.restore(backupData, {clearExisting: true})
```
### Working with Nouns (Entities)
@ -489,25 +493,25 @@ const restoreResult = await db.restore(backupData, { clearExisting: true })
```typescript
// Add a noun (automatically vectorized)
const id = await db.add(textOrVector, {
noun: NounType.Thing,
// other metadata...
noun: NounType.Thing,
// other metadata...
})
// Add multiple nouns in parallel (with multithreading and batch embedding)
const ids = await db.addBatch([
{
vectorOrData: "First item to add",
metadata: { noun: NounType.Thing, category: 'example' }
},
{
vectorOrData: "Second item to add",
metadata: { noun: NounType.Thing, category: 'example' }
},
// More items...
{
vectorOrData: "First item to add",
metadata: {noun: NounType.Thing, category: 'example'}
},
{
vectorOrData: "Second item to add",
metadata: {noun: NounType.Thing, category: 'example'}
},
// More items...
], {
forceEmbed: false,
concurrency: 4, // Control the level of parallelism (default: 4)
batchSize: 50 // Control the number of items to process in a single batch (default: 50)
forceEmbed: false,
concurrency: 4, // Control the level of parallelism (default: 4)
batchSize: 50 // Control the number of items to process in a single batch (default: 50)
})
// Retrieve a noun
@ -515,8 +519,8 @@ const noun = await db.get(id)
// Update noun metadata
await db.updateMetadata(id, {
noun: NounType.Thing,
// updated metadata...
noun: NounType.Thing,
// updated metadata...
})
// Delete a noun
@ -535,8 +539,21 @@ const thingNouns = await db.searchByNounTypes([NounType.Thing], numResults)
```typescript
// Add a relationship between nouns
await db.addVerb(sourceId, targetId, {
verb: VerbType.RelatedTo,
// other metadata...
verb: VerbType.RelatedTo,
// other metadata...
})
// Add a relationship with auto-creation of missing nouns
// This is useful when the target noun might not exist yet
await db.addVerb(sourceId, targetId, {
verb: VerbType.RelatedTo,
// Enable auto-creation of missing nouns
autoCreateMissingNouns: true,
// Optional metadata for auto-created nouns
missingNounMetadata: {
noun: NounType.Concept,
description: 'Auto-created noun'
}
})
// Get all relationships
@ -564,20 +581,20 @@ await db.deleteVerb(verbId)
```typescript
import {
BrainyData,
createTensorFlowEmbeddingFunction,
createThreadedEmbeddingFunction
BrainyData,
createTensorFlowEmbeddingFunction,
createThreadedEmbeddingFunction
} from '@soulcraft/brainy'
// Use the standard TensorFlow Universal Sentence Encoder embedding function
const db = new BrainyData({
embeddingFunction: createTensorFlowEmbeddingFunction()
embeddingFunction: createTensorFlowEmbeddingFunction()
})
await db.init()
// Or use the threaded embedding function for better performance
const threadedDb = new BrainyData({
embeddingFunction: createThreadedEmbeddingFunction()
embeddingFunction: createThreadedEmbeddingFunction()
})
await threadedDb.init()
@ -618,42 +635,42 @@ Brainy includes comprehensive multithreading support to improve performance acro
7. **Automatic Environment Detection**: Adapts to browser (Web Workers) and Node.js (Worker Threads) environments
```typescript
import { BrainyData, euclideanDistance } from '@soulcraft/brainy'
import {BrainyData, euclideanDistance} from '@soulcraft/brainy'
// Configure with custom options
const db = new BrainyData({
// Use Euclidean distance instead of default cosine distance
distanceFunction: euclideanDistance,
// Use Euclidean distance instead of default cosine distance
distanceFunction: euclideanDistance,
// HNSW index configuration for search performance
hnsw: {
M: 16, // Max connections per noun
efConstruction: 200, // Construction candidate list size
efSearch: 50, // Search candidate list size
},
// HNSW index configuration for search performance
hnsw: {
M: 16, // Max connections per noun
efConstruction: 200, // Construction candidate list size
efSearch: 50, // Search candidate list size
},
// Performance optimization options
performance: {
useParallelization: true, // Enable multithreaded search operations
},
// Performance optimization options
performance: {
useParallelization: true, // Enable multithreaded search operations
},
// Noun and Verb type validation
typeValidation: {
enforceNounTypes: true, // Validate noun types against NounType enum
enforceVerbTypes: true, // Validate verb types against VerbType enum
},
// Noun and Verb type validation
typeValidation: {
enforceNounTypes: true, // Validate noun types against NounType enum
enforceVerbTypes: true, // Validate verb types against VerbType enum
},
// Storage configuration
storage: {
requestPersistentStorage: true,
// Example configuration for cloud storage (replace with your own values):
// s3Storage: {
// bucketName: 'your-s3-bucket-name',
// region: 'your-aws-region'
// // Credentials should be provided via environment variables
// // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
// }
}
// Storage configuration
storage: {
requestPersistentStorage: true,
// Example configuration for cloud storage (replace with your own values):
// s3Storage: {
// bucketName: 'your-s3-bucket-name',
// region: 'your-aws-region'
// // Credentials should be provided via environment variables
// // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
// }
}
})
```
@ -667,34 +684,34 @@ hybrid approach:
3. **Memory-Efficient Indexing** - Optimizes memory usage for large-scale vector collections
```typescript
import { BrainyData } from '@soulcraft/brainy'
import {BrainyData} from '@soulcraft/brainy'
// Configure with optimized HNSW index for large datasets
const db = new BrainyData({
hnswOptimized: {
// Standard HNSW parameters
M: 16, // Max connections per noun
efConstruction: 200, // Construction candidate list size
efSearch: 50, // Search candidate list size
hnswOptimized: {
// Standard HNSW parameters
M: 16, // Max connections per noun
efConstruction: 200, // Construction candidate list size
efSearch: 50, // Search candidate list size
// Memory threshold in bytes - when exceeded, will use disk-based approach
memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
// Memory threshold in bytes - when exceeded, will use disk-based approach
memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
// Product quantization settings for dimensionality reduction
productQuantization: {
enabled: true, // Enable product quantization
numSubvectors: 16, // Number of subvectors to split the vector into
numCentroids: 256 // Number of centroids per subvector
// Product quantization settings for dimensionality reduction
productQuantization: {
enabled: true, // Enable product quantization
numSubvectors: 16, // Number of subvectors to split the vector into
numCentroids: 256 // Number of centroids per subvector
},
// Whether to use disk-based storage for the index
useDiskBasedIndex: true // Enable disk-based storage
},
// Whether to use disk-based storage for the index
useDiskBasedIndex: true // Enable disk-based storage
},
// Storage configuration (required for disk-based index)
storage: {
requestPersistentStorage: true
}
// Storage configuration (required for disk-based index)
storage: {
requestPersistentStorage: true
}
})
// The optimized index automatically adapts based on dataset size:
@ -759,24 +776,24 @@ Brainy's restore functionality can handle:
```typescript
// Restore data with all options
const restoreResult = await db.restore(backupData, {
clearExisting: true // Whether to clear existing data before restore
clearExisting: true // Whether to clear existing data before restore
})
// Import sparse data (without vectors)
// Vectors will be automatically created using the embedding function
const sparseData = {
nouns: [
{
id: '123',
// No vector field - will be created during import
metadata: {
noun: 'Thing',
text: 'This text will be used to generate a vector'
}
}
],
verbs: [],
version: '1.0.0'
nouns: [
{
id: '123',
// No vector field - will be created during import
metadata: {
noun: 'Thing',
text: 'This text will be used to generate a vector'
}
}
],
verbs: [],
version: '1.0.0'
}
const sparseImportResult = await db.importSparseData(sparseData)
@ -823,82 +840,82 @@ boilerplate:
```typescript
import {
createMemoryAugmentation,
createConduitAugmentation,
createSenseAugmentation,
addWebSocketSupport,
executeStreamlined,
processStaticData,
processStreamingData,
createPipeline
createMemoryAugmentation,
createConduitAugmentation,
createSenseAugmentation,
addWebSocketSupport,
executeStreamlined,
processStaticData,
processStreamingData,
createPipeline
} from '@soulcraft/brainy'
// Create a memory augmentation with minimal code
const memoryAug = createMemoryAugmentation({
name: 'simple-memory',
description: 'A simple in-memory storage augmentation',
autoRegister: true,
autoInitialize: true,
name: 'simple-memory',
description: 'A simple in-memory storage augmentation',
autoRegister: true,
autoInitialize: true,
// Implement only the methods you need
storeData: async (key, data) => {
// Your implementation here
return {
success: true,
data: true
}
},
// Implement only the methods you need
storeData: async (key, data) => {
// Your implementation here
return {
success: true,
data: true
}
},
retrieveData: async (key) => {
// Your implementation here
return {
success: true,
data: { example: 'data', key }
retrieveData: async (key) => {
// Your implementation here
return {
success: true,
data: {example: 'data', key}
}
}
}
})
// Add WebSocket support to any augmentation
const wsAugmentation = addWebSocketSupport(memoryAug, {
connectWebSocket: async (url) => {
// Your implementation here
return {
connectionId: 'ws-1',
url,
status: 'connected'
connectWebSocket: async (url) => {
// Your implementation here
return {
connectionId: 'ws-1',
url,
status: 'connected'
}
}
}
})
// Process static data through a pipeline
const result = await processStaticData(
'Input data',
[
{
augmentation: senseAug,
method: 'processRawData',
transformArgs: (data) => [data, 'text']
},
{
augmentation: memoryAug,
method: 'storeData',
transformArgs: (data) => ['processed-data', data]
}
]
'Input data',
[
{
augmentation: senseAug,
method: 'processRawData',
transformArgs: (data) => [data, 'text']
},
{
augmentation: memoryAug,
method: 'storeData',
transformArgs: (data) => ['processed-data', data]
}
]
)
// Create a reusable pipeline
const pipeline = createPipeline([
{
augmentation: senseAug,
method: 'processRawData',
transformArgs: (data) => [data, 'text']
},
{
augmentation: memoryAug,
method: 'storeData',
transformArgs: (data) => ['processed-data', data]
}
{
augmentation: senseAug,
method: 'processRawData',
transformArgs: (data) => [data, 'text']
},
{
augmentation: memoryAug,
method: 'storeData',
transformArgs: (data) => ['processed-data', data]
}
])
// Use the pipeline
@ -906,11 +923,11 @@ const result = await pipeline('New input data')
// Dynamically load augmentations at runtime
const loadedAugmentations = await loadAugmentationModule(
import('./my-augmentations.js'),
{
autoRegister: true,
autoInitialize: true
}
import('./my-augmentations.js'),
{
autoRegister: true,
autoInitialize: true
}
)
```
@ -924,60 +941,61 @@ The simplified augmentation system provides:
#### WebSocket Augmentation Types
Brainy exports several WebSocket augmentation types that can be used by augmentation creators to add WebSocket capabilities to their augmentations:
Brainy exports several WebSocket augmentation types that can be used by augmentation creators to add WebSocket
capabilities to their augmentations:
```typescript
import {
// Base WebSocket support interface
IWebSocketSupport,
// Combined WebSocket augmentation types
IWebSocketSenseAugmentation,
IWebSocketConduitAugmentation,
IWebSocketCognitionAugmentation,
IWebSocketMemoryAugmentation,
IWebSocketPerceptionAugmentation,
IWebSocketDialogAugmentation,
IWebSocketActivationAugmentation,
// Function to add WebSocket support to any augmentation
addWebSocketSupport
// Base WebSocket support interface
IWebSocketSupport,
// Combined WebSocket augmentation types
IWebSocketSenseAugmentation,
IWebSocketConduitAugmentation,
IWebSocketCognitionAugmentation,
IWebSocketMemoryAugmentation,
IWebSocketPerceptionAugmentation,
IWebSocketDialogAugmentation,
IWebSocketActivationAugmentation,
// Function to add WebSocket support to any augmentation
addWebSocketSupport
} from '@soulcraft/brainy'
// Example: Creating a typed WebSocket-enabled sense augmentation
const mySenseAug = createSenseAugmentation({
name: 'my-sense',
processRawData: async (data, dataType) => {
// Implementation
return {
success: true,
data: { nouns: [], verbs: [] }
name: 'my-sense',
processRawData: async (data, dataType) => {
// Implementation
return {
success: true,
data: {nouns: [], verbs: []}
}
}
}
}) as IWebSocketSenseAugmentation
// Add WebSocket support
addWebSocketSupport(mySenseAug, {
connectWebSocket: async (url) => {
// WebSocket implementation
return {
connectionId: 'ws-1',
url,
status: 'connected'
connectWebSocket: async (url) => {
// WebSocket implementation
return {
connectionId: 'ws-1',
url,
status: 'connected'
}
},
sendWebSocketMessage: async (connectionId, data) => {
// Send message implementation
},
onWebSocketMessage: async (connectionId, callback) => {
// Register callback implementation
},
offWebSocketMessage: async (connectionId, callback) => {
// Remove callback implementation
},
closeWebSocket: async (connectionId, code, reason) => {
// Close connection implementation
}
},
sendWebSocketMessage: async (connectionId, data) => {
// Send message implementation
},
onWebSocketMessage: async (connectionId, callback) => {
// Register callback implementation
},
offWebSocketMessage: async (connectionId, callback) => {
// Remove callback implementation
},
closeWebSocket: async (connectionId, code, reason) => {
// Close connection implementation
}
})
// Now mySenseAug has both sense augmentation methods and WebSocket methods
@ -985,7 +1003,8 @@ await mySenseAug.processRawData('data', 'text')
await mySenseAug.connectWebSocket('wss://example.com')
```
These WebSocket augmentation types combine the base augmentation interfaces with the `IWebSocketSupport` interface, providing type safety and autocompletion for augmentations with WebSocket capabilities.
These WebSocket augmentation types combine the base augmentation interfaces with the `IWebSocketSupport` interface,
providing type safety and autocompletion for augmentations with WebSocket capabilities.
### Model Control Protocol (MCP)
@ -1014,13 +1033,13 @@ everywhere.
Brainy automatically detects the environment it's running in:
```typescript
import { environment } from '@soulcraft/brainy'
import {environment} from '@soulcraft/brainy'
// Check which environment we're running in
console.log(`Running in ${
environment.isBrowser ? 'browser' :
environment.isNode ? 'Node.js' :
'serverless/unknown'
environment.isBrowser ? 'browser' :
environment.isNode ? 'Node.js' :
'serverless/unknown'
} environment`)
```
@ -1049,7 +1068,6 @@ Works in all modern browsers:
For browsers without OPFS support, falls back to in-memory storage.
## Related Projects
- **[Cartographer](https://github.com/sodal-project/cartographer)** - A companion project that provides standardized
@ -1094,9 +1112,9 @@ You can use the conduit augmentations to sync Brainy instances:
```typescript
import {
BrainyData,
pipeline,
createConduitAugmentation
BrainyData,
pipeline,
createConduitAugmentation
} from '@soulcraft/brainy'
// Create and initialize the database
@ -1112,36 +1130,36 @@ pipeline.register(wsConduit)
// Connect to another Brainy instance (server or browser)
// Replace the example URL below with your actual WebSocket server URL
const connectionResult = await pipeline.executeConduitPipeline(
'establishConnection',
['wss://example-websocket-server.com/brainy-sync', { protocols: 'brainy-sync' }]
'establishConnection',
['wss://example-websocket-server.com/brainy-sync', {protocols: 'brainy-sync'}]
)
if (connectionResult[0] && (await connectionResult[0]).success) {
const connection = (await connectionResult[0]).data
const connection = (await connectionResult[0]).data
// Read data from the remote instance
const readResult = await pipeline.executeConduitPipeline(
'readData',
[{ connectionId: connection.connectionId, query: { type: 'getAllNouns' } }]
)
// Read data from the remote instance
const readResult = await pipeline.executeConduitPipeline(
'readData',
[{connectionId: connection.connectionId, query: {type: 'getAllNouns'}}]
)
// Process and add the received data to the local instance
if (readResult[0] && (await readResult[0]).success) {
const remoteNouns = (await readResult[0]).data
for (const noun of remoteNouns) {
await db.add(noun.vector, noun.metadata)
// Process and add the received data to the local instance
if (readResult[0] && (await readResult[0]).success) {
const remoteNouns = (await readResult[0]).data
for (const noun of remoteNouns) {
await db.add(noun.vector, noun.metadata)
}
}
}
// Set up real-time sync by monitoring the stream
await wsConduit.monitorStream(connection.connectionId, async (data) => {
// Handle incoming data (e.g., new nouns, verbs, updates)
if (data.type === 'newNoun') {
await db.add(data.vector, data.metadata)
} else if (data.type === 'newVerb') {
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
}
})
// Set up real-time sync by monitoring the stream
await wsConduit.monitorStream(connection.connectionId, async (data) => {
// Handle incoming data (e.g., new nouns, verbs, updates)
if (data.type === 'newNoun') {
await db.add(data.vector, data.metadata)
} else if (data.type === 'newVerb') {
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
}
})
}
```
@ -1149,9 +1167,9 @@ if (connectionResult[0] && (await connectionResult[0]).success) {
```typescript
import {
BrainyData,
pipeline,
createConduitAugmentation
BrainyData,
pipeline,
createConduitAugmentation
} from '@soulcraft/brainy'
// Create and initialize the database
@ -1167,48 +1185,48 @@ pipeline.register(webrtcConduit)
// Connect to a peer using a signaling server
// Replace the example values below with your actual configuration
const connectionResult = await pipeline.executeConduitPipeline(
'establishConnection',
[
'peer-id-to-connect-to', // Replace with actual peer ID
{
signalServerUrl: 'wss://example-signal-server.com', // Replace with your signal server
localPeerId: 'my-local-peer-id', // Replace with your local peer ID
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] // Public STUN server
}
]
'establishConnection',
[
'peer-id-to-connect-to', // Replace with actual peer ID
{
signalServerUrl: 'wss://example-signal-server.com', // Replace with your signal server
localPeerId: 'my-local-peer-id', // Replace with your local peer ID
iceServers: [{urls: 'stun:stun.l.google.com:19302'}] // Public STUN server
}
]
)
if (connectionResult[0] && (await connectionResult[0]).success) {
const connection = (await connectionResult[0]).data
const connection = (await connectionResult[0]).data
// Set up real-time sync by monitoring the stream
await webrtcConduit.monitorStream(connection.connectionId, async (data) => {
// Handle incoming data (e.g., new nouns, verbs, updates)
if (data.type === 'newNoun') {
await db.add(data.vector, data.metadata)
} else if (data.type === 'newVerb') {
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
}
})
// When adding new data locally, also send to the peer
const nounId = await db.add("New data to sync", { noun: "Thing" })
// Send the new noun to the peer
await pipeline.executeConduitPipeline(
'writeData',
[
{
connectionId: connection.connectionId,
data: {
type: 'newNoun',
id: nounId,
vector: (await db.get(nounId)).vector,
metadata: (await db.get(nounId)).metadata
// Set up real-time sync by monitoring the stream
await webrtcConduit.monitorStream(connection.connectionId, async (data) => {
// Handle incoming data (e.g., new nouns, verbs, updates)
if (data.type === 'newNoun') {
await db.add(data.vector, data.metadata)
} else if (data.type === 'newVerb') {
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
}
}
]
)
})
// When adding new data locally, also send to the peer
const nounId = await db.add("New data to sync", {noun: "Thing"})
// Send the new noun to the peer
await pipeline.executeConduitPipeline(
'writeData',
[
{
connectionId: connection.connectionId,
data: {
type: 'newNoun',
id: nounId,
vector: (await db.get(nounId)).vector,
metadata: (await db.get(nounId)).metadata
}
}
]
)
}
```
@ -1218,39 +1236,39 @@ Brainy supports searching a server-hosted instance from a browser, storing resul
searches against the local instance:
```typescript
import { BrainyData } from '@soulcraft/brainy'
import {BrainyData} from '@soulcraft/brainy'
// Create and initialize the database with remote server configuration
// Replace the example URL below with your actual Brainy server URL
const db = new BrainyData({
remoteServer: {
url: 'wss://example-brainy-server.com/ws', // Replace with your server URL
protocols: 'brainy-sync',
autoConnect: true // Connect automatically during initialization
}
remoteServer: {
url: 'wss://example-brainy-server.com/ws', // Replace with your server URL
protocols: 'brainy-sync',
autoConnect: true // Connect automatically during initialization
}
})
await db.init()
// Or connect manually after initialization
if (!db.isConnectedToRemoteServer()) {
// Replace the example URL below with your actual Brainy server URL
await db.connectToRemoteServer('wss://example-brainy-server.com/ws', 'brainy-sync')
// Replace the example URL below with your actual Brainy server URL
await db.connectToRemoteServer('wss://example-brainy-server.com/ws', 'brainy-sync')
}
// Search the remote server (results are stored locally)
const remoteResults = await db.searchText('machine learning', 5, { searchMode: 'remote' })
const remoteResults = await db.searchText('machine learning', 5, {searchMode: 'remote'})
// Search the local database (includes previously stored results)
const localResults = await db.searchText('machine learning', 5, { searchMode: 'local' })
const localResults = await db.searchText('machine learning', 5, {searchMode: 'local'})
// Perform a combined search (local first, then remote if needed)
const combinedResults = await db.searchText('neural networks', 5, { searchMode: 'combined' })
const combinedResults = await db.searchText('neural networks', 5, {searchMode: 'combined'})
// Add data to both local and remote instances
const id = await db.addToBoth('Deep learning is a subset of machine learning', {
noun: 'Concept',
category: 'AI',
tags: ['deep learning', 'neural networks']
noun: 'Concept',
category: 'AI',
tags: ['deep learning', 'neural networks']
})
// Clean up when done (this also cleans up worker pools)

13
cli-package/package-lock.json generated
View file

@ -1,16 +1,16 @@
{
"name": "@soulcraft/brainy-cli",
"version": "0.17.0",
"version": "0.18.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@soulcraft/brainy-cli",
"version": "0.17.0",
"version": "0.18.0",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
"@soulcraft/brainy": "0.12.0",
"@soulcraft/brainy": "^0.17.0",
"commander": "^14.0.0",
"omelette": "^0.4.17"
},
@ -2087,10 +2087,9 @@
}
},
"node_modules/@soulcraft/brainy": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.12.0.tgz",
"integrity": "sha512-Wka8OAw0TQTYZqSSFD+t8cRDOr5l6OblPuZspW54theHn9HwohUtpoHzqwcvYfs5qCF6u0zC7YHhBQtlbrdkIg==",
"hasInstallScript": true,
"version": "0.17.0",
"resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.17.0.tgz",
"integrity": "sha512-QoTiNnyK7fBuNRDsGOcP89UrFfwxoqkS6lRh6VpIXAWVTtxUBy256PWxQAqtZgpslOZRAA58GlXIQf3IYHiM5g==",
"license": "MIT",
"dependencies": {
"@aws-sdk/client-s3": "^3.540.0",

4
cli-package/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@soulcraft/brainy-cli",
"version": "0.17.0",
"version": "0.18.0",
"description": "Command-line interface for the Brainy vector graph database",
"type": "module",
"bin": {
@ -46,7 +46,7 @@
"url": "git+https://github.com/soulcraft-research/brainy.git"
},
"dependencies": {
"@soulcraft/brainy": "^0.15.0",
"@soulcraft/brainy": "^0.17.0",
"commander": "^14.0.0",
"omelette": "^0.4.17"
},

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "@soulcraft/brainy",
"version": "0.16.0",
"version": "0.17.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@soulcraft/brainy",
"version": "0.16.0",
"version": "0.17.0",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "@soulcraft/brainy",
"version": "0.16.0",
"version": "0.17.0",
"description": "A vector graph database using HNSW indexing with Origin Private File System storage",
"main": "dist/unified.js",
"module": "dist/unified.js",

79
src/brainyData.ts
View file

@ -335,13 +335,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Ensure s3Storage has all required fields if it's provided
if (storageOptions.s3Storage) {
// Only include s3Storage if all required fields are present
if (storageOptions.s3Storage.bucketName &&
storageOptions.s3Storage.accessKeyId &&
if (storageOptions.s3Storage.bucketName &&
storageOptions.s3Storage.accessKeyId &&
storageOptions.s3Storage.secretAccessKey) {
// All required fields are present, keep s3Storage as is
} else {
// Missing required fields, remove s3Storage to avoid type errors
const { s3Storage, ...rest } = storageOptions
const {s3Storage, ...rest} = storageOptions
storageOptions = rest
console.warn('Ignoring s3Storage configuration due to missing required fields')
}
@ -1253,6 +1253,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
/**
* Add a verb between two nouns
* If metadata is provided and vector is not, the metadata will be vectorized using the embedding function
*
* @param sourceId ID of the source noun
* @param targetId ID of the target noun
* @param vector Optional vector for the verb
* @param options Additional options:
* - type: Type of the verb
* - weight: Weight of the verb
* - metadata: Metadata for the verb
* - forceEmbed: Force using the embedding function for metadata even if vector is provided
* - id: Optional ID to use instead of generating a new one
* - autoCreateMissingNouns: Automatically create missing nouns if they don't exist
* - missingNounMetadata: Metadata to use when auto-creating missing nouns
*
* @returns The ID of the added verb
*
* @throws Error if source or target nouns don't exist and autoCreateMissingNouns is false or auto-creation fails
*/
public async addVerb(
sourceId: string,
@ -1264,6 +1280,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
metadata?: any
forceEmbed?: boolean // Force using the embedding function for metadata even if vector is provided
id?: string // Optional ID to use instead of generating a new one
autoCreateMissingNouns?: boolean // Automatically create missing nouns
missingNounMetadata?: any // Metadata to use when auto-creating missing nouns
} = {}
): Promise<string> {
await this.ensureInitialized()
@ -1273,8 +1291,59 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try {
// Check if source and target nouns exist
const sourceNoun = this.index.getNouns().get(sourceId)
const targetNoun = this.index.getNouns().get(targetId)
let sourceNoun = this.index.getNouns().get(sourceId)
let targetNoun = this.index.getNouns().get(targetId)
// Auto-create missing nouns if option is enabled
if (!sourceNoun && options.autoCreateMissingNouns) {
try {
// Create a placeholder vector for the missing noun
const placeholderVector = new Array(this._dimensions).fill(0)
// Add metadata if provided
const metadata = options.missingNounMetadata || {
autoCreated: true,
createdAt: new Date().toISOString(),
noun: NounType.Concept
}
// Add the missing noun
await this.add(placeholderVector, metadata, {id: sourceId})
// Get the newly created noun
sourceNoun = this.index.getNouns().get(sourceId)
console.warn(`Auto-created missing source noun with ID ${sourceId}`)
} catch (createError) {
console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError)
throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`)
}
}
if (!targetNoun && options.autoCreateMissingNouns) {
try {
// Create a placeholder vector for the missing noun
const placeholderVector = new Array(this._dimensions).fill(0)
// Add metadata if provided
const metadata = options.missingNounMetadata || {
autoCreated: true,
createdAt: new Date().toISOString(),
noun: NounType.Concept
}
// Add the missing noun
await this.add(placeholderVector, metadata, {id: targetId})
// Get the newly created noun
targetNoun = this.index.getNouns().get(targetId)
console.warn(`Auto-created missing target noun with ID ${targetId}`)
} catch (createError) {
console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError)
throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`)
}
}
if (!sourceNoun) {
throw new Error(`Source noun with ID ${sourceId} not found`)