soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

Initial commit of Brainy vector database v0.1.0

Browse source
This commit is contained in:
David Snelling 2025-05-23 10:55:20 -07:00
commit 49480e694c
29 changed files with 10871 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

37
.gitignore vendored Normal file
View file

@ -0,0 +1,37 @@
/tmp
/out-tsc
/dist
/node_modules
npm-debug.log*
yarn-debug.log*
yarn-error.log*
/.pnp
.pnp.js
# Coverage directory
/coverage
# Environment files
.env
.env.local
.env.development.local
.env.test.local
.env.production.local
# IDE files
.vscode/*
# WebStorm files to ignore
.idea/workspace.xml
.idea/usage.statistics.xml
.idea/dictionaries
.idea/shelf
# Include other WebStorm project files
# OS files
.DS_Store
Thumbs.db
# Data directories created by FileSystemStorage
/brainy-data
/custom-data

11
.idea/brainy.iml generated Normal file
View file

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/dist" />
<excludeFolder url="file://$MODULE_DIR$/node_modules" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

7
.idea/jsLibraryMappings.xml generated Normal file
View file

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptLibraryMappings">
<includedPredefinedLibrary name="Node.js Core" />
<includedPredefinedLibrary name="ECMAScript 6" />
</component>
</project>

17
.idea/material_theme_project_new.xml generated Normal file
View file

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MaterialThemeProjectNewConfig">
<option name="metadata">
<MTProjectMetadataState>
<option name="migrated" value="true" />
<option name="pristineConfig" value="false" />
<option name="userId" value="-1be7adca:196364ea35f:-7ffe" />
</MTProjectMetadataState>
</option>
<option name="titleBarState">
<MTProjectTitleBarConfigState>
<option name="overrideColor" value="false" />
</MTProjectTitleBarConfigState>
</option>
</component>
</project>

9
.idea/misc.xml generated Normal file
View file

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

8
.idea/modules.xml generated Normal file
View file

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/brainy.iml" filepath="$PROJECT_DIR$/.idea/brainy.iml" />
</modules>
</component>
</project>

8
.idea/typescript-compiler.xml generated Normal file
View file

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="TypeScriptCompiler">
<option name="recompileOnChanges" value="true" />
<option name="typeScriptCompilerParams" value="--sourceMap true" />
<option name="useConfig" value="true" />
</component>
</project>

6
.idea/vcs.xml generated Normal file
View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

21
LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Sodal
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

506
README.md Normal file
View file

@ -0,0 +1,506 @@
# Soulcraft Brainy
A vector database that runs in a browser or Node.js and utilizes Origin Private File System (OPFS) for storage, with HNSW (Hierarchical Navigable Small World) for efficient vector indexing.
## Features
- **Cross-platform**: Works in both browsers and Node.js
- **Persistent storage**: Uses Origin Private File System (OPFS) in browsers, with fallback to in-memory storage
- **Efficient vector search**: Implements HNSW (Hierarchical Navigable Small World) algorithm for fast approximate nearest neighbor search
- **Automatic embedding**: Converts text and other data to vectors using embedding models
- **TensorFlow.js integration**: Uses Universal Sentence Encoder for high-quality text embeddings
- **Metadata support**: Store and retrieve metadata alongside vectors
- **TypeScript support**: Fully typed API with generics for metadata types
- **Multiple distance functions**: Supports cosine, Euclidean, Manhattan, and dot product distance metrics
- **Augmentation system**: Extensible architecture for adding specialized capabilities
- **Graph data model**: Structured representation of entities and relationships
## Installation
```bash
npm install brainy
```
## Usage
### Basic Example
```typescript
import {BrainyData} from 'brainy';
// Create a new vector database
const db = new BrainyData();
await db.init();
// Add vectors with metadata
const catId = await db.add([0.2, 0.3, 0.4, 0.1], {type: 'mammal', name: 'cat'});
const dogId = await db.add([0.3, 0.2, 0.4, 0.2], {type: 'mammal', name: 'dog'});
const fishId = await db.add([0.1, 0.1, 0.8, 0.2], {type: 'fish', name: 'fish'});
// Add text directly - it will be automatically embedded
const lionDescId = await db.add("Lions are large cats with a golden mane", {type: 'mammal', name: 'lion'});
const tigerDescId = await db.add("Tigers are large cats with striped fur", {type: 'mammal', name: 'tiger'});
// Search for similar vectors
const results = await db.search([0.2, 0.3, 0.4, 0.1], 2);
console.log(results);
// [
// { id: 'cat-id', score: 0, vector: [0.2, 0.3, 0.4, 0.1], metadata: { type: 'mammal', name: 'cat' } },
// { id: 'dog-id', score: 0.1, vector: [0.3, 0.2, 0.4, 0.2], metadata: { type: 'mammal', name: 'dog' } }
// ]
// Search with text directly - it will be automatically embedded
const catResults = await db.search("cat", 2);
console.log(catResults);
// Results will include vectors similar to the embedding of "cat"
// Get a vector by ID
const cat = await db.get(catId);
console.log(cat);
// { id: 'cat-id', vector: [0.2, 0.3, 0.4, 0.1], metadata: { type: 'mammal', name: 'cat' } }
// Update metadata
await db.updateMetadata(catId, {type: 'mammal', name: 'cat', color: 'orange'});
// Delete a vector
await db.delete(fishId);
// Clear the database
await db.clear();
```
### Configuration Options
```typescript
import {
BrainyData,
euclideanDistance,
UniversalSentenceEncoder,
createEmbeddingFunction
} from 'brainy';
// Configure the vector database
const db = new BrainyData({
// HNSW index configuration
hnsw: {
M: 16, // Max number of connections per node
efConstruction: 200, // Size of dynamic candidate list during construction
efSearch: 50, // Size of dynamic candidate list during search
ml: 16 // Max level
},
// Distance function to use (default is cosineDistance)
distanceFunction: euclideanDistance,
// Custom embedding function (optional)
// By default, it uses the Universal Sentence Encoder for high-quality text embeddings
// You can use the SimpleEmbedding for a basic character-based embedding:
// embeddingFunction: createEmbeddingFunction(new SimpleEmbedding()),
// Or create your own custom embedding function:
// embeddingFunction: async (data) => {
// // Convert data to a vector
// return [0.1, 0.2, 0.3, 0.4]; // Return a vector
// },
// Custom storage adapter (optional)
// By default, it uses OPFS in browsers, FileSystemStorage in Node.js,
// or falls back to in-memory storage if neither is available
// storageAdapter: myCustomStorageAdapter
// You can also explicitly use the FileSystemStorage with a custom directory:
// import { FileSystemStorage } from 'brainy/storage/fileSystemStorage';
// storageAdapter: new FileSystemStorage('/custom/path')
});
```
## Contributing and Publishing
Soulcraft Brainy is an open source project released under the MIT license. Contributions are welcome!
### Contributing to the Project
To contribute to the project:
1. Fork the repository
2. Create a feature branch
3. Make your changes
4. Run tests to ensure everything works:
```bash
npm test
```
5. Submit a pull request
### Publishing a New Version
To publish a new version to npm:
1. Update the version in package.json following semantic versioning
2. Build the package:
```bash
npm run build
```
3. Publish the package:
```bash
npm publish
```
### Installing the Package
To install the package in your project:
```bash
npm install brainy
```
### Requirements
- Node.js >= 18.0.0
## Augmentation System
Brainy includes a powerful augmentation system that allows extending its capabilities through specialized modules. Each augmentation implements a specific interface and provides additional functionality.
### Base Augmentation Interface
All augmentations implement the `IAugmentation` interface:
```typescript
interface IAugmentation {
readonly name: string; // Unique identifier for the augmentation
readonly description: string; // Human-readable description
initialize(): Promise<void>; // Called when Brainy starts up
shutDown(): Promise<void>; // Called when shutting down
getStatus(): Promise<'active' | 'inactive' | 'error'>; // Current status
}
```
### WebSocket Support
Augmentations can optionally implement WebSocket support:
```typescript
interface IWebSocketSupport {
connectWebSocket(url: string, protocols?: string | string[]): Promise<WebSocketConnection>;
sendWebSocketMessage(connectionId: string, data: unknown): Promise<void>;
onWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>;
closeWebSocket(connectionId: string, code?: number, reason?: string): Promise<void>;
}
```
### Specialized Augmentation Types
Brainy supports several specialized augmentation types:
#### Cognition Augmentations
For reasoning, inference, and logical operations:
```typescript
interface ICognitionAugmentation extends IAugmentation {
reason(query: string, context?: Record<string, unknown>): AugmentationResponse<{
inference: string;
confidence: number;
}>;
infer(dataSubset: Record<string, unknown>): AugmentationResponse<Record<string, unknown>>;
executeLogic(ruleId: string, input: Record<string, unknown>): AugmentationResponse<boolean>;
}
```
#### Sense Augmentations
For processing raw, unstructured data:
```typescript
interface ISenseAugmentation extends IAugmentation {
processRawData(rawData: Buffer | string, dataType: string): AugmentationResponse<{
nouns: string[];
verbs: string[];
}>;
listenToFeed(
feedUrl: string,
callback: DataCallback<{ nouns: string[]; verbs: string[] }>
): Promise<void>;
}
```
#### Perception Augmentations
For interpreting and contextualizing data:
```typescript
interface IPerceptionAugmentation extends IAugmentation {
interpret(
nouns: string[],
verbs: string[],
context?: Record<string, unknown>
): AugmentationResponse<Record<string, unknown>>;
organize(
data: Record<string, unknown>,
criteria?: Record<string, unknown>
): AugmentationResponse<Record<string, unknown>>;
generateVisualization(
data: Record<string, unknown>,
visualizationType: string
): AugmentationResponse<string | Buffer | Record<string, unknown>>;
}
```
#### Activation Augmentations
For triggering actions and generating outputs:
```typescript
interface IActivationAugmentation extends IAugmentation {
triggerAction(
actionName: string,
parameters?: Record<string, unknown>
): AugmentationResponse<unknown>;
generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown>>;
interactExternal(systemId: string, payload: Record<string, unknown>): AugmentationResponse<unknown>;
}
```
#### Dialog Augmentations
For natural language understanding and generation:
```typescript
interface IDialogAugmentation extends IAugmentation {
processUserInput(naturalLanguageQuery: string, sessionId?: string): AugmentationResponse<{
intent: string;
nouns: string[];
verbs: string[];
context: Record<string, unknown>;
}>;
generateResponse(
interpretedInput: Record<string, unknown>,
knowledgeContext: Record<string, unknown>,
sessionId?: string
): AugmentationResponse<string>;
manageContext(sessionId: string, contextUpdate: Record<string, unknown>): Promise<void>;
}
```
#### Conduit Augmentations
For establishing data exchange channels:
```typescript
interface IConduitAugmentation extends IAugmentation {
establishConnection(
targetSystemId: string,
config: Record<string, unknown>
): AugmentationResponse<WebSocketConnection>;
readData(
query: Record<string, unknown>,
options?: Record<string, unknown>
): AugmentationResponse<unknown>;
writeData(
data: Record<string, unknown>,
options?: Record<string, unknown>
): AugmentationResponse<unknown>;
monitorStream(streamId: string, callback: DataCallback<unknown>): Promise<void>;
}
```
## Graph Data Model
Brainy uses a graph-based data model to represent entities and relationships. This model consists of nouns (nodes) and verbs (edges).
### Common Types
#### Timestamp
Used for tracking creation and update times:
```typescript
interface Timestamp {
seconds: number;
nanoseconds: number;
}
```
#### CreatorMetadata
Tracks which augmentation and model created an element:
```typescript
interface CreatorMetadata {
augmentation: string; // Name of the augmentation that created this element
version: string; // Version of the augmentation
model: string; // Model identifier used in creation
modelVersion: string; // Version of the model
}
```
### Graph Elements
#### GraphNoun
Base interface for nodes (entities) in the graph:
```typescript
interface GraphNoun {
id: string; // Unique identifier for the noun
createdBy: CreatorMetadata; // Information about what created this noun
noun: NounType; // Type classification of the noun
createdAt: Timestamp; // When the noun was created
updatedAt: Timestamp; // When the noun was last updated
data?: Record<string, unknown>; // Additional flexible data storage
embedding?: number[]; // Vector representation of the noun
}
```
#### GraphVerb
Base interface for edges (relationships) in the graph:
```typescript
interface GraphVerb {
id: string; // Unique identifier for the verb
source: string; // ID of the source noun
target: string; // ID of the target noun
label?: string; // Optional descriptive label
verb: VerbType; // Type of relationship
createdAt: Timestamp; // When the verb was created
updatedAt: Timestamp; // When the verb was last updated
data?: Record<string, unknown>; // Additional flexible data storage
embedding?: number[]; // Vector representation of the relationship
confidence?: number; // Confidence score (0-1)
weight?: number; // Strength/importance of the relationship
}
```
### Noun Types
Brainy supports the following noun types:
- **Person**: Represents a person entity
- **Place**: Represents a physical location
- **Thing**: Represents a physical or virtual object
- **Event**: Represents an event or occurrence
- **Concept**: Represents an abstract concept or idea
- **Content**: Represents content (text, media, etc.)
### Verb Types
Brainy supports the following verb types:
- **AttributedTo**: Indicates attribution or authorship
- **Controls**: Indicates control or ownership
- **Created**: Indicates creation or authorship
- **Earned**: Indicates achievement or acquisition
- **Owns**: Indicates ownership
## Examples
The repository includes several examples to help you get started:
### Modern UI Demo
A complete web application that demonstrates all the features of Soulcraft Brainy with a modern user interface:
- Initialize the database with different distance functions
- Configure HNSW parameters
- Add sample vectors and custom vectors with metadata
- Search for similar vectors
- Get, update, and delete vectors
- View database size and clear the database
To run the Modern UI Demo:
1. Clone the repository
2. Build the project with `npm run build`
3. Open `examples/demo.html` in a browser
### Node.js Examples
The repository also includes TypeScript examples for Node.js:
- `src/examples/basicUsage.ts`: Demonstrates basic vector operations
- `src/examples/customStorage.ts`: Shows how to use a custom storage adapter
## How It Works
### HNSW Indexing
The Hierarchical Navigable Small World (HNSW) algorithm is used for efficient approximate nearest neighbor search. It creates a multi-layered graph structure that allows for logarithmic-time search complexity.
Key features of the HNSW implementation:
- Hierarchical graph structure for efficient navigation
- Configurable parameters for tuning performance vs. accuracy
- Support for different distance metrics
### Origin Private File System (OPFS) Storage
In browser environments, the database uses the Origin Private File System (OPFS) API for persistent storage. This provides:
- Fast, local storage that persists between sessions
- Isolation from other origins for security
- Efficient file operations
In Node.js environments, the database uses a file system-based storage adapter that stores data in JSON files. This provides:
- Persistent storage between application restarts
- Efficient file operations using Node.js fs module
- Configurable storage location
In environments where neither OPFS nor Node.js file system is available, the database automatically falls back to in-memory storage.
## API Reference
### BrainyData
The main class for interacting with the vector database.
#### Constructor
```typescript
constructor(config?: BrainyDataConfig)
```
#### Methods
- `init(): Promise<void>` - Initialize the database
- `add(vectorOrData: Vector | any, metadata?: T, options?: { forceEmbed?: boolean }): Promise<string>` - Add a vector or data to the database
- `addBatch(items: Array<{ vectorOrData: Vector | any, metadata?: T }>, options?: { forceEmbed?: boolean }): Promise<string[]>` - Add multiple vectors or data items
- `search(queryVectorOrData: Vector | any, k?: number, options?: { forceEmbed?: boolean }): Promise<SearchResult<T>[]>` - Search for similar vectors
- `get(id: string): Promise<VectorDocument<T> | null>` - Get a vector by ID
- `delete(id: string): Promise<boolean>` - Delete a vector
- `updateMetadata(id: string, metadata: T): Promise<boolean>` - Update metadata
- `clear(): Promise<void>` - Clear the database
- `size(): number` - Get the number of vectors in the database
### Distance Functions
- `euclideanDistance(a: Vector, b: Vector): number` - Euclidean (L2) distance
- `cosineDistance(a: Vector, b: Vector): number` - Cosine distance
- `manhattanDistance(a: Vector, b: Vector): number` - Manhattan (L1) distance
- `dotProductDistance(a: Vector, b: Vector): number` - Dot product distance
### Embedding Models
- `SimpleEmbedding` - A simple character-based embedding model for text
- `UniversalSentenceEncoder` - TensorFlow Universal Sentence Encoder for high-quality text embeddings
### Embedding Functions
- `createEmbeddingFunction(model: EmbeddingModel): EmbeddingFunction` - Create an embedding function from an embedding model
- `defaultEmbeddingFunction` - Default embedding function using UniversalSentenceEncoder
## Browser Compatibility
The Soulcraft Brainy database works in all modern browsers that support the Origin Private File System API:
- Chrome 86+
- Edge 86+
- Opera 72+
- Chrome for Android 86+
For browsers without OPFS support, the database will automatically fall back to in-memory storage.
## License
MIT

1083
examples/demo.html Normal file
View file

File diff suppressed because it is too large Load diff

5735
package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

86
package.json Normal file
View file

@ -0,0 +1,86 @@
{
"name": "brainy",
"version": "0.1.0",
"description": "A vector database using HNSW indexing with Origin Private File System storage",
"main": "dist/index.js",
"module": "dist/index.js",
"types": "dist/index.d.ts",
"type": "module",
"sideEffects": false,
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
}
},
"engines": {
"node": ">=18.0.0"
},
"scripts": {
"build": "tsc",
"test": "jest",
"start": "node dist/index.js"
},
"keywords": [
"vector-database",
"hnsw",
"opfs",
"origin-private-file-system",
"embeddings"
],
"author": "David Snelling (david@soulcraft.com)",
"license": "MIT",
"devDependencies": {
"@types/jest": "^29.5.3",
"@types/node": "^20.4.5",
"@types/uuid": "^10.0.0",
"@typescript-eslint/eslint-plugin": "^6.0.0",
"@typescript-eslint/parser": "^6.0.0",
"eslint": "^8.45.0",
"jest": "^29.6.2",
"ts-jest": "^29.1.1",
"typescript": "^5.1.6"
},
"dependencies": {
"@tensorflow-models/universal-sentence-encoder": "^1.3.3",
"@tensorflow/tfjs": "^4.22.0",
"@tensorflow/tfjs-backend-cpu": "^4.22.0",
"@tensorflow/tfjs-core": "^4.22.0",
"@tensorflow/tfjs-layers": "^4.22.0",
"uuid": "^9.0.0"
},
"prettier": {
"arrowParens": "always",
"bracketSameLine": true,
"bracketSpacing": true,
"htmlWhitespaceSensitivity": "css",
"printWidth": 80,
"proseWrap": "preserve",
"semi": false,
"singleQuote": true,
"tabWidth": 2,
"trailingComma": "none",
"useTabs": false
},
"eslintConfig": {
"root": true,
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"parser": "@typescript-eslint/parser",
"plugins": [
"@typescript-eslint"
],
"rules": {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": [
"warn",
{
"args": "after-used",
"argsIgnorePattern": "^_"
}
]
}
}
}

549
src/brainyData.ts Normal file
View file

@ -0,0 +1,549 @@
/**
* BrainyData
* Main class that provides the vector database functionality
*/
import { v4 as uuidv4 } from 'uuid'
import { HNSWIndex } from './hnsw/hnswIndex.js'
import { createStorage } from './storage/opfsStorage.js'
import { DistanceFunction, Edge, EmbeddingFunction, HNSWConfig, SearchResult, StorageAdapter, Vector, VectorDocument } from './coreTypes.js'
import { cosineDistance, defaultEmbeddingFunction, euclideanDistance } from './utils/index.js'
export interface BrainyDataConfig {
/**
* HNSW index configuration
*/
hnsw?: Partial<HNSWConfig>
/**
* Distance function to use for similarity calculations
*/
distanceFunction?: DistanceFunction
/**
* Custom storage adapter (if not provided, will use OPFS or memory storage)
*/
storageAdapter?: StorageAdapter
/**
* Embedding function to convert data to vectors
*/
embeddingFunction?: EmbeddingFunction
}
export class BrainyData<T = any> {
private index: HNSWIndex
private storage: StorageAdapter | null = null
private isInitialized = false
private embeddingFunction: EmbeddingFunction
/**
* Create a new vector database
*/
constructor(config: BrainyDataConfig = {}) {
// Initialize HNSW index
this.index = new HNSWIndex(
config.hnsw,
config.distanceFunction || cosineDistance
)
// Set storage if provided, otherwise it will be initialized in init()
this.storage = config.storageAdapter || null
// Set embedding function if provided, otherwise use default
this.embeddingFunction = config.embeddingFunction || defaultEmbeddingFunction
}
/**
* Initialize the database
* Loads existing data from storage if available
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
try {
// Initialize storage if not provided in constructor
if (!this.storage) {
this.storage = await createStorage()
}
// Initialize storage
await this.storage!.init()
// Load all nodes from storage
const nodes = await this.storage!.getAllNodes()
// Clear the index and add all nodes
this.index.clear()
for (const node of nodes) {
// Add to index
this.index.addItem({
id: node.id,
vector: node.vector
})
}
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize vector database:', error)
throw new Error(`Failed to initialize vector database: ${error}`)
}
}
/**
* Add a vector or data to the database
* If the input is not a vector, it will be converted using the embedding function
* @param vectorOrData Vector or data to add
* @param metadata Optional metadata to associate with the vector
* @param options Additional options
* @returns The ID of the added vector
*/
public async add(
vectorOrData: Vector | any,
metadata?: T,
options: {
forceEmbed?: boolean // Force using the embedding function even if input is a vector
} = {}
): Promise<string> {
await this.ensureInitialized()
try {
let vector: Vector
// Check if input is already a vector
if (
Array.isArray(vectorOrData) &&
vectorOrData.every((item) => typeof item === 'number') &&
!options.forceEmbed
) {
// Input is already a vector
vector = vectorOrData
} else {
// Input needs to be vectorized
try {
vector = await this.embeddingFunction(vectorOrData)
} catch (embedError) {
throw new Error(`Failed to vectorize data: ${embedError}`)
}
}
// Check if vector is defined
if (!vector) {
throw new Error('Vector is undefined or null')
}
// Generate ID if isn't provided
const id = uuidv4()
// Add to index
this.index.addItem({ id, vector })
// Get the node from the index
const node = this.index.getNodes().get(id)
if (!node) {
throw new Error(`Failed to retrieve newly created node with ID ${id}`)
}
// Save node to storage
await this.storage!.saveNode(node)
// Save metadata if provided
if (metadata !== undefined) {
await this.storage!.saveMetadata(id, metadata)
}
return id
} catch (error) {
console.error('Failed to add vector:', error)
throw new Error(`Failed to add vector: ${error}`)
}
}
/**
* Add multiple vectors or data items to the database
* @param items Array of items to add
* @param options Additional options
* @returns Array of IDs for the added items
*/
public async addBatch(
items: Array<{
vectorOrData: Vector | any;
metadata?: T
}>,
options: {
forceEmbed?: boolean // Force using the embedding function even if input is a vector
} = {}
): Promise<string[]> {
await this.ensureInitialized()
const ids: string[] = []
try {
for (const item of items) {
const id = await this.add(item.vectorOrData, item.metadata, options)
ids.push(id)
}
return ids
} catch (error) {
console.error('Failed to add batch of items:', error)
throw new Error(`Failed to add batch of items: ${error}`)
}
}
/**
* Search for similar vectors
* @param queryVectorOrData Query vector or data to search for
* @param k Number of results to return
* @param options Additional options
* @returns Array of search results
*/
public async search(
queryVectorOrData: Vector | any,
k: number = 10,
options: {
forceEmbed?: boolean // Force using the embedding function even if input is a vector
} = {}
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
try {
let queryVector: Vector
// Check if input is already a vector
if (
Array.isArray(queryVectorOrData) &&
queryVectorOrData.every((item) => typeof item === 'number') &&
!options.forceEmbed
) {
// Input is already a vector
queryVector = queryVectorOrData
} else {
// Input needs to be vectorized
try {
queryVector = await this.embeddingFunction(queryVectorOrData)
} catch (embedError) {
throw new Error(`Failed to vectorize query data: ${embedError}`)
}
}
// Check if query vector is defined
if (!queryVector) {
throw new Error('Query vector is undefined or null')
}
// Search in the index
const results = this.index.search(queryVector, k)
// Get metadata for each result
const searchResults: SearchResult<T>[] = []
for (const [id, score] of results) {
const node = this.index.getNodes().get(id)
if (!node) {
continue
}
const metadata = await this.storage!.getMetadata(id)
searchResults.push({
id,
score,
vector: node.vector,
metadata
})
}
return searchResults
} catch (error) {
console.error('Failed to search vectors:', error)
throw new Error(`Failed to search vectors: ${error}`)
}
}
/**
* Get a vector by ID
*/
public async get(id: string): Promise<VectorDocument<T> | null> {
await this.ensureInitialized()
try {
// Get node from index
const node = this.index.getNodes().get(id)
if (!node) {
return null
}
// Get metadata
const metadata = await this.storage!.getMetadata(id)
return {
id,
vector: node.vector,
metadata
}
} catch (error) {
console.error(`Failed to get vector ${id}:`, error)
throw new Error(`Failed to get vector ${id}: ${error}`)
}
}
/**
* Delete a vector by ID
*/
public async delete(id: string): Promise<boolean> {
await this.ensureInitialized()
try {
// Remove from index
const removed = this.index.removeItem(id)
if (!removed) {
return false
}
// Remove from storage
await this.storage!.deleteNode(id)
// Try to remove metadata (ignore errors)
try {
await this.storage!.saveMetadata(id, null)
} catch (error) {
// Ignore
}
return true
} catch (error) {
console.error(`Failed to delete vector ${id}:`, error)
throw new Error(`Failed to delete vector ${id}: ${error}`)
}
}
/**
* Update metadata for a vector
*/
public async updateMetadata(id: string, metadata: T): Promise<boolean> {
await this.ensureInitialized()
try {
// Check if a vector exists
const node = this.index.getNodes().get(id)
if (!node) {
return false
}
// Update metadata
await this.storage!.saveMetadata(id, metadata)
return true
} catch (error) {
console.error(`Failed to update metadata for vector ${id}:`, error)
throw new Error(`Failed to update metadata for vector ${id}: ${error}`)
}
}
/**
* Add an edge between two nodes
*/
public async addEdge(
sourceId: string,
targetId: string,
vector?: Vector,
options: {
type?: string
weight?: number
metadata?: any
} = {}
): Promise<string> {
await this.ensureInitialized()
try {
// Check if source and target nodes exist
const sourceNode = this.index.getNodes().get(sourceId)
const targetNode = this.index.getNodes().get(targetId)
if (!sourceNode) {
throw new Error(`Source node with ID ${sourceId} not found`)
}
if (!targetNode) {
throw new Error(`Target node with ID ${targetId} not found`)
}
// Generate ID for the edge
const id = uuidv4()
// Use a provided vector or average of source and target vectors
const edgeVector =
vector ||
sourceNode.vector.map((val, i) => (val + targetNode.vector[i]) / 2)
// Create edge
const edge: Edge = {
id,
vector: edgeVector,
connections: new Map(),
sourceId,
targetId,
type: options.type,
weight: options.weight,
metadata: options.metadata
}
// Add to index
this.index.addItem({ id, vector: edgeVector })
// Get the node from the index
const indexNode = this.index.getNodes().get(id)
if (!indexNode) {
throw new Error(
`Failed to retrieve newly created edge node with ID ${id}`
)
}
// Update edge connections from index
edge.connections = indexNode.connections
// Save edge to storage
await this.storage!.saveEdge(edge)
return id
} catch (error) {
console.error('Failed to add edge:', error)
throw new Error(`Failed to add edge: ${error}`)
}
}
/**
* Get an edge by ID
*/
public async getEdge(id: string): Promise<Edge | null> {
await this.ensureInitialized()
try {
return await this.storage!.getEdge(id)
} catch (error) {
console.error(`Failed to get edge ${id}:`, error)
throw new Error(`Failed to get edge ${id}: ${error}`)
}
}
/**
* Get all edges
*/
public async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
try {
return await this.storage!.getAllEdges()
} catch (error) {
console.error('Failed to get all edges:', error)
throw new Error(`Failed to get all edges: ${error}`)
}
}
/**
* Get edges by source node ID
*/
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
return await this.storage!.getEdgesBySource(sourceId)
} catch (error) {
console.error(`Failed to get edges by source ${sourceId}:`, error)
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
}
}
/**
* Get edges by target node ID
*/
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
return await this.storage!.getEdgesByTarget(targetId)
} catch (error) {
console.error(`Failed to get edges by target ${targetId}:`, error)
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
}
}
/**
* Get edges by type
*/
public async getEdgesByType(type: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
return await this.storage!.getEdgesByType(type)
} catch (error) {
console.error(`Failed to get edges by type ${type}:`, error)
throw new Error(`Failed to get edges by type ${type}: ${error}`)
}
}
/**
* Delete an edge
*/
public async deleteEdge(id: string): Promise<boolean> {
await this.ensureInitialized()
try {
// Remove from index
const removed = this.index.removeItem(id)
if (!removed) {
return false
}
// Remove from storage
await this.storage!.deleteEdge(id)
return true
} catch (error) {
console.error(`Failed to delete edge ${id}:`, error)
throw new Error(`Failed to delete edge ${id}: ${error}`)
}
}
/**
* Clear the database
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
try {
// Clear index
this.index.clear()
// Clear storage
await this.storage!.clear()
} catch (error) {
console.error('Failed to clear vector database:', error)
throw new Error(`Failed to clear vector database: ${error}`)
}
}
/**
* Get the number of vectors in the database
*/
public size(): number {
return this.index.size()
}
/**
* Ensure the database is initialized
*/
private async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.init()
}
}
}
// Export distance functions for convenience
export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js'

123
src/coreTypes.ts Normal file
View file

@ -0,0 +1,123 @@
/**
* Type definitions for the Soulcraft Brainy
*/
/**
* Vector representation - an array of numbers
*/
export type Vector = number[];
/**
* A document with a vector embedding and optional metadata
*/
export interface VectorDocument<T = any> {
id: string;
vector: Vector;
metadata?: T;
}
/**
* Search result with similarity score
*/
export interface SearchResult<T = any> {
id: string;
score: number;
vector: Vector;
metadata?: T;
}
/**
* Distance function for comparing vectors
*/
export type DistanceFunction = (a: Vector, b: Vector) => number;
/**
* Embedding function for converting data to vectors
*/
export type EmbeddingFunction = (data: any) => Promise<Vector>;
/**
* Embedding model interface
*/
export interface EmbeddingModel {
/**
* Initialize the embedding model
*/
init(): Promise<void>;
/**
* Embed data into a vector
*/
embed(data: any): Promise<Vector>;
/**
* Dispose of the model resources
*/
dispose(): Promise<void>;
}
/**
* HNSW graph node
*/
export interface HNSWNode {
id: string;
vector: Vector;
connections: Map<number, Set<string>>; // level -> set of connected node ids
}
/**
* Edge representing a relationship between nodes
* Extends HNSWNode to allow edges to be first-class entities in the data model
*/
export interface Edge extends HNSWNode {
sourceId: string; // ID of the source node
targetId: string; // ID of the target node
type?: string; // Optional type of the relationship
weight?: number; // Optional weight of the relationship
metadata?: any; // Optional metadata for the edge
}
/**
* HNSW index configuration
*/
export interface HNSWConfig {
M: number; // Maximum number of connections per node
efConstruction: number; // Size of the dynamic candidate list during construction
efSearch: number; // Size of the dynamic candidate list during search
ml: number; // Maximum level
}
/**
* Storage interface for persistence
*/
export interface StorageAdapter {
init(): Promise<void>;
saveNode(node: HNSWNode): Promise<void>;
getNode(id: string): Promise<HNSWNode | null>;
getAllNodes(): Promise<HNSWNode[]>;
deleteNode(id: string): Promise<void>;
saveEdge(edge: Edge): Promise<void>;
getEdge(id: string): Promise<Edge | null>;
getAllEdges(): Promise<Edge[]>;
getEdgesBySource(sourceId: string): Promise<Edge[]>;
getEdgesByTarget(targetId: string): Promise<Edge[]>;
getEdgesByType(type: string): Promise<Edge[]>;
deleteEdge(id: string): Promise<void>;
saveMetadata(id: string, metadata: any): Promise<void>;
getMetadata(id: string): Promise<any | null>;
clear(): Promise<void>;
}

163
src/examples/basicUsage.ts Normal file
View file

@ -0,0 +1,163 @@
/**
* Basic usage example for the Soulcraft Brainy database
*/
import { BrainyData } from '../brainyData.js'
// Example data - word embeddings
const wordEmbeddings = {
cat: [0.2, 0.3, 0.4, 0.1],
dog: [0.3, 0.2, 0.4, 0.2],
fish: [0.1, 0.1, 0.8, 0.2],
bird: [0.1, 0.4, 0.2, 0.5],
tiger: [0.3, 0.4, 0.3, 0.1],
lion: [0.4, 0.3, 0.2, 0.1],
shark: [0.2, 0.1, 0.7, 0.3],
eagle: [0.2, 0.5, 0.1, 0.4]
}
// Example metadata
const metadata = {
cat: { type: 'mammal', domesticated: true },
dog: { type: 'mammal', domesticated: true },
fish: { type: 'fish', domesticated: false },
bird: { type: 'bird', domesticated: false },
tiger: { type: 'mammal', domesticated: false },
lion: { type: 'mammal', domesticated: false },
shark: { type: 'fish', domesticated: false },
eagle: { type: 'bird', domesticated: false }
}
/**
* Run the example
*/
async function runExample() {
console.log('Initializing vector database...')
// Create a new vector database
const db = new BrainyData()
await db.init()
console.log('Adding vectors to the database...')
// Add vectors to the database
const ids: Record<string, string> = {}
const metadata: Record<string, { type: string; domesticated: boolean }> = {
cat: { type: 'mammal', domesticated: true },
dog: { type: 'mammal', domesticated: true },
fish: { type: 'fish', domesticated: false },
bird: { type: 'bird', domesticated: false },
tiger: { type: 'mammal', domesticated: false },
lion: { type: 'mammal', domesticated: false },
shark: { type: 'fish', domesticated: false },
eagle: { type: 'bird', domesticated: false }
}
for (const [word, vector] of Object.entries(wordEmbeddings)) {
ids[word] = await db.add(vector, metadata[word])
console.log(`Added "${word}" with ID: ${ids[word]}`)
}
console.log('\nDatabase size:', db.size())
// Search for similar vectors
console.log('\nSearching for vectors similar to "cat"...')
const catResults = await db.search(wordEmbeddings['cat'], 3)
console.log('Results:')
for (const result of catResults) {
const word =
Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
console.log(
`- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
result.metadata,
')'
)
}
// Search for similar vectors
console.log('\nSearching for vectors similar to "fish"...')
const fishResults = await db.search(wordEmbeddings['fish'], 3)
console.log('Results:')
for (const result of fishResults) {
const word =
Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
console.log(
`- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
result.metadata,
')'
)
}
// Update metadata
console.log('\nUpdating metadata for "bird"...')
await db.updateMetadata(ids['bird'], {
...metadata['bird'],
notes: 'Can fly'
})
// Get the updated document
const birdDoc = await db.get(ids['bird'])
console.log('Updated bird document:', birdDoc)
// Delete a vector
console.log('\nDeleting "shark"...')
await db.delete(ids['shark'])
console.log('Database size after deletion:', db.size())
// Search again to verify shark is gone
console.log('\nSearching for vectors similar to "fish" after deletion...')
const fishResultsAfterDeletion = await db.search(wordEmbeddings['fish'], 3)
console.log('Results:')
for (const result of fishResultsAfterDeletion) {
const word =
Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
console.log(
`- ${word} (score: ${result.score.toFixed(4)}, metadata:`,
result.metadata,
')'
)
}
console.log('\nExample completed successfully!')
}
// Check if we're in a browser or Node.js environment
if (typeof window !== 'undefined') {
// Browser environment
document.addEventListener('DOMContentLoaded', () => {
const button = document.createElement('button')
button.textContent = 'Run BrainyData Example'
button.addEventListener('click', async () => {
const output = document.createElement('pre')
document.body.appendChild(output)
// Redirect console.log to the output element
const originalLog = console.log
console.log = (...args) => {
originalLog(...args)
output.textContent +=
args
.map((arg) =>
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : arg
)
.join(' ') + '\n'
}
try {
await runExample()
} catch (error) {
console.error('Error running example:', error)
}
// Restore console.log
console.log = originalLog
})
document.body.appendChild(button)
})
} else {
// Node.js environment
runExample().catch((error) => {
console.error('Error running example:', error)
})
}

76
src/examples/customStorage.ts Normal file
View file

@ -0,0 +1,76 @@
/**
* Example demonstrating how to use the FileSystemStorage adapter with a custom directory
*/
import { BrainyData } from '../brainyData.js'
import { FileSystemStorage } from '../storage/fileSystemStorage.js'
import path from 'path'
// Example data - word embeddings
const wordEmbeddings = {
'cat': [0.2, 0.3, 0.4, 0.1],
'dog': [0.3, 0.2, 0.4, 0.2],
'fish': [0.1, 0.1, 0.8, 0.2]
}
// Example metadata
const metadata: {
[key: string]: { type: string; [key]: boolean } | undefined | null
} = {
'cat': { type: 'mammal', domesticated: true },
'dog': { type: 'mammal', domesticated: true },
'fish': { type: 'fish', domesticated: false }
}
/**
* Run the example
*/
async function runExample() {
console.log('Initializing vector database with custom storage location...')
// Create a custom storage adapter with a specific directory
// This will store data in ./custom-data directory relative to the current working directory
const customStoragePath = path.join(process.cwd(), 'custom-data')
const storageAdapter = new FileSystemStorage(customStoragePath)
// Create a new vector database with the custom storage adapter
const db = new BrainyData({
storageAdapter
})
await db.init()
console.log(`Using custom storage location: ${customStoragePath}`)
console.log('Adding vectors to the database...')
// Add vectors to the database
const ids: Record<string, string> = {}
for (const [word, vector] of Object.entries(wordEmbeddings)) {
ids[word] = await db.add(vector, metadata[word])
console.log(`Added "${word}" with ID: ${ids[word]}`)
}
console.log('\nDatabase size:', db.size())
// Search for similar vectors
console.log('\nSearching for vectors similar to "cat"...')
const catResults = await db.search(wordEmbeddings['cat'], 2)
console.log('Results:')
for (const result of catResults) {
const word = Object.entries(ids).find(([_, id]) => id === result.id)?.[0] || 'unknown'
console.log(`- ${word} (score: ${result.score.toFixed(4)}, metadata:`, result.metadata, ')')
}
console.log('\nExample completed successfully!')
console.log(`Data has been stored in: ${customStoragePath}`)
console.log('You can restart this example to verify that data persists between runs.')
}
// Only run in Node.js environment
if (typeof process !== 'undefined' && process.versions && process.versions.node) {
runExample().catch(error => {
console.error('Error running example:', error)
})
} else {
console.error('This example is designed to run in Node.js environments only.')
}

527
src/hnsw/hnswIndex.ts Normal file
View file

@ -0,0 +1,527 @@
/**
* HNSW (Hierarchical Navigable Small World) Index implementation
* Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
*/
import { DistanceFunction, HNSWConfig, HNSWNode, Vector, VectorDocument } from '../coreTypes.js'
import { euclideanDistance } from '../utils/index.js'
// Default HNSW parameters
const DEFAULT_CONFIG: HNSWConfig = {
M: 16, // Max number of connections per node
efConstruction: 200, // Size of a dynamic candidate list during construction
efSearch: 50, // Size of a dynamic candidate list during search
ml: 16 // Max level
}
export class HNSWIndex {
private nodes: Map<string, HNSWNode> = new Map()
private entryPointId: string | null = null
private maxLevel = 0
private config: HNSWConfig
private distanceFunction: DistanceFunction
private dimension: number | null = null
constructor(
config: Partial<HNSWConfig> = {},
distanceFunction: DistanceFunction = euclideanDistance
) {
this.config = { ...DEFAULT_CONFIG, ...config }
this.distanceFunction = distanceFunction
}
/**
* Add a vector to the index
*/
public addItem(item: VectorDocument): string {
// Check if item is defined
if (!item) {
throw new Error('Item is undefined or null')
}
const { id, vector } = item
// Check if vector is defined
if (!vector) {
throw new Error('Vector is undefined or null')
}
// Set dimension on first insert
if (this.dimension === null) {
this.dimension = vector.length
} else if (vector.length !== this.dimension) {
throw new Error(
`Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`
)
}
// Generate random level for this node
const nodeLevel = this.getRandomLevel()
// Create new node
const node: HNSWNode = {
id,
vector,
connections: new Map()
}
// Initialize empty connection sets for each level
for (let level = 0; level <= nodeLevel; level++) {
node.connections.set(level, new Set<string>())
}
// If this is the first node, make it the entry point
if (this.nodes.size === 0) {
this.entryPointId = id
this.maxLevel = nodeLevel
this.nodes.set(id, node)
return id
}
// Find entry point
if (!this.entryPointId) {
console.error('Entry point ID is null')
// If there's no entry point, this is the first node, so we should have returned earlier
// This is a safety check
this.entryPointId = id
this.maxLevel = nodeLevel
this.nodes.set(id, node)
return id
}
const entryPoint = this.nodes.get(this.entryPointId)
if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`)
// If the entry point doesn't exist, treat this as the first node
this.entryPointId = id
this.maxLevel = nodeLevel
this.nodes.set(id, node)
return id
}
let currObj = entryPoint
let currDist = this.distanceFunction(vector, entryPoint.vector)
// Traverse the graph from top to bottom to find the closest node
for (let level = this.maxLevel; level > nodeLevel; level--) {
let changed = true
while (changed) {
changed = false
// Check all neighbors at current level
const connections = currObj.connections.get(level) || new Set<string>()
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in addItem traversal`)
continue
}
const distToNeighbor = this.distanceFunction(vector, neighbor.vector)
if (distToNeighbor < currDist) {
currDist = distToNeighbor
currObj = neighbor
changed = true
}
}
}
}
// For each level from nodeLevel down to 0
for (let level = Math.min(nodeLevel, this.maxLevel); level >= 0; level--) {
// Find ef nearest elements using greedy search
const nearestNodes = this.searchLayer(
vector,
currObj,
this.config.efConstruction,
level
)
// Select M nearest neighbors
const neighbors = this.selectNeighbors(
vector,
nearestNodes,
this.config.M
)
// Add bidirectional connections
for (const [neighborId, _] of neighbors) {
const neighbor = this.nodes.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found`)
continue
}
node.connections.get(level)!.add(neighborId)
// Add reverse connection
if (!neighbor.connections.has(level)) {
neighbor.connections.set(level, new Set<string>())
}
neighbor.connections.get(level)!.add(id)
// Ensure neighbor doesn't have too many connections
if (neighbor.connections.get(level)!.size > this.config.M) {
this.pruneConnections(neighbor, level)
}
}
// Update entry point for the next level
if (nearestNodes.size > 0) {
const [nearestId, nearestDist] = [...nearestNodes][0]
if (nearestDist < currDist) {
currDist = nearestDist
const nearestNode = this.nodes.get(nearestId)
if (!nearestNode) {
console.error(`Nearest node with ID ${nearestId} not found in addItem`)
// Keep the current object as is
} else {
currObj = nearestNode
}
}
}
}
// Update max level and entry point if needed
if (nodeLevel > this.maxLevel) {
this.maxLevel = nodeLevel
this.entryPointId = id
}
// Add node to the index
this.nodes.set(id, node)
return id
}
/**
* Search for nearest neighbors
*/
public search(queryVector: Vector, k: number = 10): Array<[string, number]> {
if (this.nodes.size === 0) {
return []
}
// Check if query vector is defined
if (!queryVector) {
throw new Error('Query vector is undefined or null')
}
if (this.dimension !== null && queryVector.length !== this.dimension) {
throw new Error(
`Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`
)
}
// Start from the entry point
if (!this.entryPointId) {
console.error('Entry point ID is null')
return []
}
const entryPoint = this.nodes.get(this.entryPointId)
if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`)
return []
}
let currObj = entryPoint
let currDist = this.distanceFunction(queryVector, currObj.vector)
// Traverse the graph from top to bottom to find the closest node
for (let level = this.maxLevel; level > 0; level--) {
let changed = true
while (changed) {
changed = false
// Check all neighbors at current level
const connections = currObj.connections.get(level) || new Set<string>()
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in search`)
continue
}
const distToNeighbor = this.distanceFunction(
queryVector,
neighbor.vector
)
if (distToNeighbor < currDist) {
currDist = distToNeighbor
currObj = neighbor
changed = true
}
}
}
}
// Search at level 0 with ef = k
const nearestNodes = this.searchLayer(
queryVector,
currObj,
Math.max(this.config.efSearch, k),
0
)
// Convert to array and sort by distance
return [...nearestNodes].slice(0, k)
}
/**
* Remove an item from the index
*/
public removeItem(id: string): boolean {
if (!this.nodes.has(id)) {
return false
}
const node = this.nodes.get(id)!
// Remove connections to this node from all neighbors
for (const [level, connections] of node.connections.entries()) {
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in removeItem`)
continue
}
if (neighbor.connections.has(level)) {
neighbor.connections.get(level)!.delete(id)
// Prune connections after removing this node to ensure consistency
this.pruneConnections(neighbor, level)
}
}
}
// Also check all other nodes for references to this node and remove them
for (const [nodeId, otherNode] of this.nodes.entries()) {
if (nodeId === id) continue // Skip the node being removed
for (const [level, connections] of otherNode.connections.entries()) {
if (connections.has(id)) {
connections.delete(id)
// Prune connections after removing this reference
this.pruneConnections(otherNode, level)
}
}
}
// Remove the node
this.nodes.delete(id)
// If we removed the entry point, find a new one
if (this.entryPointId === id) {
if (this.nodes.size === 0) {
this.entryPointId = null
this.maxLevel = 0
} else {
// Find the node with the highest level
let maxLevel = 0
let newEntryPointId = null
for (const [nodeId, node] of this.nodes.entries()) {
if (node.connections.size === 0) continue // Skip nodes with no connections
const nodeLevel = Math.max(...node.connections.keys())
if (nodeLevel >= maxLevel) {
maxLevel = nodeLevel
newEntryPointId = nodeId
}
}
this.entryPointId = newEntryPointId
this.maxLevel = maxLevel
}
}
return true
}
/**
* Get all nodes in the index
*/
public getNodes(): Map<string, HNSWNode> {
return new Map(this.nodes)
}
/**
* Clear the index
*/
public clear(): void {
this.nodes.clear()
this.entryPointId = null
this.maxLevel = 0
}
/**
* Get the size of the index
*/
public size(): number {
return this.nodes.size
}
/**
* Search within a specific layer
* Returns a map of node IDs to distances, sorted by distance
*/
private searchLayer(
queryVector: Vector,
entryPoint: HNSWNode,
ef: number,
level: number
): Map<string, number> {
// Set of visited nodes
const visited = new Set<string>([entryPoint.id])
// Priority queue of candidates (closest first)
const candidates = new Map<string, number>()
candidates.set(
entryPoint.id,
this.distanceFunction(queryVector, entryPoint.vector)
)
// Priority queue of nearest neighbors found so far (closest first)
const nearest = new Map<string, number>()
nearest.set(
entryPoint.id,
this.distanceFunction(queryVector, entryPoint.vector)
)
// While there are candidates to explore
while (candidates.size > 0) {
// Get closest candidate
const [closestId, closestDist] = [...candidates][0]
candidates.delete(closestId)
// If this candidate is farther than the farthest in our result set, we're done
const farthestInNearest = [...nearest][nearest.size - 1]
if (nearest.size >= ef && closestDist > farthestInNearest[1]) {
break
}
// Explore neighbors of the closest candidate
const node = this.nodes.get(closestId)
if (!node) {
console.error(`Node with ID ${closestId} not found in searchLayer`)
continue
}
const connections = node.connections.get(level) || new Set<string>()
for (const neighborId of connections) {
if (!visited.has(neighborId)) {
visited.add(neighborId)
const neighbor = this.nodes.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in searchLayer`)
continue
}
const distToNeighbor = this.distanceFunction(
queryVector,
neighbor.vector
)
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
candidates.set(neighborId, distToNeighbor)
nearest.set(neighborId, distToNeighbor)
// If we have more than ef neighbors, remove the farthest one
if (nearest.size > ef) {
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
nearest.clear()
for (let i = 0; i < ef; i++) {
nearest.set(sortedNearest[i][0], sortedNearest[i][1])
}
}
}
}
}
}
// Sort nearest by distance
return new Map([...nearest].sort((a, b) => a[1] - b[1]))
}
/**
* Select M nearest neighbors from the candidate set
*/
private selectNeighbors(
queryVector: Vector,
candidates: Map<string, number>,
M: number
): Map<string, number> {
if (candidates.size <= M) {
return candidates
}
// Simple heuristic: just take the M closest
const sortedCandidates = [...candidates].sort((a, b) => a[1] - b[1])
const result = new Map<string, number>()
for (let i = 0; i < Math.min(M, sortedCandidates.length); i++) {
result.set(sortedCandidates[i][0], sortedCandidates[i][1])
}
return result
}
/**
* Ensure a node doesn't have too many connections at a given level
*/
private pruneConnections(node: HNSWNode, level: number): void {
const connections = node.connections.get(level)!
if (connections.size <= this.config.M) {
return
}
// Calculate distances to all neighbors
const distances = new Map<string, number>()
const validNeighborIds = new Set<string>()
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in pruneConnections`)
continue
}
// Only add valid neighbors to the distances map
distances.set(
neighborId,
this.distanceFunction(node.vector, neighbor.vector)
)
validNeighborIds.add(neighborId)
}
// Only proceed if we have valid neighbors
if (distances.size === 0) {
// If no valid neighbors, clear connections at this level
node.connections.set(level, new Set())
return
}
// Select M closest neighbors from valid ones
const selectedNeighbors = this.selectNeighbors(
node.vector,
distances,
this.config.M
)
// Update connections with only valid neighbors
node.connections.set(level, new Set(selectedNeighbors.keys()))
}
/**
* Generate a random level for a new node
* Uses the same distribution as in the original HNSW paper
*/
private getRandomLevel(): number {
const r = Math.random()
return Math.floor(-Math.log(r) * (1.0 / Math.log(this.config.M)))
}
}

61
src/index.ts Normal file
View file

@ -0,0 +1,61 @@
/**
* OPFS BrainyData
* A vector database using HNSW indexing with Origin Private File System storage
*/
// Export main BrainyData class and related types
import { BrainyData, BrainyDataConfig } from './brainyData.js'
export { BrainyData }
export type { BrainyDataConfig }
// Export distance functions for convenience
import {
euclideanDistance,
cosineDistance,
manhattanDistance,
dotProductDistance
} from './utils/index.js'
export {
euclideanDistance,
cosineDistance,
manhattanDistance,
dotProductDistance
}
// Export storage adapters
import {
OPFSStorage,
MemoryStorage,
createStorage
} from './storage/opfsStorage.js'
export {
OPFSStorage,
MemoryStorage,
createStorage
}
// Export types
import type {
Vector,
VectorDocument,
SearchResult,
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNode,
Edge,
HNSWConfig,
StorageAdapter
} from './coreTypes.js'
export type {
Vector,
VectorDocument,
SearchResult,
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNode,
Edge,
HNSWConfig,
StorageAdapter
}

432
src/storage/fileSystemStorage.ts Normal file
View file

@ -0,0 +1,432 @@
import fs from 'fs'
import path from 'path'
import { Edge, HNSWNode, StorageAdapter } from '../coreTypes.js'
// Constants for directory and file names
const ROOT_DIR = 'brainy-data'
const NODES_DIR = 'nodes'
const EDGES_DIR = 'edges'
const METADATA_DIR = 'metadata'
/**
* File system storage adapter for Node.js environments
*/
export class FileSystemStorage implements StorageAdapter {
private rootDir: string
private nodesDir: string
private edgesDir: string
private metadataDir: string
private isInitialized = false
constructor(rootDirectory?: string) {
// Use the provided root directory or default to the current working directory
this.rootDir = rootDirectory ? path.resolve(rootDirectory, ROOT_DIR) : path.resolve(process.cwd(), ROOT_DIR)
this.nodesDir = path.join(this.rootDir, NODES_DIR)
this.edgesDir = path.join(this.rootDir, EDGES_DIR)
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
}
/**
* Initialize the storage adapter
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
try {
// Create directories if they don't exist
await this.ensureDirectoryExists(this.rootDir)
await this.ensureDirectoryExists(this.nodesDir)
await this.ensureDirectoryExists(this.edgesDir)
await this.ensureDirectoryExists(this.metadataDir)
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize file system storage:', error)
throw new Error(`Failed to initialize file system storage: ${error}`)
}
}
/**
* Save a node to storage
*/
public async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) => Array.from(set))
}
const filePath = path.join(this.nodesDir, `${node.id}.json`)
await fs.promises.writeFile(
filePath,
JSON.stringify(serializableNode, null, 2),
'utf8'
)
} catch (error) {
console.error(`Failed to save node ${node.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`)
}
}
/**
* Get a node from storage
*/
public async getNode(id: string): Promise<HNSWNode | null> {
await this.ensureInitialized()
try {
const filePath = path.join(this.nodesDir, `${id}.json`)
// Check if a file exists
try {
await fs.promises.access(filePath)
} catch {
return null // File doesn't exist
}
const data = await fs.promises.readFile(filePath, 'utf8')
const parsedNode = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch (error) {
console.error(`Failed to get node ${id}:`, error)
return null
}
}
/**
* Get all nodes from storage
*/
public async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
const files = await fs.promises.readdir(this.nodesDir)
const nodePromises = files
.filter(file => file.endsWith('.json'))
.map(file => {
const id = path.basename(file, '.json')
return this.getNode(id)
})
const nodes = await Promise.all(nodePromises)
return nodes.filter((node): node is HNSWNode => node !== null)
} catch (error) {
console.error('Failed to get all nodes:', error)
throw new Error(`Failed to get all nodes: ${error}`)
}
}
/**
* Delete a node from storage
*/
public async deleteNode(id: string): Promise<void> {
await this.ensureInitialized()
try {
const filePath = path.join(this.nodesDir, `${id}.json`)
// Check if a file exists before attempting to delete
try {
await fs.promises.access(filePath)
} catch {
return // File doesn't exist, nothing to delete
}
await fs.promises.unlink(filePath)
} catch (error) {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
}
}
/**
* Save an edge to storage
*/
public async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) => Array.from(set))
}
const filePath = path.join(this.edgesDir, `${edge.id}.json`)
await fs.promises.writeFile(
filePath,
JSON.stringify(serializableEdge, null, 2),
'utf8'
)
} catch (error) {
console.error(`Failed to save edge ${edge.id}:`, error)
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
}
}
/**
* Get an edge from storage
*/
public async getEdge(id: string): Promise<Edge | null> {
await this.ensureInitialized()
try {
const filePath = path.join(this.edgesDir, `${id}.json`)
// Check if a file exists
try {
await fs.promises.access(filePath)
} catch {
return null // File doesn't exist
}
const data = await fs.promises.readFile(filePath, 'utf8')
const parsedEdge = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
connections,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
type: parsedEdge.type,
weight: parsedEdge.weight,
metadata: parsedEdge.metadata
}
} catch (error) {
console.error(`Failed to get edge ${id}:`, error)
return null
}
}
/**
* Get all edges from storage
*/
public async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
try {
const files = await fs.promises.readdir(this.edgesDir)
const edgePromises = files
.filter(file => file.endsWith('.json'))
.map(file => {
const id = path.basename(file, '.json')
return this.getEdge(id)
})
const edges = await Promise.all(edgePromises)
return edges.filter((edge): edge is Edge => edge !== null)
} catch (error) {
console.error('Failed to get all edges:', error)
throw new Error(`Failed to get all edges: ${error}`)
}
}
/**
* Get edges by source node ID
*/
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter(edge => edge.sourceId === sourceId)
} catch (error) {
console.error(`Failed to get edges by source ${sourceId}:`, error)
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
}
}
/**
* Get edges by target node ID
*/
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter(edge => edge.targetId === targetId)
} catch (error) {
console.error(`Failed to get edges by target ${targetId}:`, error)
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
}
}
/**
* Get edges by type
*/
public async getEdgesByType(type: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter(edge => edge.type === type)
} catch (error) {
console.error(`Failed to get edges by type ${type}:`, error)
throw new Error(`Failed to get edges by type ${type}: ${error}`)
}
}
/**
* Delete an edge from storage
*/
public async deleteEdge(id: string): Promise<void> {
await this.ensureInitialized()
try {
const filePath = path.join(this.edgesDir, `${id}.json`)
// Check if a file exists before attempting to delete
try {
await fs.promises.access(filePath)
} catch {
return // File doesn't exist, nothing to delete
}
await fs.promises.unlink(filePath)
} catch (error) {
console.error(`Failed to delete edge ${id}:`, error)
throw new Error(`Failed to delete edge ${id}: ${error}`)
}
}
/**
* Save metadata to storage
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
try {
const filePath = path.join(this.metadataDir, `${id}.json`)
await fs.promises.writeFile(
filePath,
JSON.stringify(metadata, null, 2),
'utf8'
)
} catch (error) {
console.error(`Failed to save metadata for ${id}:`, error)
throw new Error(`Failed to save metadata for ${id}: ${error}`)
}
}
/**
* Get metadata from storage
*/
public async getMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
try {
const filePath = path.join(this.metadataDir, `${id}.json`)
// Check if a file exists
try {
await fs.promises.access(filePath)
} catch {
return null // File doesn't exist
}
const data = await fs.promises.readFile(filePath, 'utf8')
return JSON.parse(data)
} catch (error) {
console.error(`Failed to get metadata for ${id}:`, error)
return null
}
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
try {
// Delete and recreate the nodes, edges, and metadata directories
await this.deleteDirectory(this.nodesDir)
await this.deleteDirectory(this.edgesDir)
await this.deleteDirectory(this.metadataDir)
await this.ensureDirectoryExists(this.nodesDir)
await this.ensureDirectoryExists(this.edgesDir)
await this.ensureDirectoryExists(this.metadataDir)
} catch (error) {
console.error('Failed to clear storage:', error)
throw new Error(`Failed to clear storage: ${error}`)
}
}
/**
* Ensure the storage adapter is initialized
*/
private async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.init()
}
}
/**
* Ensure a directory exists, creating it if necessary
*/
private async ensureDirectoryExists(dirPath: string): Promise<void> {
try {
await fs.promises.access(dirPath)
} catch {
// Directory doesn't exist, create it
await fs.promises.mkdir(dirPath, { recursive: true })
}
}
/**
* Delete a directory and all its contents
*/
private async deleteDirectory(dirPath: string): Promise<void> {
try {
const files = await fs.promises.readdir(dirPath)
for (const file of files) {
const filePath = path.join(dirPath, file)
await fs.promises.unlink(filePath)
}
} catch (error) {
// If the directory doesn't exist, that's fine
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
throw error
}
}
}
/**
* Convert a Map to a plain object for serialization
*/
private mapToObject<K extends string | number, V>(
map: Map<K, V>,
valueTransformer: (value: V) => any = (v) => v
): Record<string, any> {
const obj: Record<string, any> = {}
for (const [key, value] of map.entries()) {
obj[key.toString()] = valueTransformer(value)
}
return obj
}
}

680
src/storage/opfsStorage.ts Normal file
View file

@ -0,0 +1,680 @@
/**
* OPFS (Origin Private File System) Storage Adapter
* Provides persistent storage for the vector database using the Origin Private File System API
*/
import { Edge, HNSWNode, StorageAdapter } from '../coreTypes.js'
// Directory and file names
const ROOT_DIR = 'opfs-vector-db'
const NODES_DIR = 'nodes'
const EDGES_DIR = 'edges'
const METADATA_DIR = 'metadata'
const DB_INFO_FILE = 'db-info.json'
export class OPFSStorage implements StorageAdapter {
private rootDir: FileSystemDirectoryHandle | null = null
private nodesDir: FileSystemDirectoryHandle | null = null
private edgesDir: FileSystemDirectoryHandle | null = null
private metadataDir: FileSystemDirectoryHandle | null = null
private isInitialized = false
private isAvailable = false
constructor() {
// Check if OPFS is available
this.isAvailable =
typeof navigator !== 'undefined' &&
'storage' in navigator &&
'getDirectory' in navigator.storage
}
/**
* Initialize the storage adapter
*/
public async init(): Promise<void> {
if (this.isInitialized) {
return
}
if (!this.isAvailable) {
throw new Error(
'Origin Private File System is not available in this environment'
)
}
try {
// Get the root directory
const root = await navigator.storage.getDirectory()
// Create or get our app's root directory
this.rootDir = await root.getDirectoryHandle(ROOT_DIR, { create: true })
// Create or get nodes directory
this.nodesDir = await this.rootDir.getDirectoryHandle(NODES_DIR, {
create: true
})
// Create or get edges directory
this.edgesDir = await this.rootDir.getDirectoryHandle(EDGES_DIR, {
create: true
})
// Create or get metadata directory
this.metadataDir = await this.rootDir.getDirectoryHandle(METADATA_DIR, {
create: true
})
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize OPFS storage:', error)
throw new Error(`Failed to initialize OPFS storage: ${error}`)
}
}
/**
* Check if OPFS is available in the current environment
*/
public isOPFSAvailable(): boolean {
return this.isAvailable
}
/**
* Save a node to storage
*/
public async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) =>
Array.from(set)
)
}
// Create or get the file for this node
const fileHandle = await this.nodesDir!.getFileHandle(node.id, {
create: true
})
// Write the node data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableNode))
await writable.close()
} catch (error) {
console.error(`Failed to save node ${node.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`)
}
}
/**
* Get a node from storage
*/
public async getNode(id: string): Promise<HNSWNode | null> {
await this.ensureInitialized()
try {
// Get the file handle for this node
const fileHandle = await this.nodesDir!.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch (error) {
// If the file doesn't exist, return null
if ((error as any).name === 'NotFoundError') {
return null
}
console.error(`Failed to get node ${id}:`, error)
throw new Error(`Failed to get node ${id}: ${error}`)
}
}
/**
* Get all nodes from storage
*/
public async getAllNodes(): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
const nodes: HNSWNode[] = []
// Get all keys (filenames) in the nodes directory
// @ts-ignore - TypeScript doesn't recognize FileSystemDirectoryHandle.keys() properly
const keys = this.nodesDir!.keys()
// Iterate through all keys and get the corresponding nodes
for await (const name of keys) {
const node = await this.getNode(name)
if (node) {
nodes.push(node)
}
}
return nodes
} catch (error) {
console.error('Failed to get all nodes:', error)
throw new Error(`Failed to get all nodes: ${error}`)
}
}
/**
* Delete a node from storage
*/
public async deleteNode(id: string): Promise<void> {
await this.ensureInitialized()
try {
await this.nodesDir!.removeEntry(id)
} catch (error) {
// Ignore if the file doesn't exist
if ((error as any).name !== 'NotFoundError') {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
}
}
}
/**
* Save an edge to storage
*/
public async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set)
)
}
// Create or get the file for this edge
const fileHandle = await this.edgesDir!.getFileHandle(edge.id, {
create: true
})
// Write the edge data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableEdge))
await writable.close()
} catch (error) {
console.error(`Failed to save edge ${edge.id}:`, error)
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
}
}
/**
* Get an edge from storage
*/
public async getEdge(id: string): Promise<Edge | null> {
await this.ensureInitialized()
try {
// Get the file handle for this edge
const fileHandle = await this.edgesDir!.getFileHandle(id)
// Read the edge data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections,
sourceId: data.sourceId,
targetId: data.targetId,
type: data.type,
weight: data.weight,
metadata: data.metadata
}
} catch (error) {
// If the file doesn't exist, return null
if ((error as any).name === 'NotFoundError') {
return null
}
console.error(`Failed to get edge ${id}:`, error)
throw new Error(`Failed to get edge ${id}: ${error}`)
}
}
/**
* Get all edges from storage
*/
public async getAllEdges(): Promise<Edge[]> {
await this.ensureInitialized()
try {
const edges: Edge[] = []
// Get all keys (filenames) in the edges directory
// @ts-ignore - TypeScript doesn't recognize FileSystemDirectoryHandle.keys() properly
const keys = this.edgesDir!.keys()
// Iterate through all keys and get the corresponding edges
for await (const name of keys) {
const edge = await this.getEdge(name)
if (edge) {
edges.push(edge)
}
}
return edges
} catch (error) {
console.error('Failed to get all edges:', error)
throw new Error(`Failed to get all edges: ${error}`)
}
}
/**
* Delete an edge from storage
*/
public async deleteEdge(id: string): Promise<void> {
await this.ensureInitialized()
try {
await this.edgesDir!.removeEntry(id)
} catch (error) {
// Ignore if the file doesn't exist
if ((error as any).name !== 'NotFoundError') {
console.error(`Failed to delete edge ${id}:`, error)
throw new Error(`Failed to delete edge ${id}: ${error}`)
}
}
}
/**
* Get edges by source node ID
*/
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter(edge => edge.sourceId === sourceId)
} catch (error) {
console.error(`Failed to get edges by source ${sourceId}:`, error)
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
}
}
/**
* Get edges by target node ID
*/
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter(edge => edge.targetId === targetId)
} catch (error) {
console.error(`Failed to get edges by target ${targetId}:`, error)
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
}
}
/**
* Get edges by type
*/
public async getEdgesByType(type: string): Promise<Edge[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
return allEdges.filter(edge => edge.type === type)
} catch (error) {
console.error(`Failed to get edges by type ${type}:`, error)
throw new Error(`Failed to get edges by type ${type}: ${error}`)
}
}
/**
* Save metadata for a node
*/
public async saveMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
try {
// Create or get the file for this metadata
const fileHandle = await this.metadataDir!.getFileHandle(id, {
create: true
})
// Write the metadata to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(metadata))
await writable.close()
} catch (error) {
console.error(`Failed to save metadata for ${id}:`, error)
throw new Error(`Failed to save metadata for ${id}: ${error}`)
}
}
/**
* Get metadata for a node
*/
public async getMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
try {
// Get the file handle for this metadata
const fileHandle = await this.metadataDir!.getFileHandle(id)
// Read the metadata from the file
const file = await fileHandle.getFile()
const text = await file.text()
return JSON.parse(text)
} catch (error) {
// If the file doesn't exist, return null
if ((error as any).name === 'NotFoundError') {
return null
}
console.error(`Failed to get metadata for ${id}:`, error)
throw new Error(`Failed to get metadata for ${id}: ${error}`)
}
}
/**
* Clear all data from storage
*/
public async clear(): Promise<void> {
await this.ensureInitialized()
try {
// Delete and recreate the nodes directory
await this.rootDir!.removeEntry(NODES_DIR, { recursive: true })
this.nodesDir = await this.rootDir!.getDirectoryHandle(NODES_DIR, {
create: true
})
// Delete and recreate the edges directory
await this.rootDir!.removeEntry(EDGES_DIR, { recursive: true })
this.edgesDir = await this.rootDir!.getDirectoryHandle(EDGES_DIR, {
create: true
})
// Delete and recreate the metadata directory
await this.rootDir!.removeEntry(METADATA_DIR, { recursive: true })
this.metadataDir = await this.rootDir!.getDirectoryHandle(METADATA_DIR, {
create: true
})
} catch (error) {
console.error('Failed to clear storage:', error)
throw new Error(`Failed to clear storage: ${error}`)
}
}
/**
* Ensure the storage adapter is initialized
*/
private async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.init()
}
}
/**
* Convert a Map to a plain object for serialization
*/
private mapToObject<K extends string | number, V>(
map: Map<K, V>,
valueTransformer: (value: V) => any = (v) => v
): Record<string, any> {
const obj: Record<string, any> = {}
for (const [key, value] of map.entries()) {
obj[key.toString()] = valueTransformer(value)
}
return obj
}
}
/**
* In-memory storage adapter for environments where OPFS is not available
*/
export class MemoryStorage implements StorageAdapter {
private nodes: Map<string, HNSWNode> = new Map()
private edges: Map<string, Edge> = new Map()
private metadata: Map<string, any> = new Map()
public async init(): Promise<void> {
// Nothing to initialize for in-memory storage
}
public async saveNode(node: HNSWNode): Promise<void> {
// Create a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
this.nodes.set(node.id, nodeCopy)
}
public async getNode(id: string): Promise<HNSWNode | null> {
const node = this.nodes.get(id)
if (!node) {
return null
}
// Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
id: node.id,
vector: [...node.vector],
connections: new Map()
}
// Copy connections
for (const [level, connections] of node.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections))
}
return nodeCopy
}
public async getAllNodes(): Promise<HNSWNode[]> {
const nodes: HNSWNode[] = []
for (const nodeId of this.nodes.keys()) {
const node = await this.getNode(nodeId)
if (node) {
nodes.push(node)
}
}
return nodes
}
public async deleteNode(id: string): Promise<void> {
this.nodes.delete(id)
}
public async saveMetadata(id: string, metadata: any): Promise<void> {
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)))
}
public async getMetadata(id: string): Promise<any | null> {
const metadata = this.metadata.get(id)
if (!metadata) {
return null
}
return JSON.parse(JSON.stringify(metadata))
}
public async saveEdge(edge: Edge): Promise<void> {
// Create a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata ? JSON.parse(JSON.stringify(edge.metadata)) : undefined
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
this.edges.set(edge.id, edgeCopy)
}
public async getEdge(id: string): Promise<Edge | null> {
const edge = this.edges.get(id)
if (!edge) {
return null
}
// Return a deep copy to avoid reference issues
const edgeCopy: Edge = {
id: edge.id,
vector: [...edge.vector],
connections: new Map(),
sourceId: edge.sourceId,
targetId: edge.targetId,
type: edge.type,
weight: edge.weight,
metadata: edge.metadata ? JSON.parse(JSON.stringify(edge.metadata)) : undefined
}
// Copy connections
for (const [level, connections] of edge.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections))
}
return edgeCopy
}
public async getAllEdges(): Promise<Edge[]> {
const edges: Edge[] = []
for (const edgeId of this.edges.keys()) {
const edge = await this.getEdge(edgeId)
if (edge) {
edges.push(edge)
}
}
return edges
}
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges: Edge[] = []
for (const edge of this.edges.values()) {
if (edge.sourceId === sourceId) {
const edgeCopy = await this.getEdge(edge.id)
if (edgeCopy) {
edges.push(edgeCopy)
}
}
}
return edges
}
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges: Edge[] = []
for (const edge of this.edges.values()) {
if (edge.targetId === targetId) {
const edgeCopy = await this.getEdge(edge.id)
if (edgeCopy) {
edges.push(edgeCopy)
}
}
}
return edges
}
public async getEdgesByType(type: string): Promise<Edge[]> {
const edges: Edge[] = []
for (const edge of this.edges.values()) {
if (edge.type === type) {
const edgeCopy = await this.getEdge(edge.id)
if (edgeCopy) {
edges.push(edgeCopy)
}
}
}
return edges
}
public async deleteEdge(id: string): Promise<void> {
this.edges.delete(id)
}
public async clear(): Promise<void> {
this.nodes.clear()
this.edges.clear()
this.metadata.clear()
}
}
/**
* Factory function to create the appropriate storage adapter based on the environment
*/
export async function createStorage(): Promise<StorageAdapter> {
// Check if we're in a Node.js environment
const isNode = typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null
if (isNode) {
// In Node.js, use FileSystemStorage
try {
const fileSystemModule = await import('./fileSystemStorage.js')
return new fileSystemModule.FileSystemStorage()
} catch (error) {
console.warn('Failed to load FileSystemStorage, falling back to in-memory storage:', error)
return new MemoryStorage()
}
} else {
// In browser, try OPFS first
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
return opfsStorage
} else {
console.warn('OPFS is not available, falling back to in-memory storage')
return new MemoryStorage()
}
}
}

283
src/types/augmentations.ts Normal file
View file

@ -0,0 +1,283 @@
/** Common types for augmentation system */
type WebSocketConnection = {
connectionId: string
url: string
status: 'connected' | 'disconnected' | 'error'
}
type DataCallback<T> = (data: T) => void
type AugmentationResponse<T> = Promise<{
success: boolean
data: T
error?: string
}>
/**
* Base interface for all Brainy augmentations.
* All augmentations must implement these core properties.
*/
interface IAugmentation {
/** A unique identifier for the augmentation (e.g., "my-reasoner-v1") */
readonly name: string
/** A human-readable description of the augmentation's purpose */
readonly description: string
/**
* Initializes the augmentation. This method is called when Brainy starts up.
* @returns A Promise that resolves when initialization is complete
*/
initialize(): Promise<void>
shutDown(): Promise<void>
getStatus(): Promise<'active' | 'inactive' | 'error'>
}
/**
* Interface for WebSocket support.
* Augmentations that implement this interface can communicate via WebSockets.
*/
interface IWebSocketSupport {
/**
* Establishes a WebSocket connection.
* @param url The WebSocket server URL to connect to
* @param protocols Optional subprotocols
* @returns A Promise resolving to a connection handle or status
*/
connectWebSocket(url: string, protocols?: string | string[]): Promise<WebSocketConnection>
/**
* Sends data through an established WebSocket connection.
* @param connectionId The identifier of the established connection
* @param data The data to send (will be serialized if not a string)
*/
sendWebSocketMessage(connectionId: string, data: unknown): Promise<void>
/**
* Registers a callback for incoming WebSocket messages.
* @param connectionId The identifier of the established connection
* @param callback The function to call when a message is received
*/
onWebSocketMessage(connectionId: string, callback: DataCallback<unknown>): Promise<void>
/**
* Closes an established WebSocket connection.
* @param connectionId The identifier of the established connection
* @param code Optional close code
* @param reason Optional close reason
*/
closeWebSocket(connectionId: string, code?: number, reason?: string): Promise<void>
}
namespace BrainyAugmentations {
/**
* Interface for Cognitions augmentations.
* These augmentations enable advanced reasoning, inference, and logical operations.
*/
export interface ICognitionAugmentation extends IAugmentation {
/**
* Performs a reasoning operation based on current knowledge.
* @param query The specific reasoning task or question
* @param context Optional additional context for the reasoning
*/
reason(query: string, context?: Record<string, unknown>): AugmentationResponse<{
inference: string
confidence: number
}>
/**
* Infers relationships or new facts from existing data.
* @param dataSubset A subset of data to infer from
*/
infer(dataSubset: Record<string, unknown>): AugmentationResponse<Record<string, unknown>>
/**
* Executes a logical operation or rule set.
* @param ruleId The identifier of the rule or logic to apply
* @param input Data to apply the logic to
*/
executeLogic(ruleId: string, input: Record<string, unknown>): AugmentationResponse<boolean>
}
/**
* Interface for Senses augmentations.
* These augmentations ingest and process raw, unstructured data into nouns and verbs.
*/
export interface ISenseAugmentation extends IAugmentation {
/**
* Processes raw input data into structured nouns and verbs.
* @param rawData The raw, unstructured data (e.g., text, image buffer, audio stream)
* @param dataType The type of raw data (e.g., 'text', 'image', 'audio')
*/
processRawData(rawData: Buffer | string, dataType: string): AugmentationResponse<{
nouns: string[]
verbs: string[]
}>
/**
* Registers a listener for real-time data feeds.
* @param feedUrl The URL or identifier of the real-time feed
* @param callback A function to call with processed data
*/
listenToFeed(
feedUrl: string,
callback: DataCallback<{ nouns: string[]; verbs: string[] }>
): Promise<void>
}
/**
* Interface for Perceptions augmentations.
* These augmentations interpret, contextualize, and visualize identified nouns and verbs.
*/
export interface IPerceptionAugmentation extends IAugmentation {
/**
* Interprets and contextualizes processed nouns and verbs.
* @param nouns The list of identified nouns
* @param verbs The list of identified verbs
* @param context Optional additional context for interpretation
*/
interpret(
nouns: string[],
verbs: string[],
context?: Record<string, unknown>
): AugmentationResponse<Record<string, unknown>>
/**
* Organizes and filters information.
* @param data The data to organize (e.g., interpreted perceptions)
* @param criteria Optional criteria for filtering/prioritization
*/
organize(
data: Record<string, unknown>,
criteria?: Record<string, unknown>
): AugmentationResponse<Record<string, unknown>>
/**
* Generates a visualization based on the provided data.
* @param data The data to visualize (e.g., interpreted patterns)
* @param visualizationType The desired type of visualization (e.g., 'graph', 'chart')
*/
generateVisualization(
data: Record<string, unknown>,
visualizationType: string
): AugmentationResponse<string | Buffer | Record<string, unknown>>
}
/**
* Interface for Activations augmentations.
* These augmentations dictate how Brainy initiates actions, responses, or data manipulations.
*/
export interface IActivationAugmentation extends IAugmentation {
/**
* Triggers an action based on a processed command or internal state.
* @param actionName The name of the action to trigger
* @param parameters Optional parameters for the action
*/
triggerAction(
actionName: string,
parameters?: Record<string, unknown>
): AugmentationResponse<unknown>
/**
* Generates an expressive output or response from Brainy.
* @param knowledgeId The identifier of the knowledge to express
* @param format The desired output format (e.g., 'text', 'json')
*/
generateOutput(knowledgeId: string, format: string): AugmentationResponse<string | Record<string, unknown>>
/**
* Interacts with an external system or API.
* @param systemId The identifier of the external system
* @param payload The data to send to the external system
*/
interactExternal(systemId: string, payload: Record<string, unknown>): AugmentationResponse<unknown>
}
/**
* Interface for Dialogs augmentations.
* These augmentations facilitate natural language understanding and generation for conversational interaction.
*/
export interface IDialogAugmentation extends IAugmentation {
/**
* Processes a user's natural language input (query).
* @param naturalLanguageQuery The raw text query from the user
* @param sessionId An optional session ID for conversational context
*/
processUserInput(naturalLanguageQuery: string, sessionId?: string): AugmentationResponse<{
intent: string
nouns: string[]
verbs: string[]
context: Record<string, unknown>
}>
/**
* Generates a natural language response based on Brainy's knowledge and interpreted input.
* @param interpretedInput The output from `processUserInput` or similar
* @param knowledgeContext Relevant knowledge retrieved from Brainy
* @param sessionId An optional session ID for conversational context
*/
generateResponse(
interpretedInput: Record<string, unknown>,
knowledgeContext: Record<string, unknown>,
sessionId?: string
): AugmentationResponse<string>
/**
* Manages and updates conversational context.
* @param sessionId The session ID
* @param contextUpdate The data to update the context with
*/
manageContext(sessionId: string, contextUpdate: Record<string, unknown>): Promise<void>
}
/**
* Interface for Conduits augmentations.
* These augmentations establish and manage high-bandwidth, dedicated channels for structured, programmatic two-way data exchange.
*/
export interface IConduitAugmentation extends IAugmentation {
/**
* Establishes a connection for programmatic data exchange.
* @param targetSystemId The identifier of the external system to connect to
* @param config Configuration details for the connection (e.g., API keys, endpoints)
*/
establishConnection(
targetSystemId: string,
config: Record<string, unknown>
): AugmentationResponse<WebSocketConnection>
/**
* Reads structured data directly from Brainy's knowledge graph.
* @param query A structured query (e.g., graph query language, object path)
* @param options Optional query options (e.g., depth, filters)
*/
readData(
query: Record<string, unknown>,
options?: Record<string, unknown>
): AugmentationResponse<unknown>
/**
* Writes or updates structured data directly into Brainy's knowledge graph.
* @param data The structured data to write/update
* @param options Optional write options (e.g., merge, overwrite)
*/
writeData(
data: Record<string, unknown>,
options?: Record<string, unknown>
): AugmentationResponse<unknown>
/**
* Monitors a specific data stream or event within Brainy for external systems.
* @param streamId The identifier of the data stream or event
* @param callback A function to call when new data/events occur
*/
monitorStream(streamId: string, callback: DataCallback<unknown>): Promise<void>
}
}
/** WebSocket-enabled augmentation interfaces */
type IWebSocketCognitionAugmentation = BrainyAugmentations.ICognitionAugmentation & IWebSocketSupport
type IWebSocketSenseAugmentation = BrainyAugmentations.ISenseAugmentation & IWebSocketSupport
type IWebSocketPerceptionAugmentation = BrainyAugmentations.IPerceptionAugmentation & IWebSocketSupport
type IWebSocketActivationAugmentation = BrainyAugmentations.IActivationAugmentation & IWebSocketSupport
type IWebSocketDialogAugmentation = BrainyAugmentations.IDialogAugmentation & IWebSocketSupport
type IWebSocketConduitAugmentation = BrainyAugmentations.IConduitAugmentation & IWebSocketSupport

12
src/types/fileSystemTypes.ts Normal file
View file

@ -0,0 +1,12 @@
/**
* Type declarations for the File System Access API
* Extends the FileSystemDirectoryHandle interface to include the [Symbol.asyncIterator] method
*/
// Extend the FileSystemDirectoryHandle interface
interface FileSystemDirectoryHandle {
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileSystemHandle]>;
}
// Export something to make this a module
export const fileSystemTypesLoaded = true;

129
src/types/graphTypes.ts Normal file
View file

@ -0,0 +1,129 @@
// Common metadata types
/**
* Represents a high-precision timestamp with seconds and nanoseconds
* Used for tracking creation and update times of graph elements
*/
interface Timestamp {
seconds: number
nanoseconds: number
}
/**
* Metadata about the creator/source of a graph noun
* Tracks which augmentation and model created the element
*/
interface CreatorMetadata {
augmentation: string // Name of the augmentation that created this element
version: string // Version of the augmentation
model: string // Model identifier used in creation
modelVersion: string // Version of the model
}
/**
* Base interface for nodes (nouns) in the graph
* Represents entities like people, places, things, etc.
*/
export interface GraphNoun {
id: string // Unique identifier for the noun
createdBy: CreatorMetadata // Information about what created this noun
noun: NounType // Type classification of the noun
createdAt: Timestamp // When the noun was created
updatedAt: Timestamp // When the noun was last updated
data?: Record<string, unknown> // Additional flexible data storage
embedding?: number[] // Vector representation of the noun
}
/**
* Base interface for edges (verbs) in the graph
* Represents relationships between nouns
*/
export interface GraphVerb {
id: string // Unique identifier for the verb
source: string // ID of the source noun
target: string // ID of the target noun
label?: string // Optional descriptive label
verb: VerbType // Type of relationship
createdAt: Timestamp // When the verb was created
updatedAt: Timestamp // When the verb was last updated
data?: Record<string, unknown> // Additional flexible data storage
embedding?: number[] // Vector representation of the relationship
confidence?: number // Confidence score (0-1)
weight?: number // Strength/importance of the relationship
}
/**
* Version of GraphVerb for embedded relationships
* Used when the source is implicit from the parent document
*/
export type EmbeddedGraphVerb = Omit<GraphVerb, 'source'>
// Proper Noun interfaces - extend GraphNoun with specific noun types
/**
* Represents a person entity in the graph
*/
export interface Person extends GraphNoun {
noun: typeof NounType.Person
}
/**
* Represents a physical location in the graph
*/
export interface Place extends GraphNoun {
noun: typeof NounType.Place
}
/**
* Represents a physical or virtual object in the graph
*/
export interface Thing extends GraphNoun {
noun: typeof NounType.Thing
}
/**
* Represents an event or occurrence in the graph
*/
export interface Event extends GraphNoun {
noun: typeof NounType.Event
}
/**
* Represents an abstract concept or idea in the graph
*/
export interface Concept extends GraphNoun {
noun: typeof NounType.Concept
}
/**
* Represents content (text, media, etc.) in the graph
*/
export interface Content extends GraphNoun {
noun: typeof NounType.Content
}
/**
* Defines valid noun types for graph entities
* Used for categorizing different types of nodes
*/
export const NounType = {
Person: 'person', // Person entities
Place: 'place', // Physical locations
Thing: 'thing', // Physical or virtual objects
Event: 'event', // Events or occurrences
Concept: 'concept', // Abstract concepts or ideas
Content: 'content' // Content items
} as const
export type NounType = (typeof NounType)[keyof typeof NounType]
/**
* Defines valid verb types for relationships
* Used for categorizing different types of connections
*/
export const VerbType = {
AttributedTo: 'attributedTo', // Indicates attribution or authorship
Controls: 'controls', // Indicates control or ownership
Created: 'created', // Indicates creation or authorship
Earned: 'earned', // Indicates achievement or acquisition
Owns: 'owns' // Indicates ownership
} as const
export type VerbType = (typeof VerbType)[keyof typeof VerbType]

14
src/types/tensorflowTypes.ts Normal file
View file

@ -0,0 +1,14 @@
// Type declarations for TensorFlow.js models
// This file is a placeholder for TensorFlow.js model types
// We're using type assertions in the code instead of module declarations
// Define some basic types that might be useful
export interface TensorflowModel {
load(): Promise<any>;
embed(data: string[]): any;
dispose(): void;
}
// Export a dummy constant to make this a proper module
export const tensorflowModelsLoaded = true;

88
src/utils/distance.ts Normal file
View file

@ -0,0 +1,88 @@
/**
* Distance functions for vector similarity calculations
*/
import { DistanceFunction, Vector } from '../coreTypes.js'
/**
* Calculates the Euclidean distance between two vectors
* Lower values indicate higher similarity
*/
export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions')
}
let sum = 0
for (let i = 0; i < a.length; i++) {
const diff = a[i] - b[i]
sum += diff * diff
}
return Math.sqrt(sum)
}
/**
* Calculates the cosine distance between two vectors
* Lower values indicate higher similarity
* Range: 0 (identical) to 2 (opposite)
*/
export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions')
}
let dotProduct = 0
let normA = 0
let normB = 0
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i]
normA += a[i] * a[i]
normB += b[i] * b[i]
}
if (normA === 0 || normB === 0) {
return 2 // Maximum distance for zero vectors
}
const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
// Convert cosine similarity (-1 to 1) to distance (0 to 2)
return 1 - similarity
}
/**
* Calculates the Manhattan (L1) distance between two vectors
* Lower values indicate higher similarity
*/
export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions')
}
let sum = 0
for (let i = 0; i < a.length; i++) {
sum += Math.abs(a[i] - b[i])
}
return sum
}
/**
* Calculates the dot product similarity between two vectors
* Higher values indicate higher similarity
* Converted to a distance metric (lower is better)
*/
export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions')
}
let dotProduct = 0
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i]
}
// Convert to a distance metric (lower is better)
return -dotProduct
}

168
src/utils/embedding.ts Normal file
View file

@ -0,0 +1,168 @@
/**
* Embedding functions for converting data to vectors
*/
import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js'
/**
* Simple character-based embedding function
* This is a very basic implementation for demo purposes
*/
export class SimpleEmbedding implements EmbeddingModel {
private initialized = false
/**
* Initialize the embedding model
*/
public async init(): Promise<void> {
this.initialized = true
return Promise.resolve()
}
/**
* Embed text into a vector using character frequencies
* @param data Text to embed
*/
public async embed(data: string): Promise<Vector> {
if (!this.initialized) {
await this.init()
}
// Only handle string data
if (typeof data !== 'string') {
throw new Error('SimpleEmbedding only supports string data')
}
// Normalize the text
const normalizedText = data.toLowerCase().trim()
// Create a simple 4-dimensional vector based on character frequencies
const vector: Vector = [0, 0, 0, 0]
// Count vowels, consonants, numbers, and special characters
for (let i = 0; i < normalizedText.length; i++) {
const char = normalizedText[i]
if ('aeiou'.includes(char)) {
vector[0] += 0.1 // Vowels affect first dimension
} else if ('bcdfghjklmnpqrstvwxyz'.includes(char)) {
vector[1] += 0.1 // Consonants affect second dimension
} else if ('0123456789'.includes(char)) {
vector[2] += 0.1 // Numbers affect third dimension
} else {
vector[3] += 0.1 // Special chars affect fourth dimension
}
}
// Normalize the vector
const magnitude = Math.sqrt(
vector.reduce((sum, val) => sum + val * val, 0)
)
if (magnitude > 0) {
return vector.map((val) => val / magnitude)
}
return vector
}
/**
* Dispose of the model resources
*/
public async dispose(): Promise<void> {
this.initialized = false
return Promise.resolve()
}
}
/**
* TensorFlow Universal Sentence Encoder embedding model
* Requires @tensorflow/tfjs and @tensorflow-models/universal-sentence-encoder to be installed
*/
export class UniversalSentenceEncoder implements EmbeddingModel {
private model: any = null
private initialized = false
private tf: any = null
private use: any = null
/**
* Initialize the embedding model
*/
public async init(): Promise<void> {
try {
// Dynamically import TensorFlow.js and Universal Sentence Encoder
// Use type assertions to tell TypeScript these modules exist
this.tf = await import('@tensorflow/tfjs/dist/tf.esm.js' as any)
this.use = await import('@tensorflow-models/universal-sentence-encoder/dist/universal-sentence-encoder.esm.js' as any)
// Load the model
this.model = await this.use.load()
this.initialized = true
} catch (error) {
console.error('Failed to initialize Universal Sentence Encoder:', error)
throw new Error(`Failed to initialize Universal Sentence Encoder: ${error}`)
}
}
/**
* Embed text into a vector using Universal Sentence Encoder
* @param data Text to embed
*/
public async embed(data: string | string[]): Promise<Vector> {
if (!this.initialized) {
await this.init()
}
try {
// Handle different input types
let textToEmbed: string[]
if (typeof data === 'string') {
textToEmbed = [data]
} else if (Array.isArray(data) && data.every(item => typeof item === 'string')) {
textToEmbed = data
} else {
throw new Error('UniversalSentenceEncoder only supports string or string[] data')
}
// Get embeddings
const embeddings = await this.model.embed(textToEmbed)
// Convert to array and return the first embedding
const embeddingArray = await embeddings.array()
return embeddingArray[0]
} catch (error) {
console.error('Failed to embed text with Universal Sentence Encoder:', error)
throw new Error(`Failed to embed text with Universal Sentence Encoder: ${error}`)
}
}
/**
* Dispose of the model resources
*/
public async dispose(): Promise<void> {
if (this.model && this.tf) {
try {
// Dispose of the model and tensors
this.model.dispose()
this.tf.disposeVariables()
this.initialized = false
} catch (error) {
console.error('Failed to dispose Universal Sentence Encoder:', error)
}
}
return Promise.resolve()
}
}
/**
* Create an embedding function from an embedding model
* @param model Embedding model to use
*/
export function createEmbeddingFunction(model: EmbeddingModel): EmbeddingFunction {
return async (data: any): Promise<Vector> => {
return await model.embed(data)
}
}
/**
* Default embedding function using UniversalSentenceEncoder
*/
export const defaultEmbeddingFunction: EmbeddingFunction = createEmbeddingFunction(new UniversalSentenceEncoder())

2
src/utils/index.ts Normal file
View file

@ -0,0 +1,2 @@
export * from './distance.js'
export * from './embedding.js'

30
tsconfig.json Normal file
View file

@ -0,0 +1,30 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"strict": true,
"declaration": true,
"outDir": "./dist",
"rootDir": "./src",
"lib": [
"DOM",
"ESNext"
],
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"allowImportingTsExtensions": false,
"noEmit": false,
"preserveConstEnums": true,
"sourceMap": true
},
"include": [
"src/**/*"
],
"exclude": [
"node_modules",
"dist",
"**/*.test.ts"
]
}