soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat: add write-only mode support and example usage**

Browse source 
- Introduced `writeOnly` mode in `BrainyData` allowing optimized data ingestion by skipping index loading and disabling search operations.
- Enhanced `BrainyDataConfig` to include `writeOnly` support and validate compatibility with `readOnly` mode.
- Implemented error handling for search attempts during `writeOnly` mode.
- Updated the README with detailed usage examples for database modes, including `writeOnly` and `readOnly`.
- Added `examples/write-only-mode.js` to demonstrate practical applications of the `writeOnly` mode.

**Purpose**: Optimize memory usage and startup time for data ingestion scenarios by enabling write-only mode with comprehensive documentation and examples.
This commit is contained in:
David Snelling 2025-07-30 13:32:30 -07:00
parent 268b591e6c
commit d8e3aced11
3 changed files with 3696 additions and 3426 deletions
Download patch file Download diff file Expand all files Collapse all files

31
README.md
View file

@ -598,6 +598,37 @@ await db.deleteVerb(verbId)
## Advanced Configuration
### Database Modes
Brainy supports special operational modes that restrict certain operations:
```typescript
import { BrainyData } from '@soulcraft/brainy'
// Create and initialize the database
const db = new BrainyData()
await db.init()
// Set the database to read-only mode (prevents write operations)
db.setReadOnly(true)
// Check if the database is in read-only mode
const isReadOnly = db.isReadOnly() // Returns true
// Set the database to write-only mode (prevents search operations)
db.setWriteOnly(true)
// Check if the database is in write-only mode
const isWriteOnly = db.isWriteOnly() // Returns true
// Reset to normal mode (allows both read and write operations)
db.setReadOnly(false)
db.setWriteOnly(false)
```
- **Read-Only Mode**: When enabled, prevents all write operations (add, update, delete). Useful for deployment scenarios where you want to prevent modifications to the database.
- **Write-Only Mode**: When enabled, prevents all search operations. Useful for initial data loading or when you want to optimize for write performance.
### Embedding
```typescript

68
examples/write-only-mode.js Normal file
View file

@ -0,0 +1,68 @@
/**
* Example demonstrating the write-only mode in BrainyData
*
* This mode is useful for data ingestion scenarios where you only need to insert data
* and don't need to perform any search operations. It significantly reduces memory usage
* by not loading the index into memory.
*/
import { BrainyData } from '../dist/index.js'
// eslint-disable-next-line no-undef
const console = globalThis.console
async function demonstrateWriteOnlyMode() {
console.log('Demonstrating write-only mode in BrainyData')
// Create a BrainyData instance in write-only mode
const db = new BrainyData({
writeOnly: true,
storage: {
// Use file system storage for persistence
forceFileSystemStorage: true
}
})
// Initialize the database
// This will skip loading the index into memory, saving memory and startup time
await db.init()
console.log('Database initialized in write-only mode')
console.log('Is write-only:', db.isWriteOnly()) // Should print true
// Add some data
// This works normally even in write-only mode
const id1 = await db.add('This is a test document', { title: 'Test Document 1' })
const id2 = await db.add('Another test document with different content', { title: 'Test Document 2' })
console.log(`Added documents with IDs: ${id1}, ${id2}`)
// Try to search (this will throw an error in write-only mode)
try {
await db.search('test', 5)
} catch (error) {
console.log('Search error (expected):', error.message)
}
// Switch to normal mode to perform searches
console.log('Switching to normal mode...')
db.setWriteOnly(false)
// Re-initialize to load the index
await db.init()
console.log('Database reinitialized in normal mode')
console.log('Is write-only:', db.isWriteOnly()) // Should print false
// Now search works
const results = await db.search('test', 5)
console.log('Search results:', results.map(r => ({ id: r.id, score: r.score, title: r.metadata?.title })))
// Clean up
await db.shutDown()
}
// Run the example
demonstrateWriteOnlyMode().catch(error => {
console.error('Error in write-only mode example:', error)
})

349
src/brainyData.ts
View file

@ -3,13 +3,13 @@
* Main class that provides the vector database functionality
*/
import {v4 as uuidv4} from 'uuid'
import {HNSWIndex} from './hnsw/hnswIndex.js'
import { v4 as uuidv4 } from 'uuid'
import { HNSWIndex } from './hnsw/hnswIndex.js'
import {
HNSWIndexOptimized,
HNSWOptimizedConfig
} from './hnsw/hnswIndexOptimized.js'
import {createStorage} from './storage/storageFactory.js'
import { createStorage } from './storage/storageFactory.js'
import {
DistanceFunction,
GraphVerb,
@ -30,14 +30,18 @@ import {
euclideanDistance,
cleanupWorkerPools
} from './utils/index.js'
import {NounType, VerbType, GraphNoun} from './types/graphTypes.js'
import { NounType, VerbType, GraphNoun } from './types/graphTypes.js'
import {
ServerSearchConduitAugmentation,
createServerSearchAugmentations
} from './augmentations/serverSearchAugmentations.js'
import {WebSocketConnection, AugmentationType, IAugmentation} from './types/augmentations.js'
import {BrainyDataInterface} from './types/brainyDataInterface.js'
import {augmentationPipeline} from './augmentationPipeline.js'
import {
WebSocketConnection,
AugmentationType,
IAugmentation
} from './types/augmentations.js'
import { BrainyDataInterface } from './types/brainyDataInterface.js'
import { augmentationPipeline } from './augmentationPipeline.js'
export interface BrainyDataConfig {
/**
@ -103,6 +107,13 @@ export interface BrainyDataConfig {
*/
readOnly?: boolean
/**
* Set the database to write-only mode
* When true, the index is not loaded into memory and search operations will throw an error
* This is useful for data ingestion scenarios where only write operations are needed
*/
writeOnly?: boolean
/**
* Remote server configuration for search operations
*/
@ -230,17 +241,20 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private distanceFunction: DistanceFunction
private requestPersistentStorage: boolean
private readOnly: boolean
private writeOnly: boolean
private storageConfig: BrainyDataConfig['storage'] = {}
private useOptimizedIndex: boolean = false
private _dimensions: number
private loggingConfig: BrainyDataConfig['logging'] = {verbose: true}
private loggingConfig: BrainyDataConfig['logging'] = { verbose: true }
// Timeout and retry configuration
private timeoutConfig: BrainyDataConfig['timeouts'] = {}
private retryConfig: BrainyDataConfig['retryPolicy'] = {}
// Real-time update properties
private realtimeUpdateConfig: Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> = {
private realtimeUpdateConfig: Required<
NonNullable<BrainyDataConfig['realtimeUpdates']>
> = {
enabled: false,
interval: 30000, // 30 seconds
updateStatistics: true,
@ -323,6 +337,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Set read-only flag
this.readOnly = config.readOnly || false
// Set write-only flag
this.writeOnly = config.writeOnly || false
// Validate that readOnly and writeOnly are not both true
if (this.readOnly && this.writeOnly) {
throw new Error('Database cannot be both read-only and write-only')
}
// Store storage configuration for later use in init()
this.storageConfig = config.storage || {}
@ -356,6 +378,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
/**
* Check if the database is in write-only mode and throw an error if it is
* @throws Error if the database is in write-only mode
*/
private checkWriteOnly(): void {
if (this.writeOnly) {
throw new Error(
'Cannot perform search operation: database is in write-only mode'
)
}
}
/**
* Start real-time updates if enabled in the configuration
* This will periodically check for new data in storage and update the in-memory index and statistics
@ -372,21 +406,28 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Set the initial last known noun count
this.getNounCount().then(count => {
this.getNounCount()
.then((count) => {
this.lastKnownNounCount = count
}).catch(error => {
console.warn('Failed to get initial noun count for real-time updates:', error)
})
.catch((error) => {
console.warn(
'Failed to get initial noun count for real-time updates:',
error
)
})
// Start the update timer
this.updateTimerId = setInterval(() => {
this.checkForUpdates().catch(error => {
this.checkForUpdates().catch((error) => {
console.warn('Error during real-time update check:', error)
})
}, this.realtimeUpdateConfig.interval)
if (this.loggingConfig?.verbose) {
console.log(`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`)
console.log(
`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`
)
}
}
@ -421,7 +462,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* Enable real-time updates with the specified configuration
* @param config Configuration for real-time updates
*/
public enableRealtimeUpdates(config?: Partial<BrainyDataConfig['realtimeUpdates']>): void {
public enableRealtimeUpdates(
config?: Partial<BrainyDataConfig['realtimeUpdates']>
): void {
// Update configuration if provided
if (config) {
this.realtimeUpdateConfig = {
@ -454,7 +497,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* Get the current real-time update configuration
* @returns The current real-time update configuration
*/
public getRealtimeUpdateConfig(): Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> {
public getRealtimeUpdateConfig(): Required<
NonNullable<BrainyDataConfig['realtimeUpdates']>
> {
return { ...this.realtimeUpdateConfig }
}
@ -514,7 +559,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try {
// Get changes since the last update
const changes = await this.storage.getChangesSince(this.lastUpdateTime, 1000) // Limit to 1000 changes per batch
const changes = await this.storage.getChangesSince(
this.lastUpdateTime,
1000
) // Limit to 1000 changes per batch
let addedCount = 0
let updatedCount = 0
@ -549,7 +597,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
if (this.loggingConfig?.verbose) {
console.log(`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`)
console.log(
`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`
)
}
}
break
@ -561,26 +611,38 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
deletedCount++
if (this.loggingConfig?.verbose) {
console.log(`Removed noun ${change.entityId} from index during real-time update`)
console.log(
`Removed noun ${change.entityId} from index during real-time update`
)
}
}
break
}
} catch (changeError) {
console.error(`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`, changeError)
console.error(
`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`,
changeError
)
// Continue with other changes
}
}
if (this.loggingConfig?.verbose && (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) {
console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`)
if (
this.loggingConfig?.verbose &&
(addedCount > 0 || updatedCount > 0 || deletedCount > 0)
) {
console.log(
`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`
)
}
// Update the last known noun count
this.lastKnownNounCount = await this.getNounCount()
} catch (error) {
console.error('Failed to apply changes from log, falling back to full scan:', error)
console.error(
'Failed to apply changes from log, falling back to full scan:',
error
)
// Fallback to full scan if change log fails
await this.applyChangesFromFullScan()
}
@ -604,7 +666,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const indexNounIds = new Set(indexNouns.keys())
// Find nouns that are in storage but not in the index
const newNouns = nouns.filter(noun => !indexNounIds.has(noun.id))
const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id))
// Add new nouns to the index
for (const noun of newNouns) {
@ -623,7 +685,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
})
if (this.loggingConfig?.verbose) {
console.log(`Added new noun ${noun.id} to index during real-time update`)
console.log(
`Added new noun ${noun.id} to index during real-time update`
)
}
}
@ -631,7 +695,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.lastKnownNounCount = currentCount
if (this.loggingConfig?.verbose && newNouns.length > 0) {
console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`)
console.log(
`Real-time update: Added ${newNouns.length} new nouns to index using full scan`
)
}
}
} catch (error) {
@ -648,7 +714,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private getCurrentAugmentation(): string {
try {
// Get all registered augmentations
const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes()
const augmentationTypes =
augmentationPipeline.getAvailableAugmentationTypes()
// Check each type of augmentation
for (const type of augmentationTypes) {
@ -708,7 +775,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Try again with a different approach - use the non-threaded version
// This is a fallback in case the threaded version fails
const {createTensorFlowEmbeddingFunction} = await import(
const { createTensorFlowEmbeddingFunction } = await import(
'./utils/embedding.js'
)
const fallbackEmbeddingFunction = createTensorFlowEmbeddingFunction()
@ -742,15 +809,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Ensure s3Storage has all required fields if it's provided
if (storageOptions.s3Storage) {
// Only include s3Storage if all required fields are present
if (storageOptions.s3Storage.bucketName &&
if (
storageOptions.s3Storage.bucketName &&
storageOptions.s3Storage.accessKeyId &&
storageOptions.s3Storage.secretAccessKey) {
storageOptions.s3Storage.secretAccessKey
) {
// All required fields are present, keep s3Storage as is
} else {
// Missing required fields, remove s3Storage to avoid type errors
const {s3Storage, ...rest} = storageOptions
const { s3Storage, ...rest } = storageOptions
storageOptions = rest
console.warn('Ignoring s3Storage configuration due to missing required fields')
console.warn(
'Ignoring s3Storage configuration due to missing required fields'
)
}
}
@ -766,6 +837,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.index.setStorage(this.storage!)
}
// In write-only mode, skip loading the index into memory
if (this.writeOnly) {
if (this.loggingConfig?.verbose) {
console.log('Database is in write-only mode, skipping index loading')
}
} else {
// Load all nouns from storage
const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
@ -788,6 +865,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
vector: noun.vector
})
}
}
// Connect to remote server if configured with autoConnect
if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) {
@ -828,7 +906,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try {
// Create server search augmentations
const {conduit, connection} = await createServerSearchAugmentations(
const { conduit, connection } = await createServerSearchAugmentations(
serverUrl,
{
protocols,
@ -888,10 +966,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Check if input is already a vector
if (
Array.isArray(vectorOrData) &&
!options.forceEmbed
) {
if (Array.isArray(vectorOrData) && !options.forceEmbed) {
// Input is already a vector (and we've validated it contains only numbers)
vector = vectorOrData
} else {
@ -910,7 +985,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Validate vector dimensions
if (vector.length !== this._dimensions) {
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`)
throw new Error(
`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`
)
}
// Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID
@ -921,7 +998,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
: uuidv4())
// Add to index
await this.index.addItem({id, vector})
await this.index.addItem({ id, vector })
// Get the noun from the index
const noun = this.index.getNouns().get(id)
@ -940,7 +1017,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Save metadata if provided and not empty
if (metadata !== undefined) {
// Skip saving if metadata is an empty object
if (metadata && typeof metadata === 'object' && Object.keys(metadata).length === 0) {
if (
metadata &&
typeof metadata === 'object' &&
Object.keys(metadata).length === 0
) {
// Don't save empty metadata
// Explicitly save null to ensure no metadata is stored
await this.storage!.saveMetadata(id, null)
@ -992,13 +1073,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
let metadataToSave = metadata
if (metadata && typeof metadata === 'object') {
// Always make a copy without adding the ID
metadataToSave = {...metadata}
metadataToSave = { ...metadata }
}
await this.storage!.saveMetadata(id, metadataToSave)
// Track metadata statistics
const metadataService = options.service || this.getCurrentAugmentation()
const metadataService =
options.service || this.getCurrentAugmentation()
await this.storage!.incrementStatistic('metadata', metadataService)
}
}
@ -1041,7 +1123,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} = {}
): Promise<string> {
// Use the existing add method with forceEmbed to ensure text is embedded
return this.add(text, metadata, {...options, forceEmbed: true})
return this.add(text, metadata, { ...options, forceEmbed: true })
}
/**
@ -1066,7 +1148,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Add to local with addToRemote option
return this.add(vectorOrData, metadata, {...options, addToRemote: true})
return this.add(vectorOrData, metadata, { ...options, addToRemote: true })
}
/**
@ -1258,7 +1340,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Add to local with addToRemote option
return this.addBatch(items, {...options, addToRemote: true})
return this.addBatch(items, { ...options, addToRemote: true })
}
/**
@ -1274,7 +1356,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): R[] {
if (!service) return results
return results.filter(result => {
return results.filter((result) => {
if (!result.metadata || typeof result.metadata !== 'object') return false
if (!('createdBy' in result.metadata)) return false
@ -1316,9 +1398,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
return createdBy.augmentation === options.service
}
if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.')
throw new Error(
'BrainyData must be initialized before searching. Call init() first.'
)
}
// Check if database is in write-only mode
this.checkWriteOnly()
try {
let queryVector: Vector
@ -1346,7 +1433,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if query vector dimensions match the expected dimensions
if (queryVector.length !== this._dimensions) {
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`)
throw new Error(
`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`
)
}
// If no noun types specified, search all nouns
@ -1372,7 +1461,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Ensure metadata has the id field
if (metadata && typeof metadata === 'object') {
metadata = {...metadata, id} as T
metadata = { ...metadata, id } as T
}
searchResults.push({
@ -1432,7 +1521,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Ensure metadata has the id field
if (metadata && typeof metadata === 'object') {
metadata = {...metadata, id} as T
metadata = { ...metadata, id } as T
}
searchResults.push({
@ -1485,8 +1574,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.')
throw new Error(
'BrainyData must be initialized before searching. Call init() first.'
)
}
// Check if database is in write-only mode
this.checkWriteOnly()
// If searching for verbs directly
if (options.searchVerbs) {
const verbResults = await this.searchVerbs(queryVectorOrData, k, {
@ -1548,8 +1642,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} = {}
): Promise<SearchResult<T>[]> {
if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.')
throw new Error(
'BrainyData must be initialized before searching. Call init() first.'
)
}
// Check if database is in write-only mode
this.checkWriteOnly()
// If input is a string and not a vector, automatically vectorize it
let queryToUse = queryVectorOrData
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
@ -1636,10 +1735,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const outgoingVerbs = await this.storage!.getVerbsBySource(id)
// Filter to only include verbs of the specified type
const verbsOfType = outgoingVerbs.filter(verb => verb.type === options.relationType)
const verbsOfType = outgoingVerbs.filter(
(verb) => verb.type === options.relationType
)
// Get the target IDs
const targetIds = verbsOfType.map(verb => verb.target)
const targetIds = verbsOfType.map((verb) => verb.target)
// Get the actual entities for these IDs
const results: SearchResult<T>[] = []
@ -1858,11 +1959,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const graphNoun = metadata as unknown as GraphNoun
// Preserve existing createdBy and createdAt if they exist
const existingMetadata = await this.storage!.getMetadata(id) as any
const existingMetadata = (await this.storage!.getMetadata(id)) as any
if (existingMetadata &&
if (
existingMetadata &&
typeof existingMetadata === 'object' &&
'createdBy' in existingMetadata) {
'createdBy' in existingMetadata
) {
// Preserve the original creator information
graphNoun.createdBy = existingMetadata.createdBy
@ -2028,15 +2131,20 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Add the missing noun
await this.add(placeholderVector, metadata, {id: sourceId})
await this.add(placeholderVector, metadata, { id: sourceId })
// Get the newly created noun
sourceNoun = this.index.getNouns().get(sourceId)
console.warn(`Auto-created missing source noun with ID ${sourceId}`)
} catch (createError) {
console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError)
throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`)
console.error(
`Failed to auto-create source noun with ID ${sourceId}:`,
createError
)
throw new Error(
`Failed to auto-create source noun with ID ${sourceId}: ${createError}`
)
}
}
@ -2065,15 +2173,20 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Add the missing noun
await this.add(placeholderVector, metadata, {id: targetId})
await this.add(placeholderVector, metadata, { id: targetId })
// Get the newly created noun
targetNoun = this.index.getNouns().get(targetId)
console.warn(`Auto-created missing target noun with ID ${targetId}`)
} catch (createError) {
console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError)
throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`)
console.error(
`Failed to auto-create target noun with ID ${targetId}:`,
createError
)
throw new Error(
`Failed to auto-create target noun with ID ${targetId}: ${createError}`
)
}
}
@ -2181,7 +2294,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Add to index
await this.index.addItem({id, vector: verbVector})
await this.index.addItem({ id, vector: verbVector })
// Get the noun from the index
const indexNoun = this.index.getNouns().get(id)
@ -2358,7 +2471,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const allVerbs = await this.storage!.getAllVerbs()
// Create a set of verb IDs for faster lookup
const verbIds = new Set(allVerbs.map(verb => verb.id))
const verbIds = new Set(allVerbs.map((verb) => verb.id))
// Get all nouns from the index
const nouns = this.index.getNouns()
@ -2395,10 +2508,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* @param options Additional options for retrieving statistics
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
*/
public async getStatistics(options: {
public async getStatistics(
options: {
service?: string | string[] // Filter statistics by service(s)
forceRefresh?: boolean // Force a refresh of statistics from storage
} = {}): Promise<{
} = {}
): Promise<{
nounCount: number
verbCount: number
metadataCount: number
@ -2463,8 +2578,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Filter by service if specified
const services = options.service
? (Array.isArray(options.service) ? options.service : [options.service])
: Object.keys({...stats.nounCount, ...stats.verbCount, ...stats.metadataCount})
? Array.isArray(options.service)
? options.service
: [options.service]
: Object.keys({
...stats.nounCount,
...stats.verbCount,
...stats.metadataCount
})
// Calculate totals and service breakdown
for (const service of services) {
@ -2555,9 +2676,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Initialize persistent statistics
const service = 'default'
await this.storage!.saveStatistics({
nounCount: {[service]: nounCount},
verbCount: {[service]: verbCount},
metadataCount: {[service]: metadataCount},
nounCount: { [service]: nounCount },
verbCount: { [service]: verbCount },
metadataCount: { [service]: metadataCount },
hnswIndexSize,
lastUpdated: new Date().toISOString()
})
@ -2583,6 +2704,32 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
*/
public setReadOnly(readOnly: boolean): void {
this.readOnly = readOnly
// Ensure readOnly and writeOnly are not both true
if (readOnly && this.writeOnly) {
this.writeOnly = false
}
}
/**
* Check if the database is in write-only mode
* @returns True if the database is in write-only mode, false otherwise
*/
public isWriteOnly(): boolean {
return this.writeOnly
}
/**
* Set the database to write-only mode
* @param writeOnly True to set the database to write-only mode, false to allow searches
*/
public setWriteOnly(writeOnly: boolean): void {
this.writeOnly = writeOnly
// Ensure readOnly and writeOnly are not both true
if (writeOnly && this.readOnly) {
this.readOnly = false
}
}
/**
@ -2621,6 +2768,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<Array<GraphVerb & { similarity: number }>> {
await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
try {
let queryVector: Vector
@ -2677,7 +2827,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// If we didn't get enough results from the index, fall back to the old method
if (verbResults.length < k) {
console.warn('Not enough verb results from HNSW index, falling back to manual search')
console.warn(
'Not enough verb results from HNSW index, falling back to manual search'
)
// Get verbs to search through
let verbs: GraphVerb[] = []
@ -2700,9 +2852,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Calculate similarity for each verb not already in results
const existingIds = new Set(verbResults.map(v => v.id))
const existingIds = new Set(verbResults.map((v) => v.id))
for (const verb of verbs) {
if (!existingIds.has(verb.id) && verb.vector && verb.vector.length > 0) {
if (
!existingIds.has(verb.id) &&
verb.vector &&
verb.vector.length > 0
) {
const distance = this.index.getDistanceFunction()(
queryVector,
verb.vector
@ -2744,13 +2900,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
try {
// First, search for nouns
const nounResults = await this.searchByNounTypes(
queryVectorOrData,
k * 2, // Get more results initially to account for filtering
null,
{forceEmbed: options.forceEmbed}
{ forceEmbed: options.forceEmbed }
)
// If no verb types specified, return the noun results directly
@ -2864,6 +3023,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
try {
// Embed the query text
const queryVector = await this.embed(query)
@ -2900,6 +3062,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
// Check if connected to a remote server
if (!this.isConnectedToRemoteServer()) {
throw new Error(
@ -2962,6 +3127,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
// Check if connected to a remote server
if (!this.isConnectedToRemoteServer()) {
// If not connected to a remote server, just search locally
@ -2989,7 +3157,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const remoteResults = await this.searchRemote(
queryVectorOrData,
k - localResults.length,
{...options, storeResults: true}
{ ...options, storeResults: true }
)
// Combine results, removing duplicates
@ -3131,7 +3299,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
type: 'any',
used: 0,
quota: null,
details: {error: 'Storage not initialized'}
details: { error: 'Storage not initialized' }
}
}
@ -3221,7 +3389,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try {
await this.flushStatistics()
} catch (statsError) {
console.warn('Failed to flush statistics during shutdown:', statsError)
console.warn(
'Failed to flush statistics during shutdown:',
statsError
)
// Continue with shutdown even if statistics flush fails
}
}
@ -3428,7 +3599,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Add the noun with its vector and metadata
await this.add(noun.vector, noun.metadata, {id: noun.id})
await this.add(noun.vector, noun.metadata, { id: noun.id })
nounsRestored++
} catch (error) {
console.error(`Failed to restore noun ${noun.id}:`, error)
@ -3486,7 +3657,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Re-add all nouns to the index
for (const noun of data.nouns) {
if (noun.vector && noun.vector.length > 0) {
await this.index.addItem({id: noun.id, vector: noun.vector})
await this.index.addItem({ id: noun.id, vector: noun.vector })
}
}
@ -3547,14 +3718,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const nounIds: string[] = []
const nounDescriptions: Record<string, string> = {
[NounType.Person]: 'A person with unique characteristics',
[NounType.Place]: 'A location with specific attributes',
[NounType.Location]: 'A location with specific attributes',
[NounType.Thing]: 'An object with distinct properties',
[NounType.Event]: 'An occurrence with temporal aspects',
[NounType.Concept]: 'An abstract idea or notion',
[NounType.Content]: 'A piece of content or information',
[NounType.Group]: 'A collection of related entities',
[NounType.List]: 'An ordered sequence of items',
[NounType.Category]: 'A classification or grouping'
[NounType.Collection]: 'A collection of related entities',
[NounType.Organization]: 'An organization or institution',
[NounType.Document]: 'A document or text-based file'
}
for (let i = 0; i < nounCount; i++) {
@ -3585,10 +3756,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const verbIds: string[] = []
const verbDescriptions: Record<string, string> = {
[VerbType.AttributedTo]: 'Attribution relationship',
[VerbType.Controls]: 'Control relationship',
[VerbType.Created]: 'Creation relationship',
[VerbType.Earned]: 'Achievement relationship',
[VerbType.Owns]: 'Ownership relationship',
[VerbType.Creates]: 'Creation relationship',
[VerbType.Uses]: 'Utilization relationship',
[VerbType.BelongsTo]: 'Belonging relationship',
[VerbType.MemberOf]: 'Membership relationship',
[VerbType.RelatedTo]: 'General relationship',
[VerbType.WorksWith]: 'Collaboration relationship',