**feat: add write-only mode support and example usage**

- Introduced `writeOnly` mode in `BrainyData` allowing optimized data ingestion by skipping index loading and disabling search operations.
- Enhanced `BrainyDataConfig` to include `writeOnly` support and validate compatibility with `readOnly` mode.
- Implemented error handling for search attempts during `writeOnly` mode.
- Updated the README with detailed usage examples for database modes, including `writeOnly` and `readOnly`.
- Added `examples/write-only-mode.js` to demonstrate practical applications of the `writeOnly` mode.

**Purpose**: Optimize memory usage and startup time for data ingestion scenarios by enabling write-only mode with comprehensive documentation and examples.
This commit is contained in:
David Snelling 2025-07-30 13:32:30 -07:00
parent d70d0946cf
commit 5d82db12e2
3 changed files with 3696 additions and 3426 deletions

View file

@ -598,6 +598,37 @@ await db.deleteVerb(verbId)
## Advanced Configuration ## Advanced Configuration
### Database Modes
Brainy supports special operational modes that restrict certain operations:
```typescript
import { BrainyData } from '@soulcraft/brainy'
// Create and initialize the database
const db = new BrainyData()
await db.init()
// Set the database to read-only mode (prevents write operations)
db.setReadOnly(true)
// Check if the database is in read-only mode
const isReadOnly = db.isReadOnly() // Returns true
// Set the database to write-only mode (prevents search operations)
db.setWriteOnly(true)
// Check if the database is in write-only mode
const isWriteOnly = db.isWriteOnly() // Returns true
// Reset to normal mode (allows both read and write operations)
db.setReadOnly(false)
db.setWriteOnly(false)
```
- **Read-Only Mode**: When enabled, prevents all write operations (add, update, delete). Useful for deployment scenarios where you want to prevent modifications to the database.
- **Write-Only Mode**: When enabled, prevents all search operations. Useful for initial data loading or when you want to optimize for write performance.
### Embedding ### Embedding
```typescript ```typescript

View file

@ -0,0 +1,68 @@
/**
* Example demonstrating the write-only mode in BrainyData
*
* This mode is useful for data ingestion scenarios where you only need to insert data
* and don't need to perform any search operations. It significantly reduces memory usage
* by not loading the index into memory.
*/
import { BrainyData } from '../dist/index.js'
// eslint-disable-next-line no-undef
const console = globalThis.console
async function demonstrateWriteOnlyMode() {
console.log('Demonstrating write-only mode in BrainyData')
// Create a BrainyData instance in write-only mode
const db = new BrainyData({
writeOnly: true,
storage: {
// Use file system storage for persistence
forceFileSystemStorage: true
}
})
// Initialize the database
// This will skip loading the index into memory, saving memory and startup time
await db.init()
console.log('Database initialized in write-only mode')
console.log('Is write-only:', db.isWriteOnly()) // Should print true
// Add some data
// This works normally even in write-only mode
const id1 = await db.add('This is a test document', { title: 'Test Document 1' })
const id2 = await db.add('Another test document with different content', { title: 'Test Document 2' })
console.log(`Added documents with IDs: ${id1}, ${id2}`)
// Try to search (this will throw an error in write-only mode)
try {
await db.search('test', 5)
} catch (error) {
console.log('Search error (expected):', error.message)
}
// Switch to normal mode to perform searches
console.log('Switching to normal mode...')
db.setWriteOnly(false)
// Re-initialize to load the index
await db.init()
console.log('Database reinitialized in normal mode')
console.log('Is write-only:', db.isWriteOnly()) // Should print false
// Now search works
const results = await db.search('test', 5)
console.log('Search results:', results.map(r => ({ id: r.id, score: r.score, title: r.metadata?.title })))
// Clean up
await db.shutDown()
}
// Run the example
demonstrateWriteOnlyMode().catch(error => {
console.error('Error in write-only mode example:', error)
})

View file

@ -35,7 +35,11 @@ import {
ServerSearchConduitAugmentation, ServerSearchConduitAugmentation,
createServerSearchAugmentations createServerSearchAugmentations
} from './augmentations/serverSearchAugmentations.js' } from './augmentations/serverSearchAugmentations.js'
import {WebSocketConnection, AugmentationType, IAugmentation} from './types/augmentations.js' import {
WebSocketConnection,
AugmentationType,
IAugmentation
} from './types/augmentations.js'
import { BrainyDataInterface } from './types/brainyDataInterface.js' import { BrainyDataInterface } from './types/brainyDataInterface.js'
import { augmentationPipeline } from './augmentationPipeline.js' import { augmentationPipeline } from './augmentationPipeline.js'
@ -103,6 +107,13 @@ export interface BrainyDataConfig {
*/ */
readOnly?: boolean readOnly?: boolean
/**
* Set the database to write-only mode
* When true, the index is not loaded into memory and search operations will throw an error
* This is useful for data ingestion scenarios where only write operations are needed
*/
writeOnly?: boolean
/** /**
* Remote server configuration for search operations * Remote server configuration for search operations
*/ */
@ -230,6 +241,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private distanceFunction: DistanceFunction private distanceFunction: DistanceFunction
private requestPersistentStorage: boolean private requestPersistentStorage: boolean
private readOnly: boolean private readOnly: boolean
private writeOnly: boolean
private storageConfig: BrainyDataConfig['storage'] = {} private storageConfig: BrainyDataConfig['storage'] = {}
private useOptimizedIndex: boolean = false private useOptimizedIndex: boolean = false
private _dimensions: number private _dimensions: number
@ -240,7 +252,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private retryConfig: BrainyDataConfig['retryPolicy'] = {} private retryConfig: BrainyDataConfig['retryPolicy'] = {}
// Real-time update properties // Real-time update properties
private realtimeUpdateConfig: Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> = { private realtimeUpdateConfig: Required<
NonNullable<BrainyDataConfig['realtimeUpdates']>
> = {
enabled: false, enabled: false,
interval: 30000, // 30 seconds interval: 30000, // 30 seconds
updateStatistics: true, updateStatistics: true,
@ -323,6 +337,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Set read-only flag // Set read-only flag
this.readOnly = config.readOnly || false this.readOnly = config.readOnly || false
// Set write-only flag
this.writeOnly = config.writeOnly || false
// Validate that readOnly and writeOnly are not both true
if (this.readOnly && this.writeOnly) {
throw new Error('Database cannot be both read-only and write-only')
}
// Store storage configuration for later use in init() // Store storage configuration for later use in init()
this.storageConfig = config.storage || {} this.storageConfig = config.storage || {}
@ -356,6 +378,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
} }
/**
* Check if the database is in write-only mode and throw an error if it is
* @throws Error if the database is in write-only mode
*/
private checkWriteOnly(): void {
if (this.writeOnly) {
throw new Error(
'Cannot perform search operation: database is in write-only mode'
)
}
}
/** /**
* Start real-time updates if enabled in the configuration * Start real-time updates if enabled in the configuration
* This will periodically check for new data in storage and update the in-memory index and statistics * This will periodically check for new data in storage and update the in-memory index and statistics
@ -372,21 +406,28 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
// Set the initial last known noun count // Set the initial last known noun count
this.getNounCount().then(count => { this.getNounCount()
.then((count) => {
this.lastKnownNounCount = count this.lastKnownNounCount = count
}).catch(error => { })
console.warn('Failed to get initial noun count for real-time updates:', error) .catch((error) => {
console.warn(
'Failed to get initial noun count for real-time updates:',
error
)
}) })
// Start the update timer // Start the update timer
this.updateTimerId = setInterval(() => { this.updateTimerId = setInterval(() => {
this.checkForUpdates().catch(error => { this.checkForUpdates().catch((error) => {
console.warn('Error during real-time update check:', error) console.warn('Error during real-time update check:', error)
}) })
}, this.realtimeUpdateConfig.interval) }, this.realtimeUpdateConfig.interval)
if (this.loggingConfig?.verbose) { if (this.loggingConfig?.verbose) {
console.log(`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`) console.log(
`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`
)
} }
} }
@ -421,7 +462,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* Enable real-time updates with the specified configuration * Enable real-time updates with the specified configuration
* @param config Configuration for real-time updates * @param config Configuration for real-time updates
*/ */
public enableRealtimeUpdates(config?: Partial<BrainyDataConfig['realtimeUpdates']>): void { public enableRealtimeUpdates(
config?: Partial<BrainyDataConfig['realtimeUpdates']>
): void {
// Update configuration if provided // Update configuration if provided
if (config) { if (config) {
this.realtimeUpdateConfig = { this.realtimeUpdateConfig = {
@ -454,7 +497,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* Get the current real-time update configuration * Get the current real-time update configuration
* @returns The current real-time update configuration * @returns The current real-time update configuration
*/ */
public getRealtimeUpdateConfig(): Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> { public getRealtimeUpdateConfig(): Required<
NonNullable<BrainyDataConfig['realtimeUpdates']>
> {
return { ...this.realtimeUpdateConfig } return { ...this.realtimeUpdateConfig }
} }
@ -514,7 +559,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try { try {
// Get changes since the last update // Get changes since the last update
const changes = await this.storage.getChangesSince(this.lastUpdateTime, 1000) // Limit to 1000 changes per batch const changes = await this.storage.getChangesSince(
this.lastUpdateTime,
1000
) // Limit to 1000 changes per batch
let addedCount = 0 let addedCount = 0
let updatedCount = 0 let updatedCount = 0
@ -549,7 +597,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
if (this.loggingConfig?.verbose) { if (this.loggingConfig?.verbose) {
console.log(`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`) console.log(
`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`
)
} }
} }
break break
@ -561,26 +611,38 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
deletedCount++ deletedCount++
if (this.loggingConfig?.verbose) { if (this.loggingConfig?.verbose) {
console.log(`Removed noun ${change.entityId} from index during real-time update`) console.log(
`Removed noun ${change.entityId} from index during real-time update`
)
} }
} }
break break
} }
} catch (changeError) { } catch (changeError) {
console.error(`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`, changeError) console.error(
`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`,
changeError
)
// Continue with other changes // Continue with other changes
} }
} }
if (this.loggingConfig?.verbose && (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) { if (
console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`) this.loggingConfig?.verbose &&
(addedCount > 0 || updatedCount > 0 || deletedCount > 0)
) {
console.log(
`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`
)
} }
// Update the last known noun count // Update the last known noun count
this.lastKnownNounCount = await this.getNounCount() this.lastKnownNounCount = await this.getNounCount()
} catch (error) { } catch (error) {
console.error('Failed to apply changes from log, falling back to full scan:', error) console.error(
'Failed to apply changes from log, falling back to full scan:',
error
)
// Fallback to full scan if change log fails // Fallback to full scan if change log fails
await this.applyChangesFromFullScan() await this.applyChangesFromFullScan()
} }
@ -604,7 +666,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const indexNounIds = new Set(indexNouns.keys()) const indexNounIds = new Set(indexNouns.keys())
// Find nouns that are in storage but not in the index // Find nouns that are in storage but not in the index
const newNouns = nouns.filter(noun => !indexNounIds.has(noun.id)) const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id))
// Add new nouns to the index // Add new nouns to the index
for (const noun of newNouns) { for (const noun of newNouns) {
@ -623,7 +685,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}) })
if (this.loggingConfig?.verbose) { if (this.loggingConfig?.verbose) {
console.log(`Added new noun ${noun.id} to index during real-time update`) console.log(
`Added new noun ${noun.id} to index during real-time update`
)
} }
} }
@ -631,7 +695,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.lastKnownNounCount = currentCount this.lastKnownNounCount = currentCount
if (this.loggingConfig?.verbose && newNouns.length > 0) { if (this.loggingConfig?.verbose && newNouns.length > 0) {
console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`) console.log(
`Real-time update: Added ${newNouns.length} new nouns to index using full scan`
)
} }
} }
} catch (error) { } catch (error) {
@ -648,7 +714,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private getCurrentAugmentation(): string { private getCurrentAugmentation(): string {
try { try {
// Get all registered augmentations // Get all registered augmentations
const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes() const augmentationTypes =
augmentationPipeline.getAvailableAugmentationTypes()
// Check each type of augmentation // Check each type of augmentation
for (const type of augmentationTypes) { for (const type of augmentationTypes) {
@ -742,15 +809,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Ensure s3Storage has all required fields if it's provided // Ensure s3Storage has all required fields if it's provided
if (storageOptions.s3Storage) { if (storageOptions.s3Storage) {
// Only include s3Storage if all required fields are present // Only include s3Storage if all required fields are present
if (storageOptions.s3Storage.bucketName && if (
storageOptions.s3Storage.bucketName &&
storageOptions.s3Storage.accessKeyId && storageOptions.s3Storage.accessKeyId &&
storageOptions.s3Storage.secretAccessKey) { storageOptions.s3Storage.secretAccessKey
) {
// All required fields are present, keep s3Storage as is // All required fields are present, keep s3Storage as is
} else { } else {
// Missing required fields, remove s3Storage to avoid type errors // Missing required fields, remove s3Storage to avoid type errors
const { s3Storage, ...rest } = storageOptions const { s3Storage, ...rest } = storageOptions
storageOptions = rest storageOptions = rest
console.warn('Ignoring s3Storage configuration due to missing required fields') console.warn(
'Ignoring s3Storage configuration due to missing required fields'
)
} }
} }
@ -766,6 +837,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.index.setStorage(this.storage!) this.index.setStorage(this.storage!)
} }
// In write-only mode, skip loading the index into memory
if (this.writeOnly) {
if (this.loggingConfig?.verbose) {
console.log('Database is in write-only mode, skipping index loading')
}
} else {
// Load all nouns from storage // Load all nouns from storage
const nouns: HNSWNoun[] = await this.storage!.getAllNouns() const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
@ -788,6 +865,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
vector: noun.vector vector: noun.vector
}) })
} }
}
// Connect to remote server if configured with autoConnect // Connect to remote server if configured with autoConnect
if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) { if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) {
@ -888,10 +966,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
// Check if input is already a vector // Check if input is already a vector
if ( if (Array.isArray(vectorOrData) && !options.forceEmbed) {
Array.isArray(vectorOrData) &&
!options.forceEmbed
) {
// Input is already a vector (and we've validated it contains only numbers) // Input is already a vector (and we've validated it contains only numbers)
vector = vectorOrData vector = vectorOrData
} else { } else {
@ -910,7 +985,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Validate vector dimensions // Validate vector dimensions
if (vector.length !== this._dimensions) { if (vector.length !== this._dimensions) {
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`) throw new Error(
`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`
)
} }
// Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID // Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID
@ -940,7 +1017,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Save metadata if provided and not empty // Save metadata if provided and not empty
if (metadata !== undefined) { if (metadata !== undefined) {
// Skip saving if metadata is an empty object // Skip saving if metadata is an empty object
if (metadata && typeof metadata === 'object' && Object.keys(metadata).length === 0) { if (
metadata &&
typeof metadata === 'object' &&
Object.keys(metadata).length === 0
) {
// Don't save empty metadata // Don't save empty metadata
// Explicitly save null to ensure no metadata is stored // Explicitly save null to ensure no metadata is stored
await this.storage!.saveMetadata(id, null) await this.storage!.saveMetadata(id, null)
@ -998,7 +1079,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
await this.storage!.saveMetadata(id, metadataToSave) await this.storage!.saveMetadata(id, metadataToSave)
// Track metadata statistics // Track metadata statistics
const metadataService = options.service || this.getCurrentAugmentation() const metadataService =
options.service || this.getCurrentAugmentation()
await this.storage!.incrementStatistic('metadata', metadataService) await this.storage!.incrementStatistic('metadata', metadataService)
} }
} }
@ -1274,7 +1356,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): R[] { ): R[] {
if (!service) return results if (!service) return results
return results.filter(result => { return results.filter((result) => {
if (!result.metadata || typeof result.metadata !== 'object') return false if (!result.metadata || typeof result.metadata !== 'object') return false
if (!('createdBy' in result.metadata)) return false if (!('createdBy' in result.metadata)) return false
@ -1316,9 +1398,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
return createdBy.augmentation === options.service return createdBy.augmentation === options.service
} }
if (!this.isInitialized) { if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.') throw new Error(
'BrainyData must be initialized before searching. Call init() first.'
)
} }
// Check if database is in write-only mode
this.checkWriteOnly()
try { try {
let queryVector: Vector let queryVector: Vector
@ -1346,7 +1433,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if query vector dimensions match the expected dimensions // Check if query vector dimensions match the expected dimensions
if (queryVector.length !== this._dimensions) { if (queryVector.length !== this._dimensions) {
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`) throw new Error(
`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`
)
} }
// If no noun types specified, search all nouns // If no noun types specified, search all nouns
@ -1485,8 +1574,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
if (!this.isInitialized) { if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.') throw new Error(
'BrainyData must be initialized before searching. Call init() first.'
)
} }
// Check if database is in write-only mode
this.checkWriteOnly()
// If searching for verbs directly // If searching for verbs directly
if (options.searchVerbs) { if (options.searchVerbs) {
const verbResults = await this.searchVerbs(queryVectorOrData, k, { const verbResults = await this.searchVerbs(queryVectorOrData, k, {
@ -1548,8 +1642,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} = {} } = {}
): Promise<SearchResult<T>[]> { ): Promise<SearchResult<T>[]> {
if (!this.isInitialized) { if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.') throw new Error(
'BrainyData must be initialized before searching. Call init() first.'
)
} }
// Check if database is in write-only mode
this.checkWriteOnly()
// If input is a string and not a vector, automatically vectorize it // If input is a string and not a vector, automatically vectorize it
let queryToUse = queryVectorOrData let queryToUse = queryVectorOrData
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) { if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
@ -1636,10 +1735,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const outgoingVerbs = await this.storage!.getVerbsBySource(id) const outgoingVerbs = await this.storage!.getVerbsBySource(id)
// Filter to only include verbs of the specified type // Filter to only include verbs of the specified type
const verbsOfType = outgoingVerbs.filter(verb => verb.type === options.relationType) const verbsOfType = outgoingVerbs.filter(
(verb) => verb.type === options.relationType
)
// Get the target IDs // Get the target IDs
const targetIds = verbsOfType.map(verb => verb.target) const targetIds = verbsOfType.map((verb) => verb.target)
// Get the actual entities for these IDs // Get the actual entities for these IDs
const results: SearchResult<T>[] = [] const results: SearchResult<T>[] = []
@ -1858,11 +1959,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const graphNoun = metadata as unknown as GraphNoun const graphNoun = metadata as unknown as GraphNoun
// Preserve existing createdBy and createdAt if they exist // Preserve existing createdBy and createdAt if they exist
const existingMetadata = await this.storage!.getMetadata(id) as any const existingMetadata = (await this.storage!.getMetadata(id)) as any
if (existingMetadata && if (
existingMetadata &&
typeof existingMetadata === 'object' && typeof existingMetadata === 'object' &&
'createdBy' in existingMetadata) { 'createdBy' in existingMetadata
) {
// Preserve the original creator information // Preserve the original creator information
graphNoun.createdBy = existingMetadata.createdBy graphNoun.createdBy = existingMetadata.createdBy
@ -2035,8 +2138,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
console.warn(`Auto-created missing source noun with ID ${sourceId}`) console.warn(`Auto-created missing source noun with ID ${sourceId}`)
} catch (createError) { } catch (createError) {
console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError) console.error(
throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`) `Failed to auto-create source noun with ID ${sourceId}:`,
createError
)
throw new Error(
`Failed to auto-create source noun with ID ${sourceId}: ${createError}`
)
} }
} }
@ -2072,8 +2180,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
console.warn(`Auto-created missing target noun with ID ${targetId}`) console.warn(`Auto-created missing target noun with ID ${targetId}`)
} catch (createError) { } catch (createError) {
console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError) console.error(
throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`) `Failed to auto-create target noun with ID ${targetId}:`,
createError
)
throw new Error(
`Failed to auto-create target noun with ID ${targetId}: ${createError}`
)
} }
} }
@ -2358,7 +2471,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const allVerbs = await this.storage!.getAllVerbs() const allVerbs = await this.storage!.getAllVerbs()
// Create a set of verb IDs for faster lookup // Create a set of verb IDs for faster lookup
const verbIds = new Set(allVerbs.map(verb => verb.id)) const verbIds = new Set(allVerbs.map((verb) => verb.id))
// Get all nouns from the index // Get all nouns from the index
const nouns = this.index.getNouns() const nouns = this.index.getNouns()
@ -2395,10 +2508,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* @param options Additional options for retrieving statistics * @param options Additional options for retrieving statistics
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size * @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
*/ */
public async getStatistics(options: { public async getStatistics(
options: {
service?: string | string[] // Filter statistics by service(s) service?: string | string[] // Filter statistics by service(s)
forceRefresh?: boolean // Force a refresh of statistics from storage forceRefresh?: boolean // Force a refresh of statistics from storage
} = {}): Promise<{ } = {}
): Promise<{
nounCount: number nounCount: number
verbCount: number verbCount: number
metadataCount: number metadataCount: number
@ -2463,8 +2578,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Filter by service if specified // Filter by service if specified
const services = options.service const services = options.service
? (Array.isArray(options.service) ? options.service : [options.service]) ? Array.isArray(options.service)
: Object.keys({...stats.nounCount, ...stats.verbCount, ...stats.metadataCount}) ? options.service
: [options.service]
: Object.keys({
...stats.nounCount,
...stats.verbCount,
...stats.metadataCount
})
// Calculate totals and service breakdown // Calculate totals and service breakdown
for (const service of services) { for (const service of services) {
@ -2583,6 +2704,32 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
*/ */
public setReadOnly(readOnly: boolean): void { public setReadOnly(readOnly: boolean): void {
this.readOnly = readOnly this.readOnly = readOnly
// Ensure readOnly and writeOnly are not both true
if (readOnly && this.writeOnly) {
this.writeOnly = false
}
}
/**
* Check if the database is in write-only mode
* @returns True if the database is in write-only mode, false otherwise
*/
public isWriteOnly(): boolean {
return this.writeOnly
}
/**
* Set the database to write-only mode
* @param writeOnly True to set the database to write-only mode, false to allow searches
*/
public setWriteOnly(writeOnly: boolean): void {
this.writeOnly = writeOnly
// Ensure readOnly and writeOnly are not both true
if (writeOnly && this.readOnly) {
this.readOnly = false
}
} }
/** /**
@ -2621,6 +2768,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<Array<GraphVerb & { similarity: number }>> { ): Promise<Array<GraphVerb & { similarity: number }>> {
await this.ensureInitialized() await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
try { try {
let queryVector: Vector let queryVector: Vector
@ -2677,7 +2827,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// If we didn't get enough results from the index, fall back to the old method // If we didn't get enough results from the index, fall back to the old method
if (verbResults.length < k) { if (verbResults.length < k) {
console.warn('Not enough verb results from HNSW index, falling back to manual search') console.warn(
'Not enough verb results from HNSW index, falling back to manual search'
)
// Get verbs to search through // Get verbs to search through
let verbs: GraphVerb[] = [] let verbs: GraphVerb[] = []
@ -2700,9 +2852,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
// Calculate similarity for each verb not already in results // Calculate similarity for each verb not already in results
const existingIds = new Set(verbResults.map(v => v.id)) const existingIds = new Set(verbResults.map((v) => v.id))
for (const verb of verbs) { for (const verb of verbs) {
if (!existingIds.has(verb.id) && verb.vector && verb.vector.length > 0) { if (
!existingIds.has(verb.id) &&
verb.vector &&
verb.vector.length > 0
) {
const distance = this.index.getDistanceFunction()( const distance = this.index.getDistanceFunction()(
queryVector, queryVector,
verb.vector verb.vector
@ -2744,6 +2900,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> { ): Promise<SearchResult<T>[]> {
await this.ensureInitialized() await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
try { try {
// First, search for nouns // First, search for nouns
const nounResults = await this.searchByNounTypes( const nounResults = await this.searchByNounTypes(
@ -2864,6 +3023,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> { ): Promise<SearchResult<T>[]> {
await this.ensureInitialized() await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
try { try {
// Embed the query text // Embed the query text
const queryVector = await this.embed(query) const queryVector = await this.embed(query)
@ -2900,6 +3062,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> { ): Promise<SearchResult<T>[]> {
await this.ensureInitialized() await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
// Check if connected to a remote server // Check if connected to a remote server
if (!this.isConnectedToRemoteServer()) { if (!this.isConnectedToRemoteServer()) {
throw new Error( throw new Error(
@ -2962,6 +3127,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
): Promise<SearchResult<T>[]> { ): Promise<SearchResult<T>[]> {
await this.ensureInitialized() await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
// Check if connected to a remote server // Check if connected to a remote server
if (!this.isConnectedToRemoteServer()) { if (!this.isConnectedToRemoteServer()) {
// If not connected to a remote server, just search locally // If not connected to a remote server, just search locally
@ -3221,7 +3389,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
try { try {
await this.flushStatistics() await this.flushStatistics()
} catch (statsError) { } catch (statsError) {
console.warn('Failed to flush statistics during shutdown:', statsError) console.warn(
'Failed to flush statistics during shutdown:',
statsError
)
// Continue with shutdown even if statistics flush fails // Continue with shutdown even if statistics flush fails
} }
} }
@ -3547,14 +3718,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const nounIds: string[] = [] const nounIds: string[] = []
const nounDescriptions: Record<string, string> = { const nounDescriptions: Record<string, string> = {
[NounType.Person]: 'A person with unique characteristics', [NounType.Person]: 'A person with unique characteristics',
[NounType.Place]: 'A location with specific attributes', [NounType.Location]: 'A location with specific attributes',
[NounType.Thing]: 'An object with distinct properties', [NounType.Thing]: 'An object with distinct properties',
[NounType.Event]: 'An occurrence with temporal aspects', [NounType.Event]: 'An occurrence with temporal aspects',
[NounType.Concept]: 'An abstract idea or notion', [NounType.Concept]: 'An abstract idea or notion',
[NounType.Content]: 'A piece of content or information', [NounType.Content]: 'A piece of content or information',
[NounType.Group]: 'A collection of related entities', [NounType.Collection]: 'A collection of related entities',
[NounType.List]: 'An ordered sequence of items', [NounType.Organization]: 'An organization or institution',
[NounType.Category]: 'A classification or grouping' [NounType.Document]: 'A document or text-based file'
} }
for (let i = 0; i < nounCount; i++) { for (let i = 0; i < nounCount; i++) {
@ -3585,10 +3756,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const verbIds: string[] = [] const verbIds: string[] = []
const verbDescriptions: Record<string, string> = { const verbDescriptions: Record<string, string> = {
[VerbType.AttributedTo]: 'Attribution relationship', [VerbType.AttributedTo]: 'Attribution relationship',
[VerbType.Controls]: 'Control relationship',
[VerbType.Created]: 'Creation relationship',
[VerbType.Earned]: 'Achievement relationship',
[VerbType.Owns]: 'Ownership relationship', [VerbType.Owns]: 'Ownership relationship',
[VerbType.Creates]: 'Creation relationship',
[VerbType.Uses]: 'Utilization relationship',
[VerbType.BelongsTo]: 'Belonging relationship',
[VerbType.MemberOf]: 'Membership relationship', [VerbType.MemberOf]: 'Membership relationship',
[VerbType.RelatedTo]: 'General relationship', [VerbType.RelatedTo]: 'General relationship',
[VerbType.WorksWith]: 'Collaboration relationship', [VerbType.WorksWith]: 'Collaboration relationship',