**feat: add write-only mode support and example usage**
- Introduced `writeOnly` mode in `BrainyData` allowing optimized data ingestion by skipping index loading and disabling search operations. - Enhanced `BrainyDataConfig` to include `writeOnly` support and validate compatibility with `readOnly` mode. - Implemented error handling for search attempts during `writeOnly` mode. - Updated the README with detailed usage examples for database modes, including `writeOnly` and `readOnly`. - Added `examples/write-only-mode.js` to demonstrate practical applications of the `writeOnly` mode. **Purpose**: Optimize memory usage and startup time for data ingestion scenarios by enabling write-only mode with comprehensive documentation and examples.
This commit is contained in:
parent
d70d0946cf
commit
5d82db12e2
3 changed files with 3696 additions and 3426 deletions
31
README.md
31
README.md
|
|
@ -598,6 +598,37 @@ await db.deleteVerb(verbId)
|
||||||
|
|
||||||
## Advanced Configuration
|
## Advanced Configuration
|
||||||
|
|
||||||
|
### Database Modes
|
||||||
|
|
||||||
|
Brainy supports special operational modes that restrict certain operations:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { BrainyData } from '@soulcraft/brainy'
|
||||||
|
|
||||||
|
// Create and initialize the database
|
||||||
|
const db = new BrainyData()
|
||||||
|
await db.init()
|
||||||
|
|
||||||
|
// Set the database to read-only mode (prevents write operations)
|
||||||
|
db.setReadOnly(true)
|
||||||
|
|
||||||
|
// Check if the database is in read-only mode
|
||||||
|
const isReadOnly = db.isReadOnly() // Returns true
|
||||||
|
|
||||||
|
// Set the database to write-only mode (prevents search operations)
|
||||||
|
db.setWriteOnly(true)
|
||||||
|
|
||||||
|
// Check if the database is in write-only mode
|
||||||
|
const isWriteOnly = db.isWriteOnly() // Returns true
|
||||||
|
|
||||||
|
// Reset to normal mode (allows both read and write operations)
|
||||||
|
db.setReadOnly(false)
|
||||||
|
db.setWriteOnly(false)
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Read-Only Mode**: When enabled, prevents all write operations (add, update, delete). Useful for deployment scenarios where you want to prevent modifications to the database.
|
||||||
|
- **Write-Only Mode**: When enabled, prevents all search operations. Useful for initial data loading or when you want to optimize for write performance.
|
||||||
|
|
||||||
### Embedding
|
### Embedding
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
|
|
|
||||||
68
examples/write-only-mode.js
Normal file
68
examples/write-only-mode.js
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
/**
|
||||||
|
* Example demonstrating the write-only mode in BrainyData
|
||||||
|
*
|
||||||
|
* This mode is useful for data ingestion scenarios where you only need to insert data
|
||||||
|
* and don't need to perform any search operations. It significantly reduces memory usage
|
||||||
|
* by not loading the index into memory.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { BrainyData } from '../dist/index.js'
|
||||||
|
|
||||||
|
// eslint-disable-next-line no-undef
|
||||||
|
const console = globalThis.console
|
||||||
|
|
||||||
|
async function demonstrateWriteOnlyMode() {
|
||||||
|
console.log('Demonstrating write-only mode in BrainyData')
|
||||||
|
|
||||||
|
// Create a BrainyData instance in write-only mode
|
||||||
|
const db = new BrainyData({
|
||||||
|
writeOnly: true,
|
||||||
|
storage: {
|
||||||
|
// Use file system storage for persistence
|
||||||
|
forceFileSystemStorage: true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Initialize the database
|
||||||
|
// This will skip loading the index into memory, saving memory and startup time
|
||||||
|
await db.init()
|
||||||
|
|
||||||
|
console.log('Database initialized in write-only mode')
|
||||||
|
console.log('Is write-only:', db.isWriteOnly()) // Should print true
|
||||||
|
|
||||||
|
// Add some data
|
||||||
|
// This works normally even in write-only mode
|
||||||
|
const id1 = await db.add('This is a test document', { title: 'Test Document 1' })
|
||||||
|
const id2 = await db.add('Another test document with different content', { title: 'Test Document 2' })
|
||||||
|
|
||||||
|
console.log(`Added documents with IDs: ${id1}, ${id2}`)
|
||||||
|
|
||||||
|
// Try to search (this will throw an error in write-only mode)
|
||||||
|
try {
|
||||||
|
await db.search('test', 5)
|
||||||
|
} catch (error) {
|
||||||
|
console.log('Search error (expected):', error.message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Switch to normal mode to perform searches
|
||||||
|
console.log('Switching to normal mode...')
|
||||||
|
db.setWriteOnly(false)
|
||||||
|
|
||||||
|
// Re-initialize to load the index
|
||||||
|
await db.init()
|
||||||
|
|
||||||
|
console.log('Database reinitialized in normal mode')
|
||||||
|
console.log('Is write-only:', db.isWriteOnly()) // Should print false
|
||||||
|
|
||||||
|
// Now search works
|
||||||
|
const results = await db.search('test', 5)
|
||||||
|
console.log('Search results:', results.map(r => ({ id: r.id, score: r.score, title: r.metadata?.title })))
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
await db.shutDown()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the example
|
||||||
|
demonstrateWriteOnlyMode().catch(error => {
|
||||||
|
console.error('Error in write-only mode example:', error)
|
||||||
|
})
|
||||||
|
|
@ -35,7 +35,11 @@ import {
|
||||||
ServerSearchConduitAugmentation,
|
ServerSearchConduitAugmentation,
|
||||||
createServerSearchAugmentations
|
createServerSearchAugmentations
|
||||||
} from './augmentations/serverSearchAugmentations.js'
|
} from './augmentations/serverSearchAugmentations.js'
|
||||||
import {WebSocketConnection, AugmentationType, IAugmentation} from './types/augmentations.js'
|
import {
|
||||||
|
WebSocketConnection,
|
||||||
|
AugmentationType,
|
||||||
|
IAugmentation
|
||||||
|
} from './types/augmentations.js'
|
||||||
import { BrainyDataInterface } from './types/brainyDataInterface.js'
|
import { BrainyDataInterface } from './types/brainyDataInterface.js'
|
||||||
import { augmentationPipeline } from './augmentationPipeline.js'
|
import { augmentationPipeline } from './augmentationPipeline.js'
|
||||||
|
|
||||||
|
|
@ -103,6 +107,13 @@ export interface BrainyDataConfig {
|
||||||
*/
|
*/
|
||||||
readOnly?: boolean
|
readOnly?: boolean
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the database to write-only mode
|
||||||
|
* When true, the index is not loaded into memory and search operations will throw an error
|
||||||
|
* This is useful for data ingestion scenarios where only write operations are needed
|
||||||
|
*/
|
||||||
|
writeOnly?: boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Remote server configuration for search operations
|
* Remote server configuration for search operations
|
||||||
*/
|
*/
|
||||||
|
|
@ -230,6 +241,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
private distanceFunction: DistanceFunction
|
private distanceFunction: DistanceFunction
|
||||||
private requestPersistentStorage: boolean
|
private requestPersistentStorage: boolean
|
||||||
private readOnly: boolean
|
private readOnly: boolean
|
||||||
|
private writeOnly: boolean
|
||||||
private storageConfig: BrainyDataConfig['storage'] = {}
|
private storageConfig: BrainyDataConfig['storage'] = {}
|
||||||
private useOptimizedIndex: boolean = false
|
private useOptimizedIndex: boolean = false
|
||||||
private _dimensions: number
|
private _dimensions: number
|
||||||
|
|
@ -240,7 +252,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
private retryConfig: BrainyDataConfig['retryPolicy'] = {}
|
private retryConfig: BrainyDataConfig['retryPolicy'] = {}
|
||||||
|
|
||||||
// Real-time update properties
|
// Real-time update properties
|
||||||
private realtimeUpdateConfig: Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> = {
|
private realtimeUpdateConfig: Required<
|
||||||
|
NonNullable<BrainyDataConfig['realtimeUpdates']>
|
||||||
|
> = {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
interval: 30000, // 30 seconds
|
interval: 30000, // 30 seconds
|
||||||
updateStatistics: true,
|
updateStatistics: true,
|
||||||
|
|
@ -323,6 +337,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Set read-only flag
|
// Set read-only flag
|
||||||
this.readOnly = config.readOnly || false
|
this.readOnly = config.readOnly || false
|
||||||
|
|
||||||
|
// Set write-only flag
|
||||||
|
this.writeOnly = config.writeOnly || false
|
||||||
|
|
||||||
|
// Validate that readOnly and writeOnly are not both true
|
||||||
|
if (this.readOnly && this.writeOnly) {
|
||||||
|
throw new Error('Database cannot be both read-only and write-only')
|
||||||
|
}
|
||||||
|
|
||||||
// Store storage configuration for later use in init()
|
// Store storage configuration for later use in init()
|
||||||
this.storageConfig = config.storage || {}
|
this.storageConfig = config.storage || {}
|
||||||
|
|
||||||
|
|
@ -356,6 +378,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the database is in write-only mode and throw an error if it is
|
||||||
|
* @throws Error if the database is in write-only mode
|
||||||
|
*/
|
||||||
|
private checkWriteOnly(): void {
|
||||||
|
if (this.writeOnly) {
|
||||||
|
throw new Error(
|
||||||
|
'Cannot perform search operation: database is in write-only mode'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start real-time updates if enabled in the configuration
|
* Start real-time updates if enabled in the configuration
|
||||||
* This will periodically check for new data in storage and update the in-memory index and statistics
|
* This will periodically check for new data in storage and update the in-memory index and statistics
|
||||||
|
|
@ -372,21 +406,28 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set the initial last known noun count
|
// Set the initial last known noun count
|
||||||
this.getNounCount().then(count => {
|
this.getNounCount()
|
||||||
|
.then((count) => {
|
||||||
this.lastKnownNounCount = count
|
this.lastKnownNounCount = count
|
||||||
}).catch(error => {
|
})
|
||||||
console.warn('Failed to get initial noun count for real-time updates:', error)
|
.catch((error) => {
|
||||||
|
console.warn(
|
||||||
|
'Failed to get initial noun count for real-time updates:',
|
||||||
|
error
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
// Start the update timer
|
// Start the update timer
|
||||||
this.updateTimerId = setInterval(() => {
|
this.updateTimerId = setInterval(() => {
|
||||||
this.checkForUpdates().catch(error => {
|
this.checkForUpdates().catch((error) => {
|
||||||
console.warn('Error during real-time update check:', error)
|
console.warn('Error during real-time update check:', error)
|
||||||
})
|
})
|
||||||
}, this.realtimeUpdateConfig.interval)
|
}, this.realtimeUpdateConfig.interval)
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`)
|
console.log(
|
||||||
|
`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -421,7 +462,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
* Enable real-time updates with the specified configuration
|
* Enable real-time updates with the specified configuration
|
||||||
* @param config Configuration for real-time updates
|
* @param config Configuration for real-time updates
|
||||||
*/
|
*/
|
||||||
public enableRealtimeUpdates(config?: Partial<BrainyDataConfig['realtimeUpdates']>): void {
|
public enableRealtimeUpdates(
|
||||||
|
config?: Partial<BrainyDataConfig['realtimeUpdates']>
|
||||||
|
): void {
|
||||||
// Update configuration if provided
|
// Update configuration if provided
|
||||||
if (config) {
|
if (config) {
|
||||||
this.realtimeUpdateConfig = {
|
this.realtimeUpdateConfig = {
|
||||||
|
|
@ -454,7 +497,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
* Get the current real-time update configuration
|
* Get the current real-time update configuration
|
||||||
* @returns The current real-time update configuration
|
* @returns The current real-time update configuration
|
||||||
*/
|
*/
|
||||||
public getRealtimeUpdateConfig(): Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> {
|
public getRealtimeUpdateConfig(): Required<
|
||||||
|
NonNullable<BrainyDataConfig['realtimeUpdates']>
|
||||||
|
> {
|
||||||
return { ...this.realtimeUpdateConfig }
|
return { ...this.realtimeUpdateConfig }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -514,7 +559,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Get changes since the last update
|
// Get changes since the last update
|
||||||
const changes = await this.storage.getChangesSince(this.lastUpdateTime, 1000) // Limit to 1000 changes per batch
|
const changes = await this.storage.getChangesSince(
|
||||||
|
this.lastUpdateTime,
|
||||||
|
1000
|
||||||
|
) // Limit to 1000 changes per batch
|
||||||
|
|
||||||
let addedCount = 0
|
let addedCount = 0
|
||||||
let updatedCount = 0
|
let updatedCount = 0
|
||||||
|
|
@ -549,7 +597,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`)
|
console.log(
|
||||||
|
`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
|
|
@ -561,26 +611,38 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
deletedCount++
|
deletedCount++
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(`Removed noun ${change.entityId} from index during real-time update`)
|
console.log(
|
||||||
|
`Removed noun ${change.entityId} from index during real-time update`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
} catch (changeError) {
|
} catch (changeError) {
|
||||||
console.error(`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`, changeError)
|
console.error(
|
||||||
|
`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`,
|
||||||
|
changeError
|
||||||
|
)
|
||||||
// Continue with other changes
|
// Continue with other changes
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose && (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) {
|
if (
|
||||||
console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`)
|
this.loggingConfig?.verbose &&
|
||||||
|
(addedCount > 0 || updatedCount > 0 || deletedCount > 0)
|
||||||
|
) {
|
||||||
|
console.log(
|
||||||
|
`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update the last known noun count
|
// Update the last known noun count
|
||||||
this.lastKnownNounCount = await this.getNounCount()
|
this.lastKnownNounCount = await this.getNounCount()
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to apply changes from log, falling back to full scan:', error)
|
console.error(
|
||||||
|
'Failed to apply changes from log, falling back to full scan:',
|
||||||
|
error
|
||||||
|
)
|
||||||
// Fallback to full scan if change log fails
|
// Fallback to full scan if change log fails
|
||||||
await this.applyChangesFromFullScan()
|
await this.applyChangesFromFullScan()
|
||||||
}
|
}
|
||||||
|
|
@ -604,7 +666,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const indexNounIds = new Set(indexNouns.keys())
|
const indexNounIds = new Set(indexNouns.keys())
|
||||||
|
|
||||||
// Find nouns that are in storage but not in the index
|
// Find nouns that are in storage but not in the index
|
||||||
const newNouns = nouns.filter(noun => !indexNounIds.has(noun.id))
|
const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id))
|
||||||
|
|
||||||
// Add new nouns to the index
|
// Add new nouns to the index
|
||||||
for (const noun of newNouns) {
|
for (const noun of newNouns) {
|
||||||
|
|
@ -623,7 +685,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
})
|
})
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose) {
|
if (this.loggingConfig?.verbose) {
|
||||||
console.log(`Added new noun ${noun.id} to index during real-time update`)
|
console.log(
|
||||||
|
`Added new noun ${noun.id} to index during real-time update`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -631,7 +695,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
this.lastKnownNounCount = currentCount
|
this.lastKnownNounCount = currentCount
|
||||||
|
|
||||||
if (this.loggingConfig?.verbose && newNouns.length > 0) {
|
if (this.loggingConfig?.verbose && newNouns.length > 0) {
|
||||||
console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`)
|
console.log(
|
||||||
|
`Real-time update: Added ${newNouns.length} new nouns to index using full scan`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -648,7 +714,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
private getCurrentAugmentation(): string {
|
private getCurrentAugmentation(): string {
|
||||||
try {
|
try {
|
||||||
// Get all registered augmentations
|
// Get all registered augmentations
|
||||||
const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes()
|
const augmentationTypes =
|
||||||
|
augmentationPipeline.getAvailableAugmentationTypes()
|
||||||
|
|
||||||
// Check each type of augmentation
|
// Check each type of augmentation
|
||||||
for (const type of augmentationTypes) {
|
for (const type of augmentationTypes) {
|
||||||
|
|
@ -742,15 +809,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Ensure s3Storage has all required fields if it's provided
|
// Ensure s3Storage has all required fields if it's provided
|
||||||
if (storageOptions.s3Storage) {
|
if (storageOptions.s3Storage) {
|
||||||
// Only include s3Storage if all required fields are present
|
// Only include s3Storage if all required fields are present
|
||||||
if (storageOptions.s3Storage.bucketName &&
|
if (
|
||||||
|
storageOptions.s3Storage.bucketName &&
|
||||||
storageOptions.s3Storage.accessKeyId &&
|
storageOptions.s3Storage.accessKeyId &&
|
||||||
storageOptions.s3Storage.secretAccessKey) {
|
storageOptions.s3Storage.secretAccessKey
|
||||||
|
) {
|
||||||
// All required fields are present, keep s3Storage as is
|
// All required fields are present, keep s3Storage as is
|
||||||
} else {
|
} else {
|
||||||
// Missing required fields, remove s3Storage to avoid type errors
|
// Missing required fields, remove s3Storage to avoid type errors
|
||||||
const { s3Storage, ...rest } = storageOptions
|
const { s3Storage, ...rest } = storageOptions
|
||||||
storageOptions = rest
|
storageOptions = rest
|
||||||
console.warn('Ignoring s3Storage configuration due to missing required fields')
|
console.warn(
|
||||||
|
'Ignoring s3Storage configuration due to missing required fields'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -766,6 +837,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
this.index.setStorage(this.storage!)
|
this.index.setStorage(this.storage!)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// In write-only mode, skip loading the index into memory
|
||||||
|
if (this.writeOnly) {
|
||||||
|
if (this.loggingConfig?.verbose) {
|
||||||
|
console.log('Database is in write-only mode, skipping index loading')
|
||||||
|
}
|
||||||
|
} else {
|
||||||
// Load all nouns from storage
|
// Load all nouns from storage
|
||||||
const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
|
const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
|
||||||
|
|
||||||
|
|
@ -788,6 +865,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
vector: noun.vector
|
vector: noun.vector
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Connect to remote server if configured with autoConnect
|
// Connect to remote server if configured with autoConnect
|
||||||
if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) {
|
if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) {
|
||||||
|
|
@ -888,10 +966,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if input is already a vector
|
// Check if input is already a vector
|
||||||
if (
|
if (Array.isArray(vectorOrData) && !options.forceEmbed) {
|
||||||
Array.isArray(vectorOrData) &&
|
|
||||||
!options.forceEmbed
|
|
||||||
) {
|
|
||||||
// Input is already a vector (and we've validated it contains only numbers)
|
// Input is already a vector (and we've validated it contains only numbers)
|
||||||
vector = vectorOrData
|
vector = vectorOrData
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -910,7 +985,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// Validate vector dimensions
|
// Validate vector dimensions
|
||||||
if (vector.length !== this._dimensions) {
|
if (vector.length !== this._dimensions) {
|
||||||
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`)
|
throw new Error(
|
||||||
|
`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID
|
// Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID
|
||||||
|
|
@ -940,7 +1017,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
// Save metadata if provided and not empty
|
// Save metadata if provided and not empty
|
||||||
if (metadata !== undefined) {
|
if (metadata !== undefined) {
|
||||||
// Skip saving if metadata is an empty object
|
// Skip saving if metadata is an empty object
|
||||||
if (metadata && typeof metadata === 'object' && Object.keys(metadata).length === 0) {
|
if (
|
||||||
|
metadata &&
|
||||||
|
typeof metadata === 'object' &&
|
||||||
|
Object.keys(metadata).length === 0
|
||||||
|
) {
|
||||||
// Don't save empty metadata
|
// Don't save empty metadata
|
||||||
// Explicitly save null to ensure no metadata is stored
|
// Explicitly save null to ensure no metadata is stored
|
||||||
await this.storage!.saveMetadata(id, null)
|
await this.storage!.saveMetadata(id, null)
|
||||||
|
|
@ -998,7 +1079,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
await this.storage!.saveMetadata(id, metadataToSave)
|
await this.storage!.saveMetadata(id, metadataToSave)
|
||||||
|
|
||||||
// Track metadata statistics
|
// Track metadata statistics
|
||||||
const metadataService = options.service || this.getCurrentAugmentation()
|
const metadataService =
|
||||||
|
options.service || this.getCurrentAugmentation()
|
||||||
await this.storage!.incrementStatistic('metadata', metadataService)
|
await this.storage!.incrementStatistic('metadata', metadataService)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1274,7 +1356,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): R[] {
|
): R[] {
|
||||||
if (!service) return results
|
if (!service) return results
|
||||||
|
|
||||||
return results.filter(result => {
|
return results.filter((result) => {
|
||||||
if (!result.metadata || typeof result.metadata !== 'object') return false
|
if (!result.metadata || typeof result.metadata !== 'object') return false
|
||||||
if (!('createdBy' in result.metadata)) return false
|
if (!('createdBy' in result.metadata)) return false
|
||||||
|
|
||||||
|
|
@ -1316,9 +1398,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
return createdBy.augmentation === options.service
|
return createdBy.augmentation === options.service
|
||||||
}
|
}
|
||||||
if (!this.isInitialized) {
|
if (!this.isInitialized) {
|
||||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
throw new Error(
|
||||||
|
'BrainyData must be initialized before searching. Call init() first.'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let queryVector: Vector
|
let queryVector: Vector
|
||||||
|
|
||||||
|
|
@ -1346,7 +1433,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// Check if query vector dimensions match the expected dimensions
|
// Check if query vector dimensions match the expected dimensions
|
||||||
if (queryVector.length !== this._dimensions) {
|
if (queryVector.length !== this._dimensions) {
|
||||||
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`)
|
throw new Error(
|
||||||
|
`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If no noun types specified, search all nouns
|
// If no noun types specified, search all nouns
|
||||||
|
|
@ -1485,8 +1574,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this.isInitialized) {
|
if (!this.isInitialized) {
|
||||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
throw new Error(
|
||||||
|
'BrainyData must be initialized before searching. Call init() first.'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
// If searching for verbs directly
|
// If searching for verbs directly
|
||||||
if (options.searchVerbs) {
|
if (options.searchVerbs) {
|
||||||
const verbResults = await this.searchVerbs(queryVectorOrData, k, {
|
const verbResults = await this.searchVerbs(queryVectorOrData, k, {
|
||||||
|
|
@ -1548,8 +1642,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
} = {}
|
} = {}
|
||||||
): Promise<SearchResult<T>[]> {
|
): Promise<SearchResult<T>[]> {
|
||||||
if (!this.isInitialized) {
|
if (!this.isInitialized) {
|
||||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
throw new Error(
|
||||||
|
'BrainyData must be initialized before searching. Call init() first.'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
// If input is a string and not a vector, automatically vectorize it
|
// If input is a string and not a vector, automatically vectorize it
|
||||||
let queryToUse = queryVectorOrData
|
let queryToUse = queryVectorOrData
|
||||||
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
|
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
|
||||||
|
|
@ -1636,10 +1735,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const outgoingVerbs = await this.storage!.getVerbsBySource(id)
|
const outgoingVerbs = await this.storage!.getVerbsBySource(id)
|
||||||
|
|
||||||
// Filter to only include verbs of the specified type
|
// Filter to only include verbs of the specified type
|
||||||
const verbsOfType = outgoingVerbs.filter(verb => verb.type === options.relationType)
|
const verbsOfType = outgoingVerbs.filter(
|
||||||
|
(verb) => verb.type === options.relationType
|
||||||
|
)
|
||||||
|
|
||||||
// Get the target IDs
|
// Get the target IDs
|
||||||
const targetIds = verbsOfType.map(verb => verb.target)
|
const targetIds = verbsOfType.map((verb) => verb.target)
|
||||||
|
|
||||||
// Get the actual entities for these IDs
|
// Get the actual entities for these IDs
|
||||||
const results: SearchResult<T>[] = []
|
const results: SearchResult<T>[] = []
|
||||||
|
|
@ -1858,11 +1959,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const graphNoun = metadata as unknown as GraphNoun
|
const graphNoun = metadata as unknown as GraphNoun
|
||||||
|
|
||||||
// Preserve existing createdBy and createdAt if they exist
|
// Preserve existing createdBy and createdAt if they exist
|
||||||
const existingMetadata = await this.storage!.getMetadata(id) as any
|
const existingMetadata = (await this.storage!.getMetadata(id)) as any
|
||||||
|
|
||||||
if (existingMetadata &&
|
if (
|
||||||
|
existingMetadata &&
|
||||||
typeof existingMetadata === 'object' &&
|
typeof existingMetadata === 'object' &&
|
||||||
'createdBy' in existingMetadata) {
|
'createdBy' in existingMetadata
|
||||||
|
) {
|
||||||
// Preserve the original creator information
|
// Preserve the original creator information
|
||||||
graphNoun.createdBy = existingMetadata.createdBy
|
graphNoun.createdBy = existingMetadata.createdBy
|
||||||
|
|
||||||
|
|
@ -2035,8 +2138,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
console.warn(`Auto-created missing source noun with ID ${sourceId}`)
|
console.warn(`Auto-created missing source noun with ID ${sourceId}`)
|
||||||
} catch (createError) {
|
} catch (createError) {
|
||||||
console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError)
|
console.error(
|
||||||
throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`)
|
`Failed to auto-create source noun with ID ${sourceId}:`,
|
||||||
|
createError
|
||||||
|
)
|
||||||
|
throw new Error(
|
||||||
|
`Failed to auto-create source noun with ID ${sourceId}: ${createError}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2072,8 +2180,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
console.warn(`Auto-created missing target noun with ID ${targetId}`)
|
console.warn(`Auto-created missing target noun with ID ${targetId}`)
|
||||||
} catch (createError) {
|
} catch (createError) {
|
||||||
console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError)
|
console.error(
|
||||||
throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`)
|
`Failed to auto-create target noun with ID ${targetId}:`,
|
||||||
|
createError
|
||||||
|
)
|
||||||
|
throw new Error(
|
||||||
|
`Failed to auto-create target noun with ID ${targetId}: ${createError}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2358,7 +2471,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const allVerbs = await this.storage!.getAllVerbs()
|
const allVerbs = await this.storage!.getAllVerbs()
|
||||||
|
|
||||||
// Create a set of verb IDs for faster lookup
|
// Create a set of verb IDs for faster lookup
|
||||||
const verbIds = new Set(allVerbs.map(verb => verb.id))
|
const verbIds = new Set(allVerbs.map((verb) => verb.id))
|
||||||
|
|
||||||
// Get all nouns from the index
|
// Get all nouns from the index
|
||||||
const nouns = this.index.getNouns()
|
const nouns = this.index.getNouns()
|
||||||
|
|
@ -2395,10 +2508,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
* @param options Additional options for retrieving statistics
|
* @param options Additional options for retrieving statistics
|
||||||
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
||||||
*/
|
*/
|
||||||
public async getStatistics(options: {
|
public async getStatistics(
|
||||||
|
options: {
|
||||||
service?: string | string[] // Filter statistics by service(s)
|
service?: string | string[] // Filter statistics by service(s)
|
||||||
forceRefresh?: boolean // Force a refresh of statistics from storage
|
forceRefresh?: boolean // Force a refresh of statistics from storage
|
||||||
} = {}): Promise<{
|
} = {}
|
||||||
|
): Promise<{
|
||||||
nounCount: number
|
nounCount: number
|
||||||
verbCount: number
|
verbCount: number
|
||||||
metadataCount: number
|
metadataCount: number
|
||||||
|
|
@ -2463,8 +2578,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// Filter by service if specified
|
// Filter by service if specified
|
||||||
const services = options.service
|
const services = options.service
|
||||||
? (Array.isArray(options.service) ? options.service : [options.service])
|
? Array.isArray(options.service)
|
||||||
: Object.keys({...stats.nounCount, ...stats.verbCount, ...stats.metadataCount})
|
? options.service
|
||||||
|
: [options.service]
|
||||||
|
: Object.keys({
|
||||||
|
...stats.nounCount,
|
||||||
|
...stats.verbCount,
|
||||||
|
...stats.metadataCount
|
||||||
|
})
|
||||||
|
|
||||||
// Calculate totals and service breakdown
|
// Calculate totals and service breakdown
|
||||||
for (const service of services) {
|
for (const service of services) {
|
||||||
|
|
@ -2583,6 +2704,32 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
*/
|
*/
|
||||||
public setReadOnly(readOnly: boolean): void {
|
public setReadOnly(readOnly: boolean): void {
|
||||||
this.readOnly = readOnly
|
this.readOnly = readOnly
|
||||||
|
|
||||||
|
// Ensure readOnly and writeOnly are not both true
|
||||||
|
if (readOnly && this.writeOnly) {
|
||||||
|
this.writeOnly = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the database is in write-only mode
|
||||||
|
* @returns True if the database is in write-only mode, false otherwise
|
||||||
|
*/
|
||||||
|
public isWriteOnly(): boolean {
|
||||||
|
return this.writeOnly
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the database to write-only mode
|
||||||
|
* @param writeOnly True to set the database to write-only mode, false to allow searches
|
||||||
|
*/
|
||||||
|
public setWriteOnly(writeOnly: boolean): void {
|
||||||
|
this.writeOnly = writeOnly
|
||||||
|
|
||||||
|
// Ensure readOnly and writeOnly are not both true
|
||||||
|
if (writeOnly && this.readOnly) {
|
||||||
|
this.readOnly = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -2621,6 +2768,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): Promise<Array<GraphVerb & { similarity: number }>> {
|
): Promise<Array<GraphVerb & { similarity: number }>> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let queryVector: Vector
|
let queryVector: Vector
|
||||||
|
|
||||||
|
|
@ -2677,7 +2827,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
|
|
||||||
// If we didn't get enough results from the index, fall back to the old method
|
// If we didn't get enough results from the index, fall back to the old method
|
||||||
if (verbResults.length < k) {
|
if (verbResults.length < k) {
|
||||||
console.warn('Not enough verb results from HNSW index, falling back to manual search')
|
console.warn(
|
||||||
|
'Not enough verb results from HNSW index, falling back to manual search'
|
||||||
|
)
|
||||||
|
|
||||||
// Get verbs to search through
|
// Get verbs to search through
|
||||||
let verbs: GraphVerb[] = []
|
let verbs: GraphVerb[] = []
|
||||||
|
|
@ -2700,9 +2852,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate similarity for each verb not already in results
|
// Calculate similarity for each verb not already in results
|
||||||
const existingIds = new Set(verbResults.map(v => v.id))
|
const existingIds = new Set(verbResults.map((v) => v.id))
|
||||||
for (const verb of verbs) {
|
for (const verb of verbs) {
|
||||||
if (!existingIds.has(verb.id) && verb.vector && verb.vector.length > 0) {
|
if (
|
||||||
|
!existingIds.has(verb.id) &&
|
||||||
|
verb.vector &&
|
||||||
|
verb.vector.length > 0
|
||||||
|
) {
|
||||||
const distance = this.index.getDistanceFunction()(
|
const distance = this.index.getDistanceFunction()(
|
||||||
queryVector,
|
queryVector,
|
||||||
verb.vector
|
verb.vector
|
||||||
|
|
@ -2744,6 +2900,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): Promise<SearchResult<T>[]> {
|
): Promise<SearchResult<T>[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// First, search for nouns
|
// First, search for nouns
|
||||||
const nounResults = await this.searchByNounTypes(
|
const nounResults = await this.searchByNounTypes(
|
||||||
|
|
@ -2864,6 +3023,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): Promise<SearchResult<T>[]> {
|
): Promise<SearchResult<T>[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Embed the query text
|
// Embed the query text
|
||||||
const queryVector = await this.embed(query)
|
const queryVector = await this.embed(query)
|
||||||
|
|
@ -2900,6 +3062,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): Promise<SearchResult<T>[]> {
|
): Promise<SearchResult<T>[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
|
|
||||||
// Check if connected to a remote server
|
// Check if connected to a remote server
|
||||||
if (!this.isConnectedToRemoteServer()) {
|
if (!this.isConnectedToRemoteServer()) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
|
|
@ -2962,6 +3127,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
): Promise<SearchResult<T>[]> {
|
): Promise<SearchResult<T>[]> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
|
// Check if database is in write-only mode
|
||||||
|
this.checkWriteOnly()
|
||||||
|
|
||||||
// Check if connected to a remote server
|
// Check if connected to a remote server
|
||||||
if (!this.isConnectedToRemoteServer()) {
|
if (!this.isConnectedToRemoteServer()) {
|
||||||
// If not connected to a remote server, just search locally
|
// If not connected to a remote server, just search locally
|
||||||
|
|
@ -3221,7 +3389,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
try {
|
try {
|
||||||
await this.flushStatistics()
|
await this.flushStatistics()
|
||||||
} catch (statsError) {
|
} catch (statsError) {
|
||||||
console.warn('Failed to flush statistics during shutdown:', statsError)
|
console.warn(
|
||||||
|
'Failed to flush statistics during shutdown:',
|
||||||
|
statsError
|
||||||
|
)
|
||||||
// Continue with shutdown even if statistics flush fails
|
// Continue with shutdown even if statistics flush fails
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -3547,14 +3718,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const nounIds: string[] = []
|
const nounIds: string[] = []
|
||||||
const nounDescriptions: Record<string, string> = {
|
const nounDescriptions: Record<string, string> = {
|
||||||
[NounType.Person]: 'A person with unique characteristics',
|
[NounType.Person]: 'A person with unique characteristics',
|
||||||
[NounType.Place]: 'A location with specific attributes',
|
[NounType.Location]: 'A location with specific attributes',
|
||||||
[NounType.Thing]: 'An object with distinct properties',
|
[NounType.Thing]: 'An object with distinct properties',
|
||||||
[NounType.Event]: 'An occurrence with temporal aspects',
|
[NounType.Event]: 'An occurrence with temporal aspects',
|
||||||
[NounType.Concept]: 'An abstract idea or notion',
|
[NounType.Concept]: 'An abstract idea or notion',
|
||||||
[NounType.Content]: 'A piece of content or information',
|
[NounType.Content]: 'A piece of content or information',
|
||||||
[NounType.Group]: 'A collection of related entities',
|
[NounType.Collection]: 'A collection of related entities',
|
||||||
[NounType.List]: 'An ordered sequence of items',
|
[NounType.Organization]: 'An organization or institution',
|
||||||
[NounType.Category]: 'A classification or grouping'
|
[NounType.Document]: 'A document or text-based file'
|
||||||
}
|
}
|
||||||
|
|
||||||
for (let i = 0; i < nounCount; i++) {
|
for (let i = 0; i < nounCount; i++) {
|
||||||
|
|
@ -3585,10 +3756,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
const verbIds: string[] = []
|
const verbIds: string[] = []
|
||||||
const verbDescriptions: Record<string, string> = {
|
const verbDescriptions: Record<string, string> = {
|
||||||
[VerbType.AttributedTo]: 'Attribution relationship',
|
[VerbType.AttributedTo]: 'Attribution relationship',
|
||||||
[VerbType.Controls]: 'Control relationship',
|
|
||||||
[VerbType.Created]: 'Creation relationship',
|
|
||||||
[VerbType.Earned]: 'Achievement relationship',
|
|
||||||
[VerbType.Owns]: 'Ownership relationship',
|
[VerbType.Owns]: 'Ownership relationship',
|
||||||
|
[VerbType.Creates]: 'Creation relationship',
|
||||||
|
[VerbType.Uses]: 'Utilization relationship',
|
||||||
|
[VerbType.BelongsTo]: 'Belonging relationship',
|
||||||
[VerbType.MemberOf]: 'Membership relationship',
|
[VerbType.MemberOf]: 'Membership relationship',
|
||||||
[VerbType.RelatedTo]: 'General relationship',
|
[VerbType.RelatedTo]: 'General relationship',
|
||||||
[VerbType.WorksWith]: 'Collaboration relationship',
|
[VerbType.WorksWith]: 'Collaboration relationship',
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue