**feat: add write-only mode support and example usage**
- Introduced `writeOnly` mode in `BrainyData` allowing optimized data ingestion by skipping index loading and disabling search operations. - Enhanced `BrainyDataConfig` to include `writeOnly` support and validate compatibility with `readOnly` mode. - Implemented error handling for search attempts during `writeOnly` mode. - Updated the README with detailed usage examples for database modes, including `writeOnly` and `readOnly`. - Added `examples/write-only-mode.js` to demonstrate practical applications of the `writeOnly` mode. **Purpose**: Optimize memory usage and startup time for data ingestion scenarios by enabling write-only mode with comprehensive documentation and examples.
This commit is contained in:
parent
d70d0946cf
commit
5d82db12e2
3 changed files with 3696 additions and 3426 deletions
31
README.md
31
README.md
|
|
@ -598,6 +598,37 @@ await db.deleteVerb(verbId)
|
|||
|
||||
## Advanced Configuration
|
||||
|
||||
### Database Modes
|
||||
|
||||
Brainy supports special operational modes that restrict certain operations:
|
||||
|
||||
```typescript
|
||||
import { BrainyData } from '@soulcraft/brainy'
|
||||
|
||||
// Create and initialize the database
|
||||
const db = new BrainyData()
|
||||
await db.init()
|
||||
|
||||
// Set the database to read-only mode (prevents write operations)
|
||||
db.setReadOnly(true)
|
||||
|
||||
// Check if the database is in read-only mode
|
||||
const isReadOnly = db.isReadOnly() // Returns true
|
||||
|
||||
// Set the database to write-only mode (prevents search operations)
|
||||
db.setWriteOnly(true)
|
||||
|
||||
// Check if the database is in write-only mode
|
||||
const isWriteOnly = db.isWriteOnly() // Returns true
|
||||
|
||||
// Reset to normal mode (allows both read and write operations)
|
||||
db.setReadOnly(false)
|
||||
db.setWriteOnly(false)
|
||||
```
|
||||
|
||||
- **Read-Only Mode**: When enabled, prevents all write operations (add, update, delete). Useful for deployment scenarios where you want to prevent modifications to the database.
|
||||
- **Write-Only Mode**: When enabled, prevents all search operations. Useful for initial data loading or when you want to optimize for write performance.
|
||||
|
||||
### Embedding
|
||||
|
||||
```typescript
|
||||
|
|
|
|||
68
examples/write-only-mode.js
Normal file
68
examples/write-only-mode.js
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Example demonstrating the write-only mode in BrainyData
|
||||
*
|
||||
* This mode is useful for data ingestion scenarios where you only need to insert data
|
||||
* and don't need to perform any search operations. It significantly reduces memory usage
|
||||
* by not loading the index into memory.
|
||||
*/
|
||||
|
||||
import { BrainyData } from '../dist/index.js'
|
||||
|
||||
// eslint-disable-next-line no-undef
|
||||
const console = globalThis.console
|
||||
|
||||
async function demonstrateWriteOnlyMode() {
|
||||
console.log('Demonstrating write-only mode in BrainyData')
|
||||
|
||||
// Create a BrainyData instance in write-only mode
|
||||
const db = new BrainyData({
|
||||
writeOnly: true,
|
||||
storage: {
|
||||
// Use file system storage for persistence
|
||||
forceFileSystemStorage: true
|
||||
}
|
||||
})
|
||||
|
||||
// Initialize the database
|
||||
// This will skip loading the index into memory, saving memory and startup time
|
||||
await db.init()
|
||||
|
||||
console.log('Database initialized in write-only mode')
|
||||
console.log('Is write-only:', db.isWriteOnly()) // Should print true
|
||||
|
||||
// Add some data
|
||||
// This works normally even in write-only mode
|
||||
const id1 = await db.add('This is a test document', { title: 'Test Document 1' })
|
||||
const id2 = await db.add('Another test document with different content', { title: 'Test Document 2' })
|
||||
|
||||
console.log(`Added documents with IDs: ${id1}, ${id2}`)
|
||||
|
||||
// Try to search (this will throw an error in write-only mode)
|
||||
try {
|
||||
await db.search('test', 5)
|
||||
} catch (error) {
|
||||
console.log('Search error (expected):', error.message)
|
||||
}
|
||||
|
||||
// Switch to normal mode to perform searches
|
||||
console.log('Switching to normal mode...')
|
||||
db.setWriteOnly(false)
|
||||
|
||||
// Re-initialize to load the index
|
||||
await db.init()
|
||||
|
||||
console.log('Database reinitialized in normal mode')
|
||||
console.log('Is write-only:', db.isWriteOnly()) // Should print false
|
||||
|
||||
// Now search works
|
||||
const results = await db.search('test', 5)
|
||||
console.log('Search results:', results.map(r => ({ id: r.id, score: r.score, title: r.metadata?.title })))
|
||||
|
||||
// Clean up
|
||||
await db.shutDown()
|
||||
}
|
||||
|
||||
// Run the example
|
||||
demonstrateWriteOnlyMode().catch(error => {
|
||||
console.error('Error in write-only mode example:', error)
|
||||
})
|
||||
|
|
@ -35,7 +35,11 @@ import {
|
|||
ServerSearchConduitAugmentation,
|
||||
createServerSearchAugmentations
|
||||
} from './augmentations/serverSearchAugmentations.js'
|
||||
import {WebSocketConnection, AugmentationType, IAugmentation} from './types/augmentations.js'
|
||||
import {
|
||||
WebSocketConnection,
|
||||
AugmentationType,
|
||||
IAugmentation
|
||||
} from './types/augmentations.js'
|
||||
import { BrainyDataInterface } from './types/brainyDataInterface.js'
|
||||
import { augmentationPipeline } from './augmentationPipeline.js'
|
||||
|
||||
|
|
@ -103,6 +107,13 @@ export interface BrainyDataConfig {
|
|||
*/
|
||||
readOnly?: boolean
|
||||
|
||||
/**
|
||||
* Set the database to write-only mode
|
||||
* When true, the index is not loaded into memory and search operations will throw an error
|
||||
* This is useful for data ingestion scenarios where only write operations are needed
|
||||
*/
|
||||
writeOnly?: boolean
|
||||
|
||||
/**
|
||||
* Remote server configuration for search operations
|
||||
*/
|
||||
|
|
@ -230,6 +241,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
private distanceFunction: DistanceFunction
|
||||
private requestPersistentStorage: boolean
|
||||
private readOnly: boolean
|
||||
private writeOnly: boolean
|
||||
private storageConfig: BrainyDataConfig['storage'] = {}
|
||||
private useOptimizedIndex: boolean = false
|
||||
private _dimensions: number
|
||||
|
|
@ -240,7 +252,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
private retryConfig: BrainyDataConfig['retryPolicy'] = {}
|
||||
|
||||
// Real-time update properties
|
||||
private realtimeUpdateConfig: Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> = {
|
||||
private realtimeUpdateConfig: Required<
|
||||
NonNullable<BrainyDataConfig['realtimeUpdates']>
|
||||
> = {
|
||||
enabled: false,
|
||||
interval: 30000, // 30 seconds
|
||||
updateStatistics: true,
|
||||
|
|
@ -323,6 +337,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Set read-only flag
|
||||
this.readOnly = config.readOnly || false
|
||||
|
||||
// Set write-only flag
|
||||
this.writeOnly = config.writeOnly || false
|
||||
|
||||
// Validate that readOnly and writeOnly are not both true
|
||||
if (this.readOnly && this.writeOnly) {
|
||||
throw new Error('Database cannot be both read-only and write-only')
|
||||
}
|
||||
|
||||
// Store storage configuration for later use in init()
|
||||
this.storageConfig = config.storage || {}
|
||||
|
||||
|
|
@ -356,6 +378,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the database is in write-only mode and throw an error if it is
|
||||
* @throws Error if the database is in write-only mode
|
||||
*/
|
||||
private checkWriteOnly(): void {
|
||||
if (this.writeOnly) {
|
||||
throw new Error(
|
||||
'Cannot perform search operation: database is in write-only mode'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start real-time updates if enabled in the configuration
|
||||
* This will periodically check for new data in storage and update the in-memory index and statistics
|
||||
|
|
@ -372,21 +406,28 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
// Set the initial last known noun count
|
||||
this.getNounCount().then(count => {
|
||||
this.getNounCount()
|
||||
.then((count) => {
|
||||
this.lastKnownNounCount = count
|
||||
}).catch(error => {
|
||||
console.warn('Failed to get initial noun count for real-time updates:', error)
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn(
|
||||
'Failed to get initial noun count for real-time updates:',
|
||||
error
|
||||
)
|
||||
})
|
||||
|
||||
// Start the update timer
|
||||
this.updateTimerId = setInterval(() => {
|
||||
this.checkForUpdates().catch(error => {
|
||||
this.checkForUpdates().catch((error) => {
|
||||
console.warn('Error during real-time update check:', error)
|
||||
})
|
||||
}, this.realtimeUpdateConfig.interval)
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`)
|
||||
console.log(
|
||||
`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -421,7 +462,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
* Enable real-time updates with the specified configuration
|
||||
* @param config Configuration for real-time updates
|
||||
*/
|
||||
public enableRealtimeUpdates(config?: Partial<BrainyDataConfig['realtimeUpdates']>): void {
|
||||
public enableRealtimeUpdates(
|
||||
config?: Partial<BrainyDataConfig['realtimeUpdates']>
|
||||
): void {
|
||||
// Update configuration if provided
|
||||
if (config) {
|
||||
this.realtimeUpdateConfig = {
|
||||
|
|
@ -454,7 +497,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
* Get the current real-time update configuration
|
||||
* @returns The current real-time update configuration
|
||||
*/
|
||||
public getRealtimeUpdateConfig(): Required<NonNullable<BrainyDataConfig['realtimeUpdates']>> {
|
||||
public getRealtimeUpdateConfig(): Required<
|
||||
NonNullable<BrainyDataConfig['realtimeUpdates']>
|
||||
> {
|
||||
return { ...this.realtimeUpdateConfig }
|
||||
}
|
||||
|
||||
|
|
@ -514,7 +559,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
try {
|
||||
// Get changes since the last update
|
||||
const changes = await this.storage.getChangesSince(this.lastUpdateTime, 1000) // Limit to 1000 changes per batch
|
||||
const changes = await this.storage.getChangesSince(
|
||||
this.lastUpdateTime,
|
||||
1000
|
||||
) // Limit to 1000 changes per batch
|
||||
|
||||
let addedCount = 0
|
||||
let updatedCount = 0
|
||||
|
|
@ -549,7 +597,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`)
|
||||
console.log(
|
||||
`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`
|
||||
)
|
||||
}
|
||||
}
|
||||
break
|
||||
|
|
@ -561,26 +611,38 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
deletedCount++
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(`Removed noun ${change.entityId} from index during real-time update`)
|
||||
console.log(
|
||||
`Removed noun ${change.entityId} from index during real-time update`
|
||||
)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
} catch (changeError) {
|
||||
console.error(`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`, changeError)
|
||||
console.error(
|
||||
`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`,
|
||||
changeError
|
||||
)
|
||||
// Continue with other changes
|
||||
}
|
||||
}
|
||||
|
||||
if (this.loggingConfig?.verbose && (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) {
|
||||
console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`)
|
||||
if (
|
||||
this.loggingConfig?.verbose &&
|
||||
(addedCount > 0 || updatedCount > 0 || deletedCount > 0)
|
||||
) {
|
||||
console.log(
|
||||
`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`
|
||||
)
|
||||
}
|
||||
|
||||
// Update the last known noun count
|
||||
this.lastKnownNounCount = await this.getNounCount()
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to apply changes from log, falling back to full scan:', error)
|
||||
console.error(
|
||||
'Failed to apply changes from log, falling back to full scan:',
|
||||
error
|
||||
)
|
||||
// Fallback to full scan if change log fails
|
||||
await this.applyChangesFromFullScan()
|
||||
}
|
||||
|
|
@ -604,7 +666,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const indexNounIds = new Set(indexNouns.keys())
|
||||
|
||||
// Find nouns that are in storage but not in the index
|
||||
const newNouns = nouns.filter(noun => !indexNounIds.has(noun.id))
|
||||
const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id))
|
||||
|
||||
// Add new nouns to the index
|
||||
for (const noun of newNouns) {
|
||||
|
|
@ -623,7 +685,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
})
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(`Added new noun ${noun.id} to index during real-time update`)
|
||||
console.log(
|
||||
`Added new noun ${noun.id} to index during real-time update`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -631,7 +695,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
this.lastKnownNounCount = currentCount
|
||||
|
||||
if (this.loggingConfig?.verbose && newNouns.length > 0) {
|
||||
console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`)
|
||||
console.log(
|
||||
`Real-time update: Added ${newNouns.length} new nouns to index using full scan`
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
|
|
@ -648,7 +714,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
private getCurrentAugmentation(): string {
|
||||
try {
|
||||
// Get all registered augmentations
|
||||
const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes()
|
||||
const augmentationTypes =
|
||||
augmentationPipeline.getAvailableAugmentationTypes()
|
||||
|
||||
// Check each type of augmentation
|
||||
for (const type of augmentationTypes) {
|
||||
|
|
@ -742,15 +809,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Ensure s3Storage has all required fields if it's provided
|
||||
if (storageOptions.s3Storage) {
|
||||
// Only include s3Storage if all required fields are present
|
||||
if (storageOptions.s3Storage.bucketName &&
|
||||
if (
|
||||
storageOptions.s3Storage.bucketName &&
|
||||
storageOptions.s3Storage.accessKeyId &&
|
||||
storageOptions.s3Storage.secretAccessKey) {
|
||||
storageOptions.s3Storage.secretAccessKey
|
||||
) {
|
||||
// All required fields are present, keep s3Storage as is
|
||||
} else {
|
||||
// Missing required fields, remove s3Storage to avoid type errors
|
||||
const { s3Storage, ...rest } = storageOptions
|
||||
storageOptions = rest
|
||||
console.warn('Ignoring s3Storage configuration due to missing required fields')
|
||||
console.warn(
|
||||
'Ignoring s3Storage configuration due to missing required fields'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -766,6 +837,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
this.index.setStorage(this.storage!)
|
||||
}
|
||||
|
||||
// In write-only mode, skip loading the index into memory
|
||||
if (this.writeOnly) {
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Database is in write-only mode, skipping index loading')
|
||||
}
|
||||
} else {
|
||||
// Load all nouns from storage
|
||||
const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
|
||||
|
||||
|
|
@ -788,6 +865,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
vector: noun.vector
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to remote server if configured with autoConnect
|
||||
if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) {
|
||||
|
|
@ -888,10 +966,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
// Check if input is already a vector
|
||||
if (
|
||||
Array.isArray(vectorOrData) &&
|
||||
!options.forceEmbed
|
||||
) {
|
||||
if (Array.isArray(vectorOrData) && !options.forceEmbed) {
|
||||
// Input is already a vector (and we've validated it contains only numbers)
|
||||
vector = vectorOrData
|
||||
} else {
|
||||
|
|
@ -910,7 +985,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Validate vector dimensions
|
||||
if (vector.length !== this._dimensions) {
|
||||
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`)
|
||||
throw new Error(
|
||||
`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`
|
||||
)
|
||||
}
|
||||
|
||||
// Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID
|
||||
|
|
@ -940,7 +1017,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Save metadata if provided and not empty
|
||||
if (metadata !== undefined) {
|
||||
// Skip saving if metadata is an empty object
|
||||
if (metadata && typeof metadata === 'object' && Object.keys(metadata).length === 0) {
|
||||
if (
|
||||
metadata &&
|
||||
typeof metadata === 'object' &&
|
||||
Object.keys(metadata).length === 0
|
||||
) {
|
||||
// Don't save empty metadata
|
||||
// Explicitly save null to ensure no metadata is stored
|
||||
await this.storage!.saveMetadata(id, null)
|
||||
|
|
@ -998,7 +1079,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
await this.storage!.saveMetadata(id, metadataToSave)
|
||||
|
||||
// Track metadata statistics
|
||||
const metadataService = options.service || this.getCurrentAugmentation()
|
||||
const metadataService =
|
||||
options.service || this.getCurrentAugmentation()
|
||||
await this.storage!.incrementStatistic('metadata', metadataService)
|
||||
}
|
||||
}
|
||||
|
|
@ -1274,7 +1356,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
): R[] {
|
||||
if (!service) return results
|
||||
|
||||
return results.filter(result => {
|
||||
return results.filter((result) => {
|
||||
if (!result.metadata || typeof result.metadata !== 'object') return false
|
||||
if (!('createdBy' in result.metadata)) return false
|
||||
|
||||
|
|
@ -1316,9 +1398,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
return createdBy.augmentation === options.service
|
||||
}
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
||||
throw new Error(
|
||||
'BrainyData must be initialized before searching. Call init() first.'
|
||||
)
|
||||
}
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
try {
|
||||
let queryVector: Vector
|
||||
|
||||
|
|
@ -1346,7 +1433,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Check if query vector dimensions match the expected dimensions
|
||||
if (queryVector.length !== this._dimensions) {
|
||||
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`)
|
||||
throw new Error(
|
||||
`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`
|
||||
)
|
||||
}
|
||||
|
||||
// If no noun types specified, search all nouns
|
||||
|
|
@ -1485,8 +1574,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
||||
throw new Error(
|
||||
'BrainyData must be initialized before searching. Call init() first.'
|
||||
)
|
||||
}
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
// If searching for verbs directly
|
||||
if (options.searchVerbs) {
|
||||
const verbResults = await this.searchVerbs(queryVectorOrData, k, {
|
||||
|
|
@ -1548,8 +1642,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('BrainyData must be initialized before searching. Call init() first.')
|
||||
throw new Error(
|
||||
'BrainyData must be initialized before searching. Call init() first.'
|
||||
)
|
||||
}
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
// If input is a string and not a vector, automatically vectorize it
|
||||
let queryToUse = queryVectorOrData
|
||||
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
|
||||
|
|
@ -1636,10 +1735,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const outgoingVerbs = await this.storage!.getVerbsBySource(id)
|
||||
|
||||
// Filter to only include verbs of the specified type
|
||||
const verbsOfType = outgoingVerbs.filter(verb => verb.type === options.relationType)
|
||||
const verbsOfType = outgoingVerbs.filter(
|
||||
(verb) => verb.type === options.relationType
|
||||
)
|
||||
|
||||
// Get the target IDs
|
||||
const targetIds = verbsOfType.map(verb => verb.target)
|
||||
const targetIds = verbsOfType.map((verb) => verb.target)
|
||||
|
||||
// Get the actual entities for these IDs
|
||||
const results: SearchResult<T>[] = []
|
||||
|
|
@ -1858,11 +1959,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const graphNoun = metadata as unknown as GraphNoun
|
||||
|
||||
// Preserve existing createdBy and createdAt if they exist
|
||||
const existingMetadata = await this.storage!.getMetadata(id) as any
|
||||
const existingMetadata = (await this.storage!.getMetadata(id)) as any
|
||||
|
||||
if (existingMetadata &&
|
||||
if (
|
||||
existingMetadata &&
|
||||
typeof existingMetadata === 'object' &&
|
||||
'createdBy' in existingMetadata) {
|
||||
'createdBy' in existingMetadata
|
||||
) {
|
||||
// Preserve the original creator information
|
||||
graphNoun.createdBy = existingMetadata.createdBy
|
||||
|
||||
|
|
@ -2035,8 +2138,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
console.warn(`Auto-created missing source noun with ID ${sourceId}`)
|
||||
} catch (createError) {
|
||||
console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError)
|
||||
throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`)
|
||||
console.error(
|
||||
`Failed to auto-create source noun with ID ${sourceId}:`,
|
||||
createError
|
||||
)
|
||||
throw new Error(
|
||||
`Failed to auto-create source noun with ID ${sourceId}: ${createError}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2072,8 +2180,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
console.warn(`Auto-created missing target noun with ID ${targetId}`)
|
||||
} catch (createError) {
|
||||
console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError)
|
||||
throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`)
|
||||
console.error(
|
||||
`Failed to auto-create target noun with ID ${targetId}:`,
|
||||
createError
|
||||
)
|
||||
throw new Error(
|
||||
`Failed to auto-create target noun with ID ${targetId}: ${createError}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2358,7 +2471,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const allVerbs = await this.storage!.getAllVerbs()
|
||||
|
||||
// Create a set of verb IDs for faster lookup
|
||||
const verbIds = new Set(allVerbs.map(verb => verb.id))
|
||||
const verbIds = new Set(allVerbs.map((verb) => verb.id))
|
||||
|
||||
// Get all nouns from the index
|
||||
const nouns = this.index.getNouns()
|
||||
|
|
@ -2395,10 +2508,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
* @param options Additional options for retrieving statistics
|
||||
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
||||
*/
|
||||
public async getStatistics(options: {
|
||||
public async getStatistics(
|
||||
options: {
|
||||
service?: string | string[] // Filter statistics by service(s)
|
||||
forceRefresh?: boolean // Force a refresh of statistics from storage
|
||||
} = {}): Promise<{
|
||||
} = {}
|
||||
): Promise<{
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
|
|
@ -2463,8 +2578,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Filter by service if specified
|
||||
const services = options.service
|
||||
? (Array.isArray(options.service) ? options.service : [options.service])
|
||||
: Object.keys({...stats.nounCount, ...stats.verbCount, ...stats.metadataCount})
|
||||
? Array.isArray(options.service)
|
||||
? options.service
|
||||
: [options.service]
|
||||
: Object.keys({
|
||||
...stats.nounCount,
|
||||
...stats.verbCount,
|
||||
...stats.metadataCount
|
||||
})
|
||||
|
||||
// Calculate totals and service breakdown
|
||||
for (const service of services) {
|
||||
|
|
@ -2583,6 +2704,32 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
*/
|
||||
public setReadOnly(readOnly: boolean): void {
|
||||
this.readOnly = readOnly
|
||||
|
||||
// Ensure readOnly and writeOnly are not both true
|
||||
if (readOnly && this.writeOnly) {
|
||||
this.writeOnly = false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the database is in write-only mode
|
||||
* @returns True if the database is in write-only mode, false otherwise
|
||||
*/
|
||||
public isWriteOnly(): boolean {
|
||||
return this.writeOnly
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the database to write-only mode
|
||||
* @param writeOnly True to set the database to write-only mode, false to allow searches
|
||||
*/
|
||||
public setWriteOnly(writeOnly: boolean): void {
|
||||
this.writeOnly = writeOnly
|
||||
|
||||
// Ensure readOnly and writeOnly are not both true
|
||||
if (writeOnly && this.readOnly) {
|
||||
this.readOnly = false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -2621,6 +2768,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
): Promise<Array<GraphVerb & { similarity: number }>> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
try {
|
||||
let queryVector: Vector
|
||||
|
||||
|
|
@ -2677,7 +2827,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// If we didn't get enough results from the index, fall back to the old method
|
||||
if (verbResults.length < k) {
|
||||
console.warn('Not enough verb results from HNSW index, falling back to manual search')
|
||||
console.warn(
|
||||
'Not enough verb results from HNSW index, falling back to manual search'
|
||||
)
|
||||
|
||||
// Get verbs to search through
|
||||
let verbs: GraphVerb[] = []
|
||||
|
|
@ -2700,9 +2852,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
// Calculate similarity for each verb not already in results
|
||||
const existingIds = new Set(verbResults.map(v => v.id))
|
||||
const existingIds = new Set(verbResults.map((v) => v.id))
|
||||
for (const verb of verbs) {
|
||||
if (!existingIds.has(verb.id) && verb.vector && verb.vector.length > 0) {
|
||||
if (
|
||||
!existingIds.has(verb.id) &&
|
||||
verb.vector &&
|
||||
verb.vector.length > 0
|
||||
) {
|
||||
const distance = this.index.getDistanceFunction()(
|
||||
queryVector,
|
||||
verb.vector
|
||||
|
|
@ -2744,6 +2900,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
try {
|
||||
// First, search for nouns
|
||||
const nounResults = await this.searchByNounTypes(
|
||||
|
|
@ -2864,6 +3023,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
try {
|
||||
// Embed the query text
|
||||
const queryVector = await this.embed(query)
|
||||
|
|
@ -2900,6 +3062,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
// Check if connected to a remote server
|
||||
if (!this.isConnectedToRemoteServer()) {
|
||||
throw new Error(
|
||||
|
|
@ -2962,6 +3127,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
// Check if connected to a remote server
|
||||
if (!this.isConnectedToRemoteServer()) {
|
||||
// If not connected to a remote server, just search locally
|
||||
|
|
@ -3221,7 +3389,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
try {
|
||||
await this.flushStatistics()
|
||||
} catch (statsError) {
|
||||
console.warn('Failed to flush statistics during shutdown:', statsError)
|
||||
console.warn(
|
||||
'Failed to flush statistics during shutdown:',
|
||||
statsError
|
||||
)
|
||||
// Continue with shutdown even if statistics flush fails
|
||||
}
|
||||
}
|
||||
|
|
@ -3547,14 +3718,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const nounIds: string[] = []
|
||||
const nounDescriptions: Record<string, string> = {
|
||||
[NounType.Person]: 'A person with unique characteristics',
|
||||
[NounType.Place]: 'A location with specific attributes',
|
||||
[NounType.Location]: 'A location with specific attributes',
|
||||
[NounType.Thing]: 'An object with distinct properties',
|
||||
[NounType.Event]: 'An occurrence with temporal aspects',
|
||||
[NounType.Concept]: 'An abstract idea or notion',
|
||||
[NounType.Content]: 'A piece of content or information',
|
||||
[NounType.Group]: 'A collection of related entities',
|
||||
[NounType.List]: 'An ordered sequence of items',
|
||||
[NounType.Category]: 'A classification or grouping'
|
||||
[NounType.Collection]: 'A collection of related entities',
|
||||
[NounType.Organization]: 'An organization or institution',
|
||||
[NounType.Document]: 'A document or text-based file'
|
||||
}
|
||||
|
||||
for (let i = 0; i < nounCount; i++) {
|
||||
|
|
@ -3585,10 +3756,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const verbIds: string[] = []
|
||||
const verbDescriptions: Record<string, string> = {
|
||||
[VerbType.AttributedTo]: 'Attribution relationship',
|
||||
[VerbType.Controls]: 'Control relationship',
|
||||
[VerbType.Created]: 'Creation relationship',
|
||||
[VerbType.Earned]: 'Achievement relationship',
|
||||
[VerbType.Owns]: 'Ownership relationship',
|
||||
[VerbType.Creates]: 'Creation relationship',
|
||||
[VerbType.Uses]: 'Utilization relationship',
|
||||
[VerbType.BelongsTo]: 'Belonging relationship',
|
||||
[VerbType.MemberOf]: 'Membership relationship',
|
||||
[VerbType.RelatedTo]: 'General relationship',
|
||||
[VerbType.WorksWith]: 'Collaboration relationship',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue