soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: enhance storage options and introduce read-only mode

Browse source 
Added extensive configuration support for storage adapters, including S3-compatible storage (R2, Amazon S3, Google Cloud Storage). Introduced environment variable fallback for seamless cloud integration. Updated createStorage logic for improved compatibility. Implemented read-only mode to restrict write operations and provide a safer database configuration.
This commit is contained in:
David Snelling 2025-06-04 10:01:59 -07:00
parent a774bb0f82
commit 811bbbdbfd
3 changed files with 1326 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

112
src/brainyData.ts
View file

@ -34,6 +34,41 @@ export interface BrainyDataConfig {
*/
storageAdapter?: StorageAdapter
/**
* Storage configuration options
* These will be passed to createStorage if storageAdapter is not provided
*/
storage?: {
requestPersistentStorage?: boolean;
r2Storage?: {
bucketName?: string;
accountId?: string;
accessKeyId?: string;
secretAccessKey?: string;
};
s3Storage?: {
bucketName?: string;
accessKeyId?: string;
secretAccessKey?: string;
region?: string;
};
gcsStorage?: {
bucketName?: string;
accessKeyId?: string;
secretAccessKey?: string;
endpoint?: string;
};
customS3Storage?: {
bucketName?: string;
accessKeyId?: string;
secretAccessKey?: string;
endpoint?: string;
region?: string;
};
forceFileSystemStorage?: boolean;
forceMemoryStorage?: boolean;
}
/**
* Embedding function to convert data to vectors
*/
@ -42,8 +77,15 @@ export interface BrainyDataConfig {
/**
* Request persistent storage when running in a browser
* This will prompt the user for permission to use persistent storage
* @deprecated Use storage.requestPersistentStorage instead
*/
requestPersistentStorage?: boolean
/**
* Set the database to read-only mode
* When true, all write operations will throw an error
*/
readOnly?: boolean
}
export class BrainyData<T = any> {
@ -52,6 +94,8 @@ export class BrainyData<T = any> {
private isInitialized = false
private embeddingFunction: EmbeddingFunction
private requestPersistentStorage: boolean
private readOnly: boolean
private storageConfig: BrainyDataConfig['storage'] = {}
/**
* Create a new vector database
@ -69,8 +113,27 @@ export class BrainyData<T = any> {
// Set embedding function if provided, otherwise use default
this.embeddingFunction = config.embeddingFunction || defaultEmbeddingFunction
// Set persistent storage request flag
this.requestPersistentStorage = config.requestPersistentStorage || false
// Set persistent storage request flag (support both new and deprecated options)
this.requestPersistentStorage =
(config.storage?.requestPersistentStorage !== undefined)
? config.storage.requestPersistentStorage
: (config.requestPersistentStorage || false)
// Set read-only flag
this.readOnly = config.readOnly || false
// Store storage configuration for later use in init()
this.storageConfig = config.storage || {}
}
/**
* Check if the database is in read-only mode and throw an error if it is
* @throws Error if the database is in read-only mode
*/
private checkReadOnly(): void {
if (this.readOnly) {
throw new Error('Cannot perform write operation: database is in read-only mode')
}
}
/**
@ -85,9 +148,13 @@ export class BrainyData<T = any> {
try {
// Initialize storage if not provided in constructor
if (!this.storage) {
this.storage = await createStorage({
// Combine storage config with requestPersistentStorage for backward compatibility
const storageOptions = {
...this.storageConfig,
requestPersistentStorage: this.requestPersistentStorage
})
};
this.storage = await createStorage(storageOptions);
}
// Initialize storage
@ -130,6 +197,9 @@ export class BrainyData<T = any> {
): Promise<string> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
try {
let vector: Vector
@ -200,6 +270,9 @@ export class BrainyData<T = any> {
): Promise<string[]> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
const ids: string[] = []
try {
@ -395,6 +468,9 @@ export class BrainyData<T = any> {
public async delete(id: string): Promise<boolean> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
try {
// Remove from index
const removed = this.index.removeItem(id)
@ -425,6 +501,9 @@ export class BrainyData<T = any> {
public async updateMetadata(id: string, metadata: T): Promise<boolean> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
try {
// Check if a vector exists
const node = this.index.getNodes().get(id)
@ -457,6 +536,9 @@ export class BrainyData<T = any> {
): Promise<string> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
try {
// Check if source and target nodes exist
const sourceNode = this.index.getNodes().get(sourceId)
@ -591,6 +673,9 @@ export class BrainyData<T = any> {
public async deleteEdge(id: string): Promise<boolean> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
try {
// Remove from index
const removed = this.index.removeItem(id)
@ -614,6 +699,9 @@ export class BrainyData<T = any> {
public async clear(): Promise<void> {
await this.ensureInitialized()
// Check if database is in read-only mode
this.checkReadOnly()
try {
// Clear index
this.index.clear()
@ -633,6 +721,22 @@ export class BrainyData<T = any> {
return this.index.size()
}
/**
* Check if the database is in read-only mode
* @returns True if the database is in read-only mode, false otherwise
*/
public isReadOnly(): boolean {
return this.readOnly
}
/**
* Set the database to read-only mode
* @param readOnly True to set the database to read-only mode, false to allow writes
*/
public setReadOnly(readOnly: boolean): void {
this.readOnly = readOnly
}
/**
* Embed text or data into a vector using the same embedding function used by this instance
* This allows clients to use the same TensorFlow Universal Sentence Encoder throughout their application

208
src/storage/opfsStorage.ts
View file

@ -1442,12 +1442,164 @@ export class MemoryStorage implements StorageAdapter {
* @returns Promise that resolves to a StorageAdapter instance
*/
export async function createStorage(options: {
requestPersistentStorage?: boolean
requestPersistentStorage?: boolean;
r2Storage?: {
bucketName?: string;
accountId?: string;
accessKeyId?: string;
secretAccessKey?: string;
};
s3Storage?: {
bucketName?: string;
accessKeyId?: string;
secretAccessKey?: string;
region?: string;
};
gcsStorage?: {
bucketName?: string;
accessKeyId?: string;
secretAccessKey?: string;
endpoint?: string;
};
customS3Storage?: {
bucketName?: string;
accessKeyId?: string;
secretAccessKey?: string;
endpoint?: string;
region?: string;
};
forceFileSystemStorage?: boolean;
forceMemoryStorage?: boolean;
} = {}): Promise<StorageAdapter> {
// Check if we're in a Node.js environment
const isNode = typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null
// Check for environment variables for cloud storage (only in Node.js environment)
const isNode = typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null;
// Default empty values
const defaultEnvStorage = {
bucketName: undefined,
accountId: undefined,
accessKeyId: undefined,
secretAccessKey: undefined,
region: undefined,
endpoint: undefined
};
// Only try to access process.env in Node.js environment
const envR2Storage = isNode ? {
bucketName: process.env.R2_BUCKET_NAME,
accountId: process.env.R2_ACCOUNT_ID,
accessKeyId: process.env.R2_ACCESS_KEY_ID,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
} : defaultEnvStorage;
const envS3Storage = isNode ? {
bucketName: process.env.S3_BUCKET_NAME,
accessKeyId: process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY,
region: process.env.S3_REGION || process.env.AWS_REGION
} : defaultEnvStorage;
const envGCSStorage = isNode ? {
bucketName: process.env.GCS_BUCKET_NAME,
accessKeyId: process.env.GCS_ACCESS_KEY_ID,
secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY,
endpoint: process.env.GCS_ENDPOINT
} : defaultEnvStorage;
// Merge environment variables with provided options
const mergedOptions = {
...options,
r2Storage: options.r2Storage ? {
...envR2Storage,
...options.r2Storage
} : (
// Only use environment variables if they have the required fields
envR2Storage.bucketName && envR2Storage.accessKeyId && envR2Storage.secretAccessKey ?
envR2Storage : undefined
),
s3Storage: options.s3Storage ? {
...envS3Storage,
...options.s3Storage
} : (
// Only use environment variables if they have the required fields
envS3Storage.bucketName && envS3Storage.accessKeyId && envS3Storage.secretAccessKey ?
envS3Storage : undefined
),
gcsStorage: options.gcsStorage ? {
...envGCSStorage,
...options.gcsStorage
} : (
// Only use environment variables if they have the required fields
envGCSStorage.bucketName && envGCSStorage.accessKeyId && envGCSStorage.secretAccessKey ?
envGCSStorage : undefined
)
};
// If any S3-compatible storage options are provided, use S3CompatibleStorage
if (mergedOptions.r2Storage || mergedOptions.s3Storage || mergedOptions.gcsStorage || mergedOptions.customS3Storage) {
try {
const s3Module = await import('./s3CompatibleStorage.js')
if (mergedOptions.r2Storage &&
mergedOptions.r2Storage.bucketName &&
mergedOptions.r2Storage.accountId &&
mergedOptions.r2Storage.accessKeyId &&
mergedOptions.r2Storage.secretAccessKey) {
return new s3Module.R2Storage({
bucketName: mergedOptions.r2Storage.bucketName,
accountId: mergedOptions.r2Storage.accountId,
accessKeyId: mergedOptions.r2Storage.accessKeyId,
secretAccessKey: mergedOptions.r2Storage.secretAccessKey
})
} else if (mergedOptions.s3Storage &&
mergedOptions.s3Storage.bucketName &&
mergedOptions.s3Storage.accessKeyId &&
mergedOptions.s3Storage.secretAccessKey) {
return new s3Module.S3CompatibleStorage({
bucketName: mergedOptions.s3Storage.bucketName,
accessKeyId: mergedOptions.s3Storage.accessKeyId,
secretAccessKey: mergedOptions.s3Storage.secretAccessKey,
region: mergedOptions.s3Storage.region,
serviceType: 's3'
})
} else if (mergedOptions.gcsStorage &&
mergedOptions.gcsStorage.bucketName &&
mergedOptions.gcsStorage.accessKeyId &&
mergedOptions.gcsStorage.secretAccessKey) {
return new s3Module.S3CompatibleStorage({
bucketName: mergedOptions.gcsStorage.bucketName,
accessKeyId: mergedOptions.gcsStorage.accessKeyId,
secretAccessKey: mergedOptions.gcsStorage.secretAccessKey,
endpoint: mergedOptions.gcsStorage.endpoint,
serviceType: 'gcs'
})
} else if (mergedOptions.customS3Storage &&
mergedOptions.customS3Storage.bucketName &&
mergedOptions.customS3Storage.accessKeyId &&
mergedOptions.customS3Storage.secretAccessKey) {
return new s3Module.S3CompatibleStorage({
bucketName: mergedOptions.customS3Storage.bucketName,
accessKeyId: mergedOptions.customS3Storage.accessKeyId,
secretAccessKey: mergedOptions.customS3Storage.secretAccessKey,
endpoint: mergedOptions.customS3Storage.endpoint,
region: mergedOptions.customS3Storage.region,
serviceType: 'custom'
})
}
} catch (error) {
console.warn('Failed to load S3CompatibleStorage, falling back to default storage:', error)
// Continue to default storage selection
}
}
// If force memory storage is specified, use MemoryStorage
if (options.forceMemoryStorage) {
return new MemoryStorage()
}
// Reuse the isNode variable from above
if (isNode) {
// In Node.js, use FileSystemStorage first, then fall back to memory
@ -1459,31 +1611,33 @@ export async function createStorage(options: {
return new MemoryStorage()
}
} else {
// In browser, try OPFS first
const opfsStorage = new OPFSStorage()
// In browser, try OPFS first (unless force FileSystem is specified)
if (!options.forceFileSystemStorage) {
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
// Request persistent storage if specified
if (options.requestPersistentStorage) {
const isPersistentGranted = await opfsStorage.requestPersistentStorage()
if (isPersistentGranted) {
console.log('Persistent storage permission granted')
} else {
console.warn('Persistent storage permission denied')
if (opfsStorage.isOPFSAvailable()) {
// Request persistent storage if specified
if (options.requestPersistentStorage) {
const isPersistentGranted = await opfsStorage.requestPersistentStorage()
if (isPersistentGranted) {
console.log('Persistent storage permission granted')
} else {
console.warn('Persistent storage permission denied')
}
}
return opfsStorage
}
return opfsStorage
} else {
// OPFS is not available, try to use FileSystem API if available
try {
// Try to load FileSystemStorage for browser environments
// Note: This will likely fail as FileSystemStorage is designed for Node.js
const fileSystemModule = await import('./fileSystemStorage.js')
return new fileSystemModule.FileSystemStorage()
} catch (error) {
console.warn('FileSystem storage is not available, falling back to in-memory storage')
return new MemoryStorage()
}
}
// OPFS is not available or force FileSystem is specified, try to use FileSystem API if available
try {
// Try to load FileSystemStorage for browser environments
// Note: This will likely fail as FileSystemStorage is designed for Node.js
const fileSystemModule = await import('./fileSystemStorage.js')
return new fileSystemModule.FileSystemStorage()
} catch (error) {
console.warn('FileSystem storage is not available, falling back to in-memory storage')
return new MemoryStorage()
}
}
}

1037
src/storage/s3CompatibleStorage.ts Normal file
View file

File diff suppressed because it is too large Load diff