- Add logging configuration to BrainyData for better control over verbose output.

- Refactor embedding functions to support customizable verbosity through new helper methods (`getDefaultEmbeddingFunction`, `getDefaultBatchEmbeddingFunction`).
- Ensure metadata initialization with default values when null during search results.
This commit is contained in:
David Snelling 2025-07-18 10:40:37 -07:00
parent 51892e46b8
commit d3e6dbd898
2 changed files with 152 additions and 35 deletions

View file

@ -25,6 +25,8 @@ import {
cosineDistance, cosineDistance,
defaultEmbeddingFunction, defaultEmbeddingFunction,
defaultBatchEmbeddingFunction, defaultBatchEmbeddingFunction,
getDefaultEmbeddingFunction,
getDefaultBatchEmbeddingFunction,
euclideanDistance, euclideanDistance,
cleanupWorkerPools cleanupWorkerPools
} from './utils/index.js' } from './utils/index.js'
@ -128,6 +130,18 @@ export interface BrainyDataConfig {
*/ */
autoConnect?: boolean autoConnect?: boolean
} }
/**
* Logging configuration
*/
logging?: {
/**
* Whether to enable verbose logging
* When false, suppresses non-essential log messages like model loading progress
* Default: true
*/
verbose?: boolean
}
} }
export class BrainyData<T = any> implements BrainyDataInterface<T> { export class BrainyData<T = any> implements BrainyDataInterface<T> {
@ -142,6 +156,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private storageConfig: BrainyDataConfig['storage'] = {} private storageConfig: BrainyDataConfig['storage'] = {}
private useOptimizedIndex: boolean = false private useOptimizedIndex: boolean = false
private _dimensions: number private _dimensions: number
private loggingConfig: BrainyDataConfig['logging'] = { verbose: true }
// Remote server properties // Remote server properties
private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null
@ -203,9 +218,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Set storage if provided, otherwise it will be initialized in init() // Set storage if provided, otherwise it will be initialized in init()
this.storage = config.storageAdapter || null this.storage = config.storageAdapter || null
// Set embedding function if provided, otherwise use default // Store logging configuration
this.embeddingFunction = if (config.logging !== undefined) {
config.embeddingFunction || defaultEmbeddingFunction this.loggingConfig = {
...this.loggingConfig,
...config.logging
}
}
// Set embedding function if provided, otherwise create one with the appropriate verbose setting
if (config.embeddingFunction) {
this.embeddingFunction = config.embeddingFunction
} else {
this.embeddingFunction = getDefaultEmbeddingFunction({
verbose: this.loggingConfig.verbose
})
}
// Set persistent storage request flag // Set persistent storage request flag
this.requestPersistentStorage = this.requestPersistentStorage =
@ -806,13 +834,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
continue continue
} }
const metadata = await this.storage!.getMetadata(id) let metadata = await this.storage!.getMetadata(id)
// Initialize metadata to an empty object if it's null
if (metadata === null) {
metadata = {} as T
}
searchResults.push({ searchResults.push({
id, id,
score, score,
vector: noun.vector, vector: noun.vector,
metadata: metadata as T | undefined metadata: metadata as T
}) })
} }
@ -855,13 +888,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
continue continue
} }
const metadata = await this.storage!.getMetadata(id) let metadata = await this.storage!.getMetadata(id)
// Initialize metadata to an empty object if it's null
if (metadata === null) {
metadata = {} as T
}
searchResults.push({ searchResults.push({
id, id,
score, score,
vector: noun.vector, vector: noun.vector,
metadata: metadata as T | undefined metadata: metadata as T
}) })
} }

View file

@ -19,6 +19,15 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
private tf: any = null private tf: any = null
private use: any = null private use: any = null
private backend: string = 'cpu' // Default to CPU private backend: string = 'cpu' // Default to CPU
private verbose: boolean = true // Whether to log non-essential messages
/**
* Create a new UniversalSentenceEncoder instance
* @param options Configuration options
*/
constructor(options: { verbose?: boolean } = {}) {
this.verbose = options.verbose !== undefined ? options.verbose : true
}
/** /**
* Add polyfills and patches for TensorFlow.js compatibility * Add polyfills and patches for TensorFlow.js compatibility
@ -93,14 +102,18 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
} }
/** /**
* Log message only if not in test environment * Log message only if verbose mode is enabled or if it's an error
* This helps suppress non-essential log messages
*/ */
private logger( private logger(
level: 'log' | 'warn' | 'error', level: 'log' | 'warn' | 'error',
message: string, message: string,
...args: any[] ...args: any[]
): void { ): void {
console[level](message, ...args) // Always log errors, but only log other messages if verbose mode is enabled
if (level === 'error' || this.verbose) {
console[level](message, ...args)
}
} }
/** /**
@ -580,14 +593,20 @@ function isTestEnvironment(): boolean {
} }
/** /**
* Log message only if not in test environment (standalone version) * Log message only if not in test environment and verbose mode is enabled (standalone version)
* @param level Log level ('log', 'warn', 'error')
* @param message Message to log
* @param args Additional arguments to log
* @param verbose Whether to log non-essential messages (default: true)
*/ */
function logIfNotTest( function logIfNotTest(
level: 'log' | 'warn' | 'error', level: 'log' | 'warn' | 'error',
message: string, message: string,
...args: any[] args: any[] = [],
verbose: boolean = true
): void { ): void {
if (!isTestEnvironment()) { // Always log errors, but only log other messages if verbose mode is enabled
if ((level === 'error' || verbose) && !isTestEnvironment()) {
console[level](message, ...args) console[level](message, ...args)
} }
} }
@ -613,18 +632,31 @@ export function createEmbeddingFunction(
* Creates a TensorFlow-based Universal Sentence Encoder embedding function * Creates a TensorFlow-based Universal Sentence Encoder embedding function
* This is the required embedding function for all text embeddings * This is the required embedding function for all text embeddings
* Uses a shared model instance for better performance across multiple calls * Uses a shared model instance for better performance across multiple calls
* @param options Configuration options
* @param options.verbose Whether to log non-essential messages (default: true)
*/ */
// Create a single shared instance of the model that persists across all embedding calls // Create a single shared instance of the model that persists across all embedding calls
const sharedModel = new UniversalSentenceEncoder() let sharedModel: UniversalSentenceEncoder | null = null
let sharedModelInitialized = false let sharedModelInitialized = false
let sharedModelVerbose = true
export function createTensorFlowEmbeddingFunction(options: { verbose?: boolean } = {}): EmbeddingFunction {
// Update verbose setting if provided
if (options.verbose !== undefined) {
sharedModelVerbose = options.verbose
}
// Create the shared model if it doesn't exist yet
if (!sharedModel) {
sharedModel = new UniversalSentenceEncoder({ verbose: sharedModelVerbose })
}
export function createTensorFlowEmbeddingFunction(): EmbeddingFunction {
return async (data: any): Promise<Vector> => { return async (data: any): Promise<Vector> => {
try { try {
// Initialize the model if it hasn't been initialized yet // Initialize the model if it hasn't been initialized yet
if (!sharedModelInitialized) { if (!sharedModelInitialized) {
try { try {
await sharedModel.init() await sharedModel!.init()
sharedModelInitialized = true sharedModelInitialized = true
} catch (initError) { } catch (initError) {
// Reset the flag so we can retry initialization on the next call // Reset the flag so we can retry initialization on the next call
@ -633,9 +665,9 @@ export function createTensorFlowEmbeddingFunction(): EmbeddingFunction {
} }
} }
return await sharedModel.embed(data) return await sharedModel!.embed(data)
} catch (error) { } catch (error) {
logIfNotTest('error', 'Failed to use TensorFlow embedding:', error) logIfNotTest('error', 'Failed to use TensorFlow embedding:', [error], sharedModelVerbose)
throw new Error( throw new Error(
`Universal Sentence Encoder is required but failed: ${error}` `Universal Sentence Encoder is required but failed: ${error}`
) )
@ -648,40 +680,87 @@ export function createTensorFlowEmbeddingFunction(): EmbeddingFunction {
* Uses UniversalSentenceEncoder for all text embeddings * Uses UniversalSentenceEncoder for all text embeddings
* TensorFlow.js is required for this to work * TensorFlow.js is required for this to work
* Uses CPU for compatibility * Uses CPU for compatibility
* @param options Configuration options
* @param options.verbose Whether to log non-essential messages (default: true)
*/ */
export const defaultEmbeddingFunction: EmbeddingFunction = export function getDefaultEmbeddingFunction(options: { verbose?: boolean } = {}): EmbeddingFunction {
createTensorFlowEmbeddingFunction() return createTensorFlowEmbeddingFunction(options)
}
/** /**
* Default batch embedding function * Default embedding function with default options
* Uses UniversalSentenceEncoder for all text embeddings * Uses UniversalSentenceEncoder for all text embeddings
* TensorFlow.js is required for this to work * TensorFlow.js is required for this to work
* Uses CPU for compatibility
*/
export const defaultEmbeddingFunction: EmbeddingFunction = getDefaultEmbeddingFunction()
/**
* Creates a batch embedding function that uses UniversalSentenceEncoder
* TensorFlow.js is required for this to work
* Processes all items in a single batch operation * Processes all items in a single batch operation
* Uses a shared model instance for better performance across multiple calls * Uses a shared model instance for better performance across multiple calls
* @param options Configuration options
* @param options.verbose Whether to log non-essential messages (default: true)
*/ */
// Create a single shared instance of the model that persists across function calls // Create a single shared instance of the model that persists across function calls
const sharedBatchModel = new UniversalSentenceEncoder() let sharedBatchModel: UniversalSentenceEncoder | null = null
let sharedBatchModelInitialized = false let sharedBatchModelInitialized = false
let sharedBatchModelVerbose = true
export const defaultBatchEmbeddingFunction: ( export function createBatchEmbeddingFunction(options: { verbose?: boolean } = {}): (
dataArray: string[] dataArray: string[]
) => Promise<Vector[]> = async (dataArray: string[]): Promise<Vector[]> => { ) => Promise<Vector[]> {
try { // Update verbose setting if provided
// Initialize the model if it hasn't been initialized yet if (options.verbose !== undefined) {
if (!sharedBatchModelInitialized) { sharedBatchModelVerbose = options.verbose
await sharedBatchModel.init() }
sharedBatchModelInitialized = true
}
return await sharedBatchModel.embedBatch(dataArray) // Create the shared model if it doesn't exist yet
} catch (error) { if (!sharedBatchModel) {
logIfNotTest('error', 'Failed to use TensorFlow batch embedding:', error) sharedBatchModel = new UniversalSentenceEncoder({ verbose: sharedBatchModelVerbose })
throw new Error( }
`Universal Sentence Encoder batch embedding failed: ${error}`
) return async (dataArray: string[]): Promise<Vector[]> => {
try {
// Initialize the model if it hasn't been initialized yet
if (!sharedBatchModelInitialized) {
await sharedBatchModel!.init()
sharedBatchModelInitialized = true
}
return await sharedBatchModel!.embedBatch(dataArray)
} catch (error) {
logIfNotTest('error', 'Failed to use TensorFlow batch embedding:', [error], sharedBatchModelVerbose)
throw new Error(
`Universal Sentence Encoder batch embedding failed: ${error}`
)
}
} }
} }
/**
* Get a batch embedding function with custom options
* Uses UniversalSentenceEncoder for all text embeddings
* TensorFlow.js is required for this to work
* Processes all items in a single batch operation
* @param options Configuration options
* @param options.verbose Whether to log non-essential messages (default: true)
*/
export function getDefaultBatchEmbeddingFunction(options: { verbose?: boolean } = {}): (
dataArray: string[]
) => Promise<Vector[]> {
return createBatchEmbeddingFunction(options)
}
/**
* Default batch embedding function with default options
* Uses UniversalSentenceEncoder for all text embeddings
* TensorFlow.js is required for this to work
* Processes all items in a single batch operation
*/
export const defaultBatchEmbeddingFunction = getDefaultBatchEmbeddingFunction()
/** /**
* Creates an embedding function that runs in a separate thread * Creates an embedding function that runs in a separate thread
* This is a wrapper around createEmbeddingFunction that uses executeInThread * This is a wrapper around createEmbeddingFunction that uses executeInThread