soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**feat(docs): add compatibility and testing guides; enforce Universal Sentence Encoder usage**

Browse source 
- Added new documentation files:
  - `COMPATIBILITY.md` detailing environment-specific compatibility and behavior (Node.js, Browser, Worker).
  - `TESTING.md` providing instructions for verifying cache detection across environments.
  - Created browser (`test-browser-cache-detection.html`) and worker (`test-worker-cache-detection.html`) test scripts to validate cache mechanisms.

- Removed fallback mechanisms for embedding:
  - Updated `embedding.ts` to enforce strict usage of Universal Sentence Encoder (USE).
  - Fallback methods (`generateFallbackVector`) and related logic have been removed.
  - Errors are thrown when USE initialization or embedding fails, ensuring stricter reliability.

- Improved error handling:
  - Standardized error throwing for all USE-related failures across single and batch embeddings.
  - Logging updated to reflect critical embedding issues without allowing degraded operations.

**Purpose**: Improve documentation for environment compatibility and testing while enforcing consistent use of Universal Sentence Encoder for deterministic embeddings, removing unreliable fallback mechanisms.
This commit is contained in:
David Snelling 2025-08-01 11:02:01 -07:00
parent 6ee0881d86
commit 885a8b403a
6 changed files with 585 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

148
src/utils/embedding.ts
View file

@ -362,11 +362,8 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
this.use = await import('@tensorflow-models/universal-sentence-encoder')
} catch (error) {
this.logger('error', 'Failed to initialize TensorFlow.js:', error)
// Don't throw here, we'll use a fallback mechanism
this.logger('warn', 'Will use fallback embedding mechanism')
// Mark as initialized with fallback
this.initialized = true
return
// No fallback allowed - throw error
throw new Error(`Universal Sentence Encoder initialization failed: ${error}`)
}
// Set the backend
@ -374,16 +371,12 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
await this.tf.setBackend(this.backend)
}
// Module structure available for debugging if needed
// Try to find the load function in different possible module structures
const loadFunction = findUSELoadFunction(this.use)
if (!loadFunction) {
this.logger('warn', 'Could not find Universal Sentence Encoder load function, using fallback')
// Mark as initialized with fallback
this.initialized = true
return
this.logger('error', 'Could not find Universal Sentence Encoder load function')
throw new Error('Universal Sentence Encoder load function not found. Fallback mechanisms are not allowed.')
}
try {
@ -392,12 +385,12 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
this.initialized = true
} catch (modelError) {
this.logger(
'warn',
'Failed to load Universal Sentence Encoder model, using fallback:',
'error',
'Failed to load Universal Sentence Encoder model:',
modelError
)
// Mark as initialized with fallback
this.initialized = true
// No fallback allowed - throw error
throw new Error(`Universal Sentence Encoder model loading failed: ${modelError}`)
}
// Restore original console.warn
@ -408,10 +401,8 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
'Failed to initialize Universal Sentence Encoder:',
error
)
// Don't throw, use fallback mechanism
this.logger('warn', 'Using fallback embedding mechanism due to initialization failure')
// Mark as initialized with fallback
this.initialized = true
// No fallback allowed - throw error
throw new Error(`Universal Sentence Encoder initialization failed: ${error}`)
}
}
@ -420,51 +411,12 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
* @param data Text to embed
*/
/**
* Generate a deterministic vector from a string
* This is used as a fallback when the Universal Sentence Encoder is not available
* @param text Input text
* @returns A 512-dimensional vector derived from the text
* This method has been removed as we should always use Universal Sentence Encoder
* and never fall back to alternative vector generation methods
* @deprecated
*/
private generateFallbackVector(text: string): Vector {
// Create a deterministic vector based on the text
const vector = new Array(512).fill(0)
if (!text || text.trim() === '') {
return vector
}
// Simple hash function to generate a number from a string
const hash = (str: string): number => {
let h = 0
for (let i = 0; i < str.length; i++) {
h = ((h << 5) - h) + str.charCodeAt(i)
h |= 0 // Convert to 32bit integer
}
return h
}
// Generate values based on the text
const words = text.split(/\s+/)
for (let i = 0; i < words.length && i < 512; i++) {
const word = words[i]
if (word) {
const h = hash(word)
// Use the hash to set a value in the vector
const index = Math.abs(h) % 512
vector[index] = (h % 1000) / 1000 // Value between -1 and 1
}
}
// Ensure the vector has some values even for short texts
if (text.length > 0) {
const h = hash(text)
for (let i = 0; i < 10; i++) {
const index = (Math.abs(h) + i * 50) % 512
vector[index] = ((h + i) % 1000) / 1000
}
}
return vector
throw new Error('Fallback vector generation is not allowed. Universal Sentence Encoder must be used for all embeddings.')
}
public async embed(data: string | string[]): Promise<Vector> {
@ -501,13 +453,11 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
)
}
// Check if we need to use the fallback mechanism
// Ensure the model is available - no fallbacks allowed
if (!this.model) {
this.logger(
'warn',
'Using fallback embedding mechanism (model not available)'
throw new Error(
'Universal Sentence Encoder model is not available. Fallback mechanisms are not allowed.'
)
return this.generateFallbackVector(textToEmbed[0])
}
// Get embeddings
@ -556,19 +506,13 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
return embedding
} catch (error) {
// No fallback - throw the error
this.logger(
'warn',
'Failed to embed text with Universal Sentence Encoder, using fallback:',
'error',
'Failed to embed text with Universal Sentence Encoder:',
error
)
// Use fallback mechanism instead of throwing
if (typeof data === 'string') {
return this.generateFallbackVector(data)
} else if (Array.isArray(data) && data.length > 0) {
return this.generateFallbackVector(data[0])
} else {
return new Array(512).fill(0)
}
throw new Error(`Universal Sentence Encoder embedding failed: ${error}`)
}
}
@ -599,20 +543,11 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
return dataArray.map(() => new Array(512).fill(0))
}
// Check if we need to use the fallback mechanism
// Ensure the model is available - no fallbacks allowed
if (!this.model) {
this.logger(
'warn',
'Using fallback embedding mechanism for batch (model not available)'
throw new Error(
'Universal Sentence Encoder model is not available. Fallback mechanisms are not allowed.'
)
// Generate fallback vectors for each text
return dataArray.map(text => {
if (typeof text === 'string' && text.trim() !== '') {
return this.generateFallbackVector(text)
} else {
return new Array(512).fill(0)
}
})
}
// Get embeddings for all texts in a single batch operation
@ -677,20 +612,13 @@ export class UniversalSentenceEncoder implements EmbeddingModel {
return results
} catch (error) {
// No fallback - throw the error
this.logger(
'warn',
'Failed to batch embed text with Universal Sentence Encoder, using fallback:',
'error',
'Failed to batch embed text with Universal Sentence Encoder:',
error
)
// Use fallback mechanism instead of throwing
return dataArray.map(text => {
if (typeof text === 'string' && text.trim() !== '') {
return this.generateFallbackVector(text)
} else {
return new Array(512).fill(0)
}
})
throw new Error(`Universal Sentence Encoder batch embedding failed: ${error}`)
}
}
@ -904,9 +832,10 @@ export function createTensorFlowEmbeddingFunction(options: { verbose?: boolean }
return await sharedModel!.embed(data)
} catch (error) {
logIfNotTest('error', 'Failed to use TensorFlow embedding:', [error], sharedModelVerbose)
logIfNotTest('error', 'Failed to use Universal Sentence Encoder:', [error], sharedModelVerbose)
// No fallback - Universal Sentence Encoder is required
throw new Error(
`Universal Sentence Encoder is required but failed: ${error}`
`Universal Sentence Encoder is required and no fallbacks are allowed: ${error}`
)
}
}
@ -962,15 +891,22 @@ export function createBatchEmbeddingFunction(options: { verbose?: boolean } = {}
try {
// Initialize the model if it hasn't been initialized yet
if (!sharedBatchModelInitialized) {
await sharedBatchModel!.init()
sharedBatchModelInitialized = true
try {
await sharedBatchModel!.init()
sharedBatchModelInitialized = true
} catch (initError) {
// Reset the flag so we can retry initialization on the next call
sharedBatchModelInitialized = false
throw initError
}
}
return await sharedBatchModel!.embedBatch(dataArray)
} catch (error) {
logIfNotTest('error', 'Failed to use TensorFlow batch embedding:', [error], sharedBatchModelVerbose)
logIfNotTest('error', 'Failed to use Universal Sentence Encoder batch embedding:', [error], sharedBatchModelVerbose)
// No fallback - Universal Sentence Encoder is required
throw new Error(
`Universal Sentence Encoder batch embedding failed: ${error}`
`Universal Sentence Encoder is required for batch embedding and no fallbacks are allowed: ${error}`
)
}
}