feat: Add Google Cloud Storage CDN for models.soulcraft.com
- Primary source: models.soulcraft.com on GCS - Backup: GitHub releases - Fallback: Hugging Face - Immutable with SHA256 verification
This commit is contained in:
parent
a9c5fd0eeb
commit
1096d285f5
13 changed files with 1058 additions and 15 deletions
|
|
@ -1271,7 +1271,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// CRITICAL: Ensure model is available before ANY operations
|
||||
// This is THE most critical part of the system
|
||||
// Without the model, users CANNOT access their data
|
||||
if (this.embeddingFunction) {
|
||||
if (typeof this.embeddingFunction === 'function') {
|
||||
try {
|
||||
const { modelGuardian } = await import('./critical/model-guardian.js')
|
||||
await modelGuardian.ensureCriticalModel()
|
||||
|
|
|
|||
|
|
@ -28,19 +28,19 @@ const CRITICAL_MODEL_CONFIG = {
|
|||
modelSize: {
|
||||
'onnx/model.onnx': 90555481, // Exact size in bytes
|
||||
'tokenizer.json': 711661
|
||||
},
|
||||
} as Record<string, number>,
|
||||
embeddingDimensions: 384,
|
||||
fallbackSources: [
|
||||
// Primary: Our GitHub releases (we control this)
|
||||
// Primary: Our Google Cloud Storage CDN (we control this, fastest)
|
||||
{
|
||||
name: 'GitHub (Primary)',
|
||||
url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
|
||||
name: 'Soulcraft CDN (Primary)',
|
||||
url: 'https://models.soulcraft.com/models/all-MiniLM-L6-v2.tar.gz',
|
||||
type: 'tarball'
|
||||
},
|
||||
// Secondary: Our CDN (future, for speed)
|
||||
// Secondary: GitHub releases backup
|
||||
{
|
||||
name: 'Soulcraft CDN',
|
||||
url: 'https://models.soulcraft.com/brainy/v1/all-MiniLM-L6-v2.tar.gz',
|
||||
name: 'GitHub Backup',
|
||||
url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
|
||||
type: 'tarball'
|
||||
},
|
||||
// Tertiary: Hugging Face (original source)
|
||||
|
|
@ -125,7 +125,7 @@ export class ModelGuardian {
|
|||
return
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`❌ ${source.name} failed:`, error.message)
|
||||
console.warn(`❌ ${source.name} failed:`, (error as Error).message)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ export class ModelManager {
|
|||
}
|
||||
|
||||
private async verifyModelFiles(modelPath: string, modelName: string): Promise<boolean> {
|
||||
const manifest = MODEL_MANIFEST[modelName]
|
||||
const manifest = (MODEL_MANIFEST as any)[modelName]
|
||||
if (!manifest) return false
|
||||
|
||||
for (const [filePath, info] of Object.entries(manifest.files)) {
|
||||
|
|
@ -140,7 +140,7 @@ export class ModelManager {
|
|||
const stats = await import('fs').then(fs =>
|
||||
fs.promises.stat(fullPath)
|
||||
)
|
||||
if (stats.size !== info.size) {
|
||||
if (stats.size !== (info as any).size) {
|
||||
console.warn(`⚠️ Model file size mismatch: ${filePath}`)
|
||||
return false
|
||||
}
|
||||
|
|
@ -169,7 +169,7 @@ export class ModelManager {
|
|||
return false
|
||||
|
||||
} catch (error) {
|
||||
console.log('⚠️ GitHub download failed:', error.message)
|
||||
console.log('⚠️ GitHub download failed:', (error as Error).message)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -190,7 +190,7 @@ export class ModelManager {
|
|||
return false
|
||||
|
||||
} catch (error) {
|
||||
console.log('⚠️ CDN download failed:', error.message)
|
||||
console.log('⚠️ CDN download failed:', (error as Error).message)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue