feat: Add Google Cloud Storage CDN for models.soulcraft.com

- Primary source: models.soulcraft.com on GCS
- Backup: GitHub releases
- Fallback: Hugging Face
- Immutable with SHA256 verification
This commit is contained in:
David Snelling 2025-08-18 18:58:26 -07:00
parent a9c5fd0eeb
commit 1096d285f5
13 changed files with 1058 additions and 15 deletions

View file

@ -1271,7 +1271,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// CRITICAL: Ensure model is available before ANY operations
// This is THE most critical part of the system
// Without the model, users CANNOT access their data
if (this.embeddingFunction) {
if (typeof this.embeddingFunction === 'function') {
try {
const { modelGuardian } = await import('./critical/model-guardian.js')
await modelGuardian.ensureCriticalModel()

View file

@ -28,19 +28,19 @@ const CRITICAL_MODEL_CONFIG = {
modelSize: {
'onnx/model.onnx': 90555481, // Exact size in bytes
'tokenizer.json': 711661
},
} as Record<string, number>,
embeddingDimensions: 384,
fallbackSources: [
// Primary: Our GitHub releases (we control this)
// Primary: Our Google Cloud Storage CDN (we control this, fastest)
{
name: 'GitHub (Primary)',
url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
name: 'Soulcraft CDN (Primary)',
url: 'https://models.soulcraft.com/models/all-MiniLM-L6-v2.tar.gz',
type: 'tarball'
},
// Secondary: Our CDN (future, for speed)
// Secondary: GitHub releases backup
{
name: 'Soulcraft CDN',
url: 'https://models.soulcraft.com/brainy/v1/all-MiniLM-L6-v2.tar.gz',
name: 'GitHub Backup',
url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
type: 'tarball'
},
// Tertiary: Hugging Face (original source)
@ -125,7 +125,7 @@ export class ModelGuardian {
return
}
} catch (error) {
console.warn(`${source.name} failed:`, error.message)
console.warn(`${source.name} failed:`, (error as Error).message)
}
}

View file

@ -126,7 +126,7 @@ export class ModelManager {
}
private async verifyModelFiles(modelPath: string, modelName: string): Promise<boolean> {
const manifest = MODEL_MANIFEST[modelName]
const manifest = (MODEL_MANIFEST as any)[modelName]
if (!manifest) return false
for (const [filePath, info] of Object.entries(manifest.files)) {
@ -140,7 +140,7 @@ export class ModelManager {
const stats = await import('fs').then(fs =>
fs.promises.stat(fullPath)
)
if (stats.size !== info.size) {
if (stats.size !== (info as any).size) {
console.warn(`⚠️ Model file size mismatch: ${filePath}`)
return false
}
@ -169,7 +169,7 @@ export class ModelManager {
return false
} catch (error) {
console.log('⚠️ GitHub download failed:', error.message)
console.log('⚠️ GitHub download failed:', (error as Error).message)
return false
}
}
@ -190,7 +190,7 @@ export class ModelManager {
return false
} catch (error) {
console.log('⚠️ CDN download failed:', error.message)
console.log('⚠️ CDN download failed:', (error as Error).message)
return false
}
}