fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -1,6 +1,6 @@
|
|||
# Capacity Planning & Operations Guide
|
||||
|
||||
**Brainy v3.36.0+ Enterprise Operations**
|
||||
**Brainy Enterprise Operations**
|
||||
|
||||
This guide provides production-ready capacity planning formulas, deployment strategies, and operational guidelines for scaling Brainy from development (2GB) to enterprise (128GB+) deployments.
|
||||
|
||||
|
|
@ -24,13 +24,13 @@ Where:
|
|||
### Adaptive Caching Strategy
|
||||
|
||||
```
|
||||
estimatedVectorMemory = entityCount × 1536 bytes // 384 dims × 4 bytes per float
|
||||
hnswCacheBudget = cacheSize × 0.80 // 80% threshold for preloading decision
|
||||
estimatedVectorMemory = entityCount × 1536 bytes // 384 dims × 4 bytes per float
|
||||
hnswCacheBudget = cacheSize × 0.80 // 80% threshold for preloading decision
|
||||
|
||||
if estimatedVectorMemory < hnswCacheBudget:
|
||||
cachingStrategy = 'preloaded' // All vectors loaded at init
|
||||
cachingStrategy = 'preloaded' // All vectors loaded at init
|
||||
else:
|
||||
cachingStrategy = 'on-demand' // Vectors loaded adaptively via UnifiedCache
|
||||
cachingStrategy = 'on-demand' // Vectors loaded adaptively via UnifiedCache
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -46,19 +46,19 @@ else:
|
|||
|
||||
**Memory Breakdown:**
|
||||
```
|
||||
System Memory: 2048 MB
|
||||
OS Reserved (20%): -410 MB
|
||||
Available: 1638 MB
|
||||
Model Memory: -140 MB
|
||||
├─ WASM + Weights: 90 MB
|
||||
└─ Workspace: 50 MB
|
||||
System Memory: 2048 MB
|
||||
OS Reserved (20%): -410 MB
|
||||
Available: 1638 MB
|
||||
Model Memory: -140 MB
|
||||
├─ WASM + Weights: 90 MB
|
||||
└─ Workspace: 50 MB
|
||||
───────────────────────────
|
||||
Available for Cache: 1488 MB
|
||||
Available for Cache: 1488 MB
|
||||
Dev Allocation (25%): 372 MB UnifiedCache
|
||||
├─ HNSW (30%): 112 MB
|
||||
├─ Metadata (40%): 149 MB
|
||||
├─ Search (20%): 74 MB
|
||||
└─ Shared (10%): 37 MB
|
||||
├─ HNSW (30%): 112 MB
|
||||
├─ Metadata (40%): 149 MB
|
||||
├─ Search (20%): 74 MB
|
||||
└─ Shared (10%): 37 MB
|
||||
```
|
||||
|
||||
**Capacity:**
|
||||
|
|
@ -75,9 +75,9 @@ Dev Allocation (25%): 372 MB UnifiedCache
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'filesystem', path: './brainy-data' },
|
||||
model: { precision: 'q8' },
|
||||
cache: { /* auto-sized to 372MB */ }
|
||||
storage: { type: 'filesystem', path: './brainy-data' },
|
||||
model: { precision: 'q8' },
|
||||
cache: { /* auto-sized to 372MB */ }
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -92,17 +92,17 @@ const brain = new Brainy({
|
|||
|
||||
**Memory Breakdown:**
|
||||
```
|
||||
System Memory: 8192 MB
|
||||
OS Reserved (20%): -1638 MB
|
||||
Available: 6554 MB
|
||||
Model Memory (Q8): -150 MB
|
||||
System Memory: 8192 MB
|
||||
OS Reserved (20%): -1638 MB
|
||||
Available: 6554 MB
|
||||
Model Memory (Q8): -150 MB
|
||||
───────────────────────────
|
||||
Available for Cache: 6404 MB
|
||||
Available for Cache: 6404 MB
|
||||
Prod Allocation (50%): 3202 MB UnifiedCache
|
||||
├─ HNSW (30%): 961 MB
|
||||
├─ Metadata (40%): 1281 MB
|
||||
├─ Search (20%): 640 MB
|
||||
└─ Shared (10%): 320 MB
|
||||
├─ HNSW (30%): 961 MB
|
||||
├─ Metadata (40%): 1281 MB
|
||||
├─ Search (20%): 640 MB
|
||||
└─ Shared (10%): 320 MB
|
||||
```
|
||||
|
||||
**Capacity:**
|
||||
|
|
@ -119,9 +119,9 @@ Prod Allocation (50%): 3202 MB UnifiedCache
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||
model: { precision: 'q8' },
|
||||
// Auto-sized cache: 3202MB
|
||||
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||
model: { precision: 'q8' },
|
||||
// Auto-sized cache: 3202MB
|
||||
})
|
||||
|
||||
// Monitor health
|
||||
|
|
@ -141,17 +141,17 @@ console.log(`Caching strategy: ${stats.cachingStrategy}`)
|
|||
|
||||
**Memory Breakdown:**
|
||||
```
|
||||
System Memory: 32768 MB
|
||||
OS Reserved (20%): -6554 MB
|
||||
Available: 26214 MB
|
||||
Model Memory (Q8): -150 MB
|
||||
System Memory: 32768 MB
|
||||
OS Reserved (20%): -6554 MB
|
||||
Available: 26214 MB
|
||||
Model Memory (Q8): -150 MB
|
||||
───────────────────────────
|
||||
Available for Cache: 26064 MB
|
||||
Available for Cache: 26064 MB
|
||||
Prod Allocation (50%): 13032 MB UnifiedCache
|
||||
├─ HNSW (30%): 3910 MB
|
||||
├─ Metadata (40%): 5213 MB
|
||||
├─ Search (20%): 2606 MB
|
||||
└─ Shared (10%): 1303 MB
|
||||
├─ HNSW (30%): 3910 MB
|
||||
├─ Metadata (40%): 5213 MB
|
||||
├─ Search (20%): 2606 MB
|
||||
└─ Shared (10%): 1303 MB
|
||||
```
|
||||
|
||||
**Capacity:**
|
||||
|
|
@ -168,11 +168,11 @@ Prod Allocation (50%): 13032 MB UnifiedCache
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'gcs-native',
|
||||
gcsNativeStorage: { bucketName: 'production-data' }
|
||||
},
|
||||
model: { precision: 'q8' } // or 'fp32' for +0.5% accuracy
|
||||
storage: {
|
||||
type: 'gcs-native',
|
||||
gcsNativeStorage: { bucketName: 'production-data' }
|
||||
},
|
||||
model: { precision: 'q8' } // or 'fp32' for +0.5% accuracy
|
||||
})
|
||||
|
||||
// Verify allocation
|
||||
|
|
@ -192,17 +192,17 @@ console.log(`Environment: ${memoryInfo.memoryInfo.environment}`)
|
|||
|
||||
**Memory Breakdown:**
|
||||
```
|
||||
System Memory: 131072 MB
|
||||
OS Reserved (20%): -26214 MB
|
||||
Available: 104858 MB
|
||||
Model Memory (FP32): -250 MB
|
||||
System Memory: 131072 MB
|
||||
OS Reserved (20%): -26214 MB
|
||||
Available: 104858 MB
|
||||
Model Memory (FP32): -250 MB
|
||||
───────────────────────────────
|
||||
Available for Cache: 104608 MB
|
||||
Available for Cache: 104608 MB
|
||||
Prod Allocation (50%): 52304 MB UnifiedCache (logarithmic scaling applies)
|
||||
├─ HNSW (30%): 15691 MB
|
||||
├─ Metadata (40%): 20922 MB
|
||||
├─ Search (20%): 10461 MB
|
||||
└─ Shared (10%): 5230 MB
|
||||
├─ HNSW (30%): 15691 MB
|
||||
├─ Metadata (40%): 20922 MB
|
||||
├─ Search (20%): 10461 MB
|
||||
└─ Shared (10%): 5230 MB
|
||||
```
|
||||
|
||||
**Logarithmic Scaling Applied:**
|
||||
|
|
@ -227,30 +227,30 @@ Actual cache size: ~40GB (prevents waste on 128GB systems)
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'enterprise-data',
|
||||
region: 'us-east-1'
|
||||
}
|
||||
},
|
||||
model: { precision: 'fp32' } // Maximum accuracy
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'enterprise-data',
|
||||
region: 'us-east-1'
|
||||
}
|
||||
},
|
||||
model: { precision: 'fp32' } // Maximum accuracy
|
||||
})
|
||||
|
||||
// Enterprise monitoring
|
||||
setInterval(() => {
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
|
||||
if (stats.fairness.fairnessViolation) {
|
||||
console.warn('FAIRNESS VIOLATION: HNSW using too much cache')
|
||||
console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access, ${stats.hnswCache.sizePercent}% size`)
|
||||
}
|
||||
if (stats.fairness.fairnessViolation) {
|
||||
console.warn('FAIRNESS VIOLATION: HNSW using too much cache')
|
||||
console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access, ${stats.hnswCache.sizePercent}% size`)
|
||||
}
|
||||
|
||||
if (stats.unifiedCache.hitRatePercent < 75) {
|
||||
console.warn(`Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`)
|
||||
console.warn('Recommendations:', stats.recommendations)
|
||||
}
|
||||
}, 60000) // Check every minute
|
||||
if (stats.unifiedCache.hitRatePercent < 75) {
|
||||
console.warn(`Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`)
|
||||
console.warn('Recommendations:', stats.recommendations)
|
||||
}
|
||||
}, 60000) // Check every minute
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -263,12 +263,12 @@ Brainy auto-detects container memory limits via cgroups v1/v2:
|
|||
|
||||
```typescript
|
||||
// Automatic detection
|
||||
const brain = new Brainy() // Detects cgroup limits automatically
|
||||
const brain = new Brainy() // Detects cgroup limits automatically
|
||||
|
||||
// Verify detection
|
||||
const memoryInfo = brain.hnsw.unifiedCache.getMemoryInfo()
|
||||
console.log(`Container: ${memoryInfo.memoryInfo.isContainer}`)
|
||||
console.log(`Source: ${memoryInfo.memoryInfo.source}`) // 'cgroup-v2' or 'cgroup-v1'
|
||||
console.log(`Source: ${memoryInfo.memoryInfo.source}`) // 'cgroup-v2' or 'cgroup-v1'
|
||||
console.log(`Limit: ${Math.round(memoryInfo.memoryInfo.available / 1024 / 1024)}MB`)
|
||||
```
|
||||
|
||||
|
|
@ -294,37 +294,37 @@ CMD ["node", "dist/index.js"]
|
|||
|
||||
```bash
|
||||
docker run \
|
||||
--memory="2g" \
|
||||
--memory-reservation="1.5g" \
|
||||
--cpus="2" \
|
||||
my-brainy-app
|
||||
--memory="2g" \
|
||||
--memory-reservation="1.5g" \
|
||||
--cpus="2" \
|
||||
my-brainy-app
|
||||
```
|
||||
|
||||
**Expected allocation:**
|
||||
```
|
||||
Container Limit: 2048 MB
|
||||
Available: 1638 MB (80% usable)
|
||||
Model Memory: -150 MB
|
||||
Available for Cache: 1488 MB
|
||||
Container Limit: 2048 MB
|
||||
Available: 1638 MB (80% usable)
|
||||
Model Memory: -150 MB
|
||||
Available for Cache: 1488 MB
|
||||
Container Ratio (40%): 595 MB UnifiedCache
|
||||
```
|
||||
|
||||
**Medium Container (8GB)**
|
||||
```bash
|
||||
docker run \
|
||||
--memory="8g" \
|
||||
--memory-reservation="6g" \
|
||||
--cpus="4" \
|
||||
-e NODE_OPTIONS="--max-old-space-size=6144" \
|
||||
my-brainy-app
|
||||
--memory="8g" \
|
||||
--memory-reservation="6g" \
|
||||
--cpus="4" \
|
||||
-e NODE_OPTIONS="--max-old-space-size=6144" \
|
||||
my-brainy-app
|
||||
```
|
||||
|
||||
**Expected allocation:**
|
||||
```
|
||||
Container Limit: 8192 MB
|
||||
Available: 6554 MB
|
||||
Model Memory: -150 MB
|
||||
Available for Cache: 6404 MB
|
||||
Container Limit: 8192 MB
|
||||
Available: 6554 MB
|
||||
Model Memory: -150 MB
|
||||
Available for Cache: 6404 MB
|
||||
Container Ratio (40%): 2562 MB UnifiedCache
|
||||
```
|
||||
|
||||
|
|
@ -335,24 +335,24 @@ Container Ratio (40%): 2562 MB UnifiedCache
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: brainy-api
|
||||
name: brainy-api
|
||||
spec:
|
||||
replicas: 3
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: brainy
|
||||
image: my-brainy-app:latest
|
||||
resources:
|
||||
requests:
|
||||
memory: "1.5Gi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "1000m"
|
||||
env:
|
||||
- name: NODE_OPTIONS
|
||||
value: "--max-old-space-size=1536"
|
||||
replicas: 3
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: brainy
|
||||
image: my-brainy-app:latest
|
||||
resources:
|
||||
requests:
|
||||
memory: "1.5Gi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "1000m"
|
||||
env:
|
||||
- name: NODE_OPTIONS
|
||||
value: "--max-old-space-size=1536"
|
||||
```
|
||||
|
||||
**Medium Pod (8GB)**
|
||||
|
|
@ -360,24 +360,24 @@ spec:
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: brainy-api
|
||||
name: brainy-api
|
||||
spec:
|
||||
replicas: 2
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: brainy
|
||||
image: my-brainy-app:latest
|
||||
resources:
|
||||
requests:
|
||||
memory: "6Gi"
|
||||
cpu: "2000m"
|
||||
limits:
|
||||
memory: "8Gi"
|
||||
cpu: "4000m"
|
||||
env:
|
||||
- name: NODE_OPTIONS
|
||||
value: "--max-old-space-size=6144"
|
||||
replicas: 2
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: brainy
|
||||
image: my-brainy-app:latest
|
||||
resources:
|
||||
requests:
|
||||
memory: "6Gi"
|
||||
cpu: "2000m"
|
||||
limits:
|
||||
memory: "8Gi"
|
||||
cpu: "4000m"
|
||||
env:
|
||||
- name: NODE_OPTIONS
|
||||
value: "--max-old-space-size=6144"
|
||||
```
|
||||
|
||||
**Best Practices:**
|
||||
|
|
@ -399,15 +399,15 @@ The system automatically chooses the optimal caching strategy:
|
|||
|
||||
**Auto-detection logic:**
|
||||
```typescript
|
||||
const vectorMemoryNeeded = entityCount × 1536 // bytes
|
||||
const vectorMemoryNeeded = entityCount × 1536 // bytes
|
||||
const hnswCacheAvailable = unifiedCache.maxSize × 0.80
|
||||
|
||||
if (vectorMemoryNeeded < hnswCacheAvailable) {
|
||||
// Preload strategy: all vectors loaded at init
|
||||
console.log('Caching strategy: preloaded (all vectors in memory)')
|
||||
// Preload strategy: all vectors loaded at init
|
||||
console.log('Caching strategy: preloaded (all vectors in memory)')
|
||||
} else {
|
||||
// On-demand strategy: vectors loaded adaptively
|
||||
console.log('Caching strategy: on-demand (adaptive loading via UnifiedCache)')
|
||||
// On-demand strategy: vectors loaded adaptively
|
||||
console.log('Caching strategy: on-demand (adaptive loading via UnifiedCache)')
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -422,12 +422,12 @@ Consider increasing RAM when:
|
|||
**Decision tree:**
|
||||
```
|
||||
If cache hit rate < 70%:
|
||||
└─> Is working set < 50% of total entities?
|
||||
├─> YES: Increase cache size (add RAM)
|
||||
└─> NO: Working set too large, consider:
|
||||
├─> Application-level caching
|
||||
├─> Query optimization
|
||||
└─> Sharding dataset
|
||||
└─> Is working set < 50% of total entities?
|
||||
├─> YES: Increase cache size (add RAM)
|
||||
└─> NO: Working set too large, consider:
|
||||
├─> Application-level caching
|
||||
├─> Query optimization
|
||||
└─> Sharding dataset
|
||||
```
|
||||
|
||||
### When to Shard/Distribute
|
||||
|
|
@ -442,17 +442,17 @@ Consider sharding when:
|
|||
```typescript
|
||||
// Example: Geographic sharding
|
||||
const usEastBrain = new Brainy({
|
||||
storage: { type: 's3', s3Storage: { bucket: 'us-east-data' } }
|
||||
storage: { type: 's3', s3Storage: { bucket: 'us-east-data' } }
|
||||
})
|
||||
|
||||
const euWestBrain = new Brainy({
|
||||
storage: { type: 's3', s3Storage: { bucket: 'eu-west-data' } }
|
||||
storage: { type: 's3', s3Storage: { bucket: 'eu-west-data' } }
|
||||
})
|
||||
|
||||
// Route queries based on user location
|
||||
async function search(query, userRegion) {
|
||||
const brain = userRegion === 'US' ? usEastBrain : euWestBrain
|
||||
return await brain.search(query)
|
||||
const brain = userRegion === 'US' ? usEastBrain : euWestBrain
|
||||
return await brain.search(query)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -482,7 +482,7 @@ console.log(`Pressure: ${memoryInfo.currentPressure.pressure}`)
|
|||
// Values: 'low', 'moderate', 'high', 'critical'
|
||||
|
||||
if (memoryInfo.currentPressure.warnings.length > 0) {
|
||||
console.warn('Memory warnings:', memoryInfo.currentPressure.warnings)
|
||||
console.warn('Memory warnings:', memoryInfo.currentPressure.warnings)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -491,9 +491,9 @@ if (memoryInfo.currentPressure.warnings.length > 0) {
|
|||
const stats = brain.hnsw.getCacheStats()
|
||||
|
||||
if (stats.fairness.fairnessViolation) {
|
||||
console.warn('Cache fairness violation detected')
|
||||
console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access`)
|
||||
console.warn(`HNSW: ${stats.hnswCache.sizePercent}% of cache`)
|
||||
console.warn('Cache fairness violation detected')
|
||||
console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access`)
|
||||
console.warn(`HNSW: ${stats.hnswCache.sizePercent}% of cache`)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -502,7 +502,7 @@ if (stats.fairness.fairnessViolation) {
|
|||
// Track search latency
|
||||
console.time('search')
|
||||
const results = await brain.search('query')
|
||||
console.timeEnd('search') // Target: <10ms for hot queries
|
||||
console.timeEnd('search') // Target: <10ms for hot queries
|
||||
```
|
||||
|
||||
### Alerting Thresholds
|
||||
|
|
@ -516,26 +516,26 @@ Set up alerts for:
|
|||
**Example monitoring script:**
|
||||
```typescript
|
||||
async function monitorHealth() {
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
|
||||
// Alert on low cache hit rate
|
||||
if (stats.unifiedCache.hitRatePercent < 70) {
|
||||
await sendAlert({
|
||||
severity: 'warning',
|
||||
message: `Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`,
|
||||
recommendations: stats.recommendations
|
||||
})
|
||||
}
|
||||
// Alert on low cache hit rate
|
||||
if (stats.unifiedCache.hitRatePercent < 70) {
|
||||
await sendAlert({
|
||||
severity: 'warning',
|
||||
message: `Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`,
|
||||
recommendations: stats.recommendations
|
||||
})
|
||||
}
|
||||
|
||||
// Alert on memory pressure
|
||||
const memoryInfo = brain.hnsw.unifiedCache.getMemoryInfo()
|
||||
if (memoryInfo.currentPressure.pressure === 'high') {
|
||||
await sendAlert({
|
||||
severity: 'critical',
|
||||
message: 'High memory pressure detected',
|
||||
warnings: memoryInfo.currentPressure.warnings
|
||||
})
|
||||
}
|
||||
// Alert on memory pressure
|
||||
const memoryInfo = brain.hnsw.unifiedCache.getMemoryInfo()
|
||||
if (memoryInfo.currentPressure.pressure === 'high') {
|
||||
await sendAlert({
|
||||
severity: 'critical',
|
||||
message: 'High memory pressure detected',
|
||||
warnings: memoryInfo.currentPressure.warnings
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Run every 60 seconds
|
||||
|
|
@ -552,9 +552,9 @@ setInterval(monitorHealth, 60000)
|
|||
|
||||
**Sizing:**
|
||||
```
|
||||
Products: 500,000
|
||||
Vector memory needed: 500K × 1536 bytes = 768 MB
|
||||
HNSW cache available: (16GB × 0.8 - 150MB) × 0.5 × 0.3 = 1,915 MB
|
||||
Products: 500,000
|
||||
Vector memory needed: 500K × 1536 bytes = 768 MB
|
||||
HNSW cache available: (16GB × 0.8 - 150MB) × 0.5 × 0.3 = 1,915 MB
|
||||
|
||||
Result: Standard mode (all vectors fit in HNSW cache)
|
||||
```
|
||||
|
|
@ -562,16 +562,16 @@ Result: Standard mode (all vectors fit in HNSW cache)
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||
model: { precision: 'q8' }
|
||||
storage: { type: 'filesystem', path: '/var/lib/brainy' },
|
||||
model: { precision: 'q8' }
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
// Verify preloaded strategy (all vectors in memory)
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'preloaded'
|
||||
console.log(`Search latency: ${stats.performance.avgSearchMs}ms`) // ~3ms
|
||||
console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'preloaded'
|
||||
console.log(`Search latency: ${stats.performance.avgSearchMs}ms`) // ~3ms
|
||||
```
|
||||
|
||||
### Example 2: Document Search (5M documents)
|
||||
|
|
@ -580,9 +580,9 @@ console.log(`Search latency: ${stats.performance.avgSearchMs}ms`) // ~3ms
|
|||
|
||||
**Sizing:**
|
||||
```
|
||||
Documents: 5,000,000
|
||||
Vector memory needed: 5M × 1536 bytes = 7,680 MB
|
||||
HNSW cache available: (32GB × 0.8 - 150MB) × 0.5 × 0.3 = 3,910 MB
|
||||
Documents: 5,000,000
|
||||
Vector memory needed: 5M × 1536 bytes = 7,680 MB
|
||||
HNSW cache available: (32GB × 0.8 - 150MB) × 0.5 × 0.3 = 3,910 MB
|
||||
|
||||
Result: On-demand caching (vectors loaded adaptively)
|
||||
```
|
||||
|
|
@ -590,20 +590,20 @@ Result: On-demand caching (vectors loaded adaptively)
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'gcs-native',
|
||||
gcsNativeStorage: { bucketName: 'docs-production' }
|
||||
},
|
||||
model: { precision: 'q8' }
|
||||
storage: {
|
||||
type: 'gcs-native',
|
||||
gcsNativeStorage: { bucketName: 'docs-production' }
|
||||
},
|
||||
model: { precision: 'q8' }
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
// Monitor cache performance
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand'
|
||||
console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >80%
|
||||
console.log(`Cold search latency: ${stats.performance.avgSearchMs}ms`) // ~12ms
|
||||
console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand'
|
||||
console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >80%
|
||||
console.log(`Cold search latency: ${stats.performance.avgSearchMs}ms`) // ~12ms
|
||||
|
||||
// Recommendations
|
||||
console.log('Recommendations:', stats.recommendations)
|
||||
|
|
@ -616,9 +616,9 @@ console.log('Recommendations:', stats.recommendations)
|
|||
|
||||
**Sizing:**
|
||||
```
|
||||
Entities: 20,000,000
|
||||
Vector memory needed: 20M × 1536 bytes = 30,720 MB
|
||||
HNSW cache available: ~15,691 MB (after logarithmic scaling)
|
||||
Entities: 20,000,000
|
||||
Vector memory needed: 20M × 1536 bytes = 30,720 MB
|
||||
HNSW cache available: ~15,691 MB (after logarithmic scaling)
|
||||
|
||||
Result: On-demand caching with high-performance adaptive loading
|
||||
```
|
||||
|
|
@ -626,14 +626,14 @@ Result: On-demand caching with high-performance adaptive loading
|
|||
**Configuration:**
|
||||
```typescript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'knowledge-graph-prod',
|
||||
region: 'us-east-1'
|
||||
}
|
||||
},
|
||||
model: { precision: 'fp32' } // Maximum accuracy
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'knowledge-graph-prod',
|
||||
region: 'us-east-1'
|
||||
}
|
||||
},
|
||||
model: { precision: 'fp32' } // Maximum accuracy
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
|
@ -641,13 +641,13 @@ await brain.init()
|
|||
// Enterprise monitoring
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
console.log(`Entities: ${stats.autoDetection.entityCount.toLocaleString()}`)
|
||||
console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand'
|
||||
console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >85%
|
||||
console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand'
|
||||
console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >85%
|
||||
console.log(`HNSW cache: ${stats.hnswCache.estimatedMemoryMB}MB`)
|
||||
|
||||
// Fairness check
|
||||
if (stats.fairness.fairnessViolation) {
|
||||
console.warn('HNSW dominating cache - consider tuning eviction policies')
|
||||
console.warn('HNSW dominating cache - consider tuning eviction policies')
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -690,8 +690,8 @@ console.log(`Warnings:`, memoryInfo.currentPressure.warnings)
|
|||
```typescript
|
||||
const stats = brain.hnsw.getCacheStats()
|
||||
if (stats.fairness.fairnessViolation) {
|
||||
console.log(`HNSW access: ${stats.fairness.hnswAccessPercent}%`)
|
||||
console.log(`HNSW cache: ${stats.hnswCache.sizePercent}%`)
|
||||
console.log(`HNSW access: ${stats.fairness.hnswAccessPercent}%`)
|
||||
console.log(`HNSW cache: ${stats.hnswCache.sizePercent}%`)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -704,8 +704,7 @@ if (stats.fairness.fairnessViolation) {
|
|||
|
||||
## 📚 Additional Resources
|
||||
|
||||
- **[Migration Guide](../guides/migration-3.36.0.md)** - Upgrading to v3.36.0
|
||||
- **[Architecture Overview](../architecture/data-storage-architecture.md)** - Deep dive into storage and caching
|
||||
- **[Migration Guide](../guides/migration-3.36.0.md)** - Upgrading to **[Architecture Overview](../architecture/data-storage-architecture.md)** - Deep dive into storage and caching
|
||||
- **[GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)** - Report problems or ask questions
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
# AWS S3 Cost Optimization Guide for Brainy v4.0.0
|
||||
|
||||
# AWS S3 Cost Optimization Guide for Brainy
|
||||
> **Cost Impact**: Reduce storage costs from $138k/year to $5.9k/year at 500TB scale (**96% savings**)
|
||||
|
||||
## Overview
|
||||
|
||||
Brainy v4.0.0 provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance.
|
||||
Brainy provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance.
|
||||
|
||||
## Cost Breakdown (Before Optimization)
|
||||
|
||||
|
|
@ -39,10 +38,10 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|||
|
||||
// Initialize Brainy with S3 storage
|
||||
const storage = new S3CompatibleStorage({
|
||||
bucket: 'my-brainy-data',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
bucket: 'my-brainy-data',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
|
|
@ -50,24 +49,24 @@ await brain.init()
|
|||
|
||||
// Set lifecycle policy for automatic archival
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
id: 'optimize-vectors',
|
||||
prefix: 'entities/nouns/vectors/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' }, // Infrequent Access after 30 days
|
||||
{ days: 90, storageClass: 'GLACIER' }, // Glacier after 90 days
|
||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 1 year
|
||||
]
|
||||
}, {
|
||||
id: 'optimize-metadata',
|
||||
prefix: 'entities/nouns/metadata/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' },
|
||||
{ days: 180, storageClass: 'GLACIER' }
|
||||
]
|
||||
}]
|
||||
rules: [{
|
||||
id: 'optimize-vectors',
|
||||
prefix: 'entities/nouns/vectors/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' }, // Infrequent Access after 30 days
|
||||
{ days: 90, storageClass: 'GLACIER' }, // Glacier after 90 days
|
||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 1 year
|
||||
]
|
||||
}, {
|
||||
id: 'optimize-metadata',
|
||||
prefix: 'entities/nouns/metadata/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' },
|
||||
{ days: 180, storageClass: 'GLACIER' }
|
||||
]
|
||||
}]
|
||||
})
|
||||
|
||||
// Verify lifecycle policy
|
||||
|
|
@ -84,10 +83,10 @@ console.log('Lifecycle policy active:', policy.rules.length, 'rules')
|
|||
- 10% of data 365+ days old (Deep Archive)
|
||||
|
||||
```
|
||||
Standard (200TB): 200TB × $0.023/GB × 12 = $55,200/year
|
||||
Standard-IA (150TB): 150TB × $0.0125/GB × 12 = $22,500/year
|
||||
Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
||||
Deep Archive (50TB): 50TB × $0.00099/GB × 12 = $594/year
|
||||
Standard (200TB): 200TB × $0.023/GB × 12 = $55,200/year
|
||||
Standard-IA (150TB): 150TB × $0.0125/GB × 12 = $22,500/year
|
||||
Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
||||
Deep Archive (50TB): 50TB × $0.00099/GB × 12 = $594/year
|
||||
|
||||
Total Storage Cost: $83,094/year (instead of $138,000)
|
||||
Total with Operations: ~$88,000/year
|
||||
|
|
@ -133,11 +132,11 @@ Intelligent-Tiering automatically moves objects between:
|
|||
- 30% Deep Archive Access
|
||||
|
||||
```
|
||||
Frequent (75TB): 75TB × $0.023/GB × 12 = $20,700/year
|
||||
Infrequent (100TB): 100TB × $0.0125/GB × 12 = $15,000/year
|
||||
Archive (175TB): 175TB × $0.004/GB × 12 = $8,400/year
|
||||
Deep Archive (150TB): 150TB × $0.00099/GB × 12 = $1,782/year
|
||||
Monitoring: ~$300/year (minimal)
|
||||
Frequent (75TB): 75TB × $0.023/GB × 12 = $20,700/year
|
||||
Infrequent (100TB): 100TB × $0.0125/GB × 12 = $15,000/year
|
||||
Archive (175TB): 175TB × $0.004/GB × 12 = $8,400/year
|
||||
Deep Archive (150TB): 150TB × $0.00099/GB × 12 = $1,782/year
|
||||
Monitoring: ~$300/year (minimal)
|
||||
|
||||
Total Storage Cost: $46,182/year
|
||||
Total with Operations: ~$51,000/year
|
||||
|
|
@ -157,21 +156,21 @@ await storage.enableIntelligentTiering('entities/verbs/vectors/', 'verbs-auto')
|
|||
|
||||
// Set lifecycle policy for metadata (less frequently accessed)
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
id: 'archive-old-metadata',
|
||||
prefix: 'entities/nouns/metadata/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' },
|
||||
{ days: 60, storageClass: 'GLACIER' },
|
||||
{ days: 180, storageClass: 'DEEP_ARCHIVE' }
|
||||
]
|
||||
}, {
|
||||
id: 'cleanup-old-system-data',
|
||||
prefix: '_system/',
|
||||
status: 'Enabled',
|
||||
expiration: { days: 365 } // Delete old statistics
|
||||
}]
|
||||
rules: [{
|
||||
id: 'archive-old-metadata',
|
||||
prefix: 'entities/nouns/metadata/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' },
|
||||
{ days: 60, storageClass: 'GLACIER' },
|
||||
{ days: 180, storageClass: 'DEEP_ARCHIVE' }
|
||||
]
|
||||
}, {
|
||||
id: 'cleanup-old-system-data',
|
||||
prefix: '_system/',
|
||||
status: 'Enabled',
|
||||
expiration: { days: 365 } // Delete old statistics
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -179,19 +178,19 @@ await storage.setLifecyclePolicy({
|
|||
|
||||
**Vectors (300TB with Intelligent-Tiering):**
|
||||
```
|
||||
Frequent (45TB): 45TB × $0.023/GB × 12 = $12,420/year
|
||||
Infrequent (60TB): 60TB × $0.0125/GB × 12 = $9,000/year
|
||||
Archive (105TB): 105TB × $0.004/GB × 12 = $5,040/year
|
||||
Deep Archive (90TB): 90TB × $0.00099/GB × 12 = $1,069/year
|
||||
Frequent (45TB): 45TB × $0.023/GB × 12 = $12,420/year
|
||||
Infrequent (60TB): 60TB × $0.0125/GB × 12 = $9,000/year
|
||||
Archive (105TB): 105TB × $0.004/GB × 12 = $5,040/year
|
||||
Deep Archive (90TB): 90TB × $0.00099/GB × 12 = $1,069/year
|
||||
Subtotal: $27,529/year
|
||||
```
|
||||
|
||||
**Metadata (200TB with Lifecycle Policy):**
|
||||
```
|
||||
Standard (60TB): 60TB × $0.023/GB × 12 = $16,560/year
|
||||
Standard-IA (40TB): 40TB × $0.0125/GB × 12 = $6,000/year
|
||||
Glacier (60TB): 60TB × $0.004/GB × 12 = $2,880/year
|
||||
Deep Archive (40TB): 40TB × $0.00099/GB × 12 = $475/year
|
||||
Standard (60TB): 60TB × $0.023/GB × 12 = $16,560/year
|
||||
Standard-IA (40TB): 40TB × $0.0125/GB × 12 = $6,000/year
|
||||
Glacier (60TB): 60TB × $0.004/GB × 12 = $2,880/year
|
||||
Deep Archive (40TB): 40TB × $0.00099/GB × 12 = $475/year
|
||||
Subtotal: $25,915/year
|
||||
```
|
||||
|
||||
|
|
@ -206,16 +205,16 @@ Subtotal: $25,915/year
|
|||
|
||||
```typescript
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
id: 'aggressive-archival',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 14, storageClass: 'STANDARD_IA' }, // IA after 2 weeks
|
||||
{ days: 30, storageClass: 'GLACIER' }, // Glacier after 1 month
|
||||
{ days: 90, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 3 months
|
||||
]
|
||||
}]
|
||||
rules: [{
|
||||
id: 'aggressive-archival',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 14, storageClass: 'STANDARD_IA' }, // IA after 2 weeks
|
||||
{ days: 30, storageClass: 'GLACIER' }, // Glacier after 1 month
|
||||
{ days: 90, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 3 months
|
||||
]
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -223,10 +222,10 @@ await storage.setLifecyclePolicy({
|
|||
|
||||
**After 1 year:**
|
||||
```
|
||||
Standard (50TB): 50TB × $0.023/GB × 12 = $13,800/year
|
||||
Standard-IA (50TB): 50TB × $0.0125/GB × 12 = $7,500/year
|
||||
Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
||||
Deep Archive (300TB): 300TB × $0.00099/GB × 12 = $3,564/year
|
||||
Standard (50TB): 50TB × $0.023/GB × 12 = $13,800/year
|
||||
Standard-IA (50TB): 50TB × $0.0125/GB × 12 = $7,500/year
|
||||
Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
||||
Deep Archive (300TB): 300TB × $0.00099/GB × 12 = $3,564/year
|
||||
|
||||
Total Storage Cost: $29,664/year
|
||||
Total with Operations: ~$34,000/year
|
||||
|
|
@ -250,16 +249,16 @@ Note: Retrieval costs may be significant if archived data is accessed frequently
|
|||
### Efficient Cleanup
|
||||
|
||||
```typescript
|
||||
// v4.0.0: Batch delete (1000 objects per request)
|
||||
// Batch delete (1000 objects per request)
|
||||
const idsToDelete = [/* array of entity IDs */]
|
||||
|
||||
// Generate paths for both vector and metadata files
|
||||
const paths = idsToDelete.flatMap(id => {
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
})
|
||||
|
||||
// Batch delete (much faster and cheaper than individual deletes)
|
||||
|
|
@ -280,14 +279,14 @@ console.log('Active rules:', policy.rules)
|
|||
|
||||
// Example output:
|
||||
// {
|
||||
// rules: [
|
||||
// {
|
||||
// id: 'optimize-vectors',
|
||||
// prefix: 'entities/nouns/vectors/',
|
||||
// status: 'Enabled',
|
||||
// transitions: [...]
|
||||
// }
|
||||
// ]
|
||||
// rules: [
|
||||
// {
|
||||
// id: 'optimize-vectors',
|
||||
// prefix: 'entities/nouns/vectors/',
|
||||
// status: 'Enabled',
|
||||
// transitions: [...]
|
||||
// }
|
||||
// ]
|
||||
// }
|
||||
```
|
||||
|
||||
|
|
@ -396,6 +395,5 @@ await storage.enableIntelligentTiering('entities/', 'new-config')
|
|||
|
||||
---
|
||||
|
||||
**Version**: v4.0.0
|
||||
**Last Updated**: 2025-10-17
|
||||
**Cloud Provider**: AWS S3
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
# Azure Blob Storage Cost Optimization Guide for Brainy v4.0.0
|
||||
|
||||
# Azure Blob Storage Cost Optimization Guide for Brainy
|
||||
> **Cost Impact**: Reduce storage costs from $107k/year to $5k/year at 500TB scale (**95% savings**)
|
||||
|
||||
## Overview
|
||||
|
||||
Brainy v4.0.0 provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations.
|
||||
Brainy provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations.
|
||||
|
||||
## Cost Breakdown (Before Optimization)
|
||||
|
||||
|
|
@ -41,8 +40,8 @@ import { AzureBlobStorage } from '@soulcraft/brainy/storage'
|
|||
|
||||
// Initialize Brainy with Azure storage
|
||||
const storage = new AzureBlobStorage({
|
||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
||||
containerName: 'brainy-data'
|
||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
||||
containerName: 'brainy-data'
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
|
|
@ -50,19 +49,19 @@ await brain.init()
|
|||
|
||||
// Change tier for a single blob
|
||||
await storage.changeBlobTier(
|
||||
'entities/nouns/vectors/00/00123456-uuid.json',
|
||||
'Cool'
|
||||
'entities/nouns/vectors/00/00123456-uuid.json',
|
||||
'Cool'
|
||||
)
|
||||
|
||||
// Batch tier changes (efficient for thousands of blobs)
|
||||
const blobsToMove = [
|
||||
'entities/nouns/vectors/01/...',
|
||||
'entities/nouns/vectors/02/...',
|
||||
// ... up to thousands of blobs
|
||||
'entities/nouns/vectors/01/...',
|
||||
'entities/nouns/vectors/02/...',
|
||||
// ... up to thousands of blobs
|
||||
]
|
||||
|
||||
await storage.batchChangeTier(blobsToMove, 'Cool') // Hot → Cool
|
||||
await storage.batchChangeTier(oldBlobs, 'Archive') // Cool → Archive
|
||||
await storage.batchChangeTier(blobsToMove, 'Cool') // Hot → Cool
|
||||
await storage.batchChangeTier(oldBlobs, 'Archive') // Cool → Archive
|
||||
```
|
||||
|
||||
### Immediate Cost Impact
|
||||
|
|
@ -90,54 +89,54 @@ Savings: $20,346/year (95% savings on moved data)
|
|||
```typescript
|
||||
// Set lifecycle policy for automatic tier management
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
name: 'optimizeVectors',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/vectors/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
name: 'optimizeMetadata',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/metadata/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 180 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
name: 'cleanupOldSystemFiles',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['_system/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
delete: { daysAfterModificationGreaterThan: 365 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
rules: [{
|
||||
name: 'optimizeVectors',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/vectors/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
name: 'optimizeMetadata',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/metadata/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 180 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
name: 'cleanupOldSystemFiles',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['_system/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
delete: { daysAfterModificationGreaterThan: 365 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
// Verify lifecycle policy
|
||||
|
|
@ -153,9 +152,9 @@ console.log('Lifecycle policy active:', policy.rules.length, 'rules')
|
|||
- 30% of data in Archive tier (90+ days old)
|
||||
|
||||
```
|
||||
Hot (150TB): 150TB × $0.0184/GB × 12 = $32,256/year
|
||||
Cool (200TB): 200TB × $0.0115/GB × 12 = $26,880/year
|
||||
Archive (150TB): 150TB × $0.00099/GB × 12 = $1,732/year
|
||||
Hot (150TB): 150TB × $0.0184/GB × 12 = $32,256/year
|
||||
Cool (200TB): 200TB × $0.0115/GB × 12 = $26,880/year
|
||||
Archive (150TB): 150TB × $0.00099/GB × 12 = $1,732/year
|
||||
|
||||
Total Storage Cost: $60,868/year
|
||||
Total with Operations: ~$65,500/year
|
||||
|
|
@ -170,23 +169,23 @@ Savings: $47,000/year (42%)
|
|||
|
||||
```typescript
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
name: 'aggressiveArchival',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 14 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 30 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
rules: [{
|
||||
name: 'aggressiveArchival',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 14 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 30 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -194,9 +193,9 @@ await storage.setLifecyclePolicy({
|
|||
|
||||
**After 6 months:**
|
||||
```
|
||||
Hot (50TB): 50TB × $0.0184/GB × 12 = $10,752/year
|
||||
Cool (100TB): 100TB × $0.0115/GB × 12 = $13,440/year
|
||||
Archive (350TB): 350TB × $0.00099/GB × 12 = $4,039/year
|
||||
Hot (50TB): 50TB × $0.0184/GB × 12 = $10,752/year
|
||||
Cool (100TB): 100TB × $0.0115/GB × 12 = $13,440/year
|
||||
Archive (350TB): 350TB × $0.00099/GB × 12 = $4,039/year
|
||||
|
||||
Total Storage Cost: $28,231/year
|
||||
Total with Operations: ~$33,000/year
|
||||
|
|
@ -211,41 +210,41 @@ Warning: Archive rehydration takes 1-15 hours
|
|||
|
||||
```typescript
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
// Vectors: Keep in Hot/Cool for search performance
|
||||
name: 'vectors-moderate',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/vectors/', 'entities/verbs/vectors/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 60 }
|
||||
// Don't archive vectors - keep searchable
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
// Metadata: Aggressive archival
|
||||
name: 'metadata-aggressive',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/metadata/', 'entities/verbs/metadata/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
rules: [{
|
||||
// Vectors: Keep in Hot/Cool for search performance
|
||||
name: 'vectors-moderate',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/vectors/', 'entities/verbs/vectors/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 60 }
|
||||
// Don't archive vectors - keep searchable
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
// Metadata: Aggressive archival
|
||||
name: 'metadata-aggressive',
|
||||
enabled: true,
|
||||
type: 'Lifecycle',
|
||||
definition: {
|
||||
filters: {
|
||||
blobTypes: ['blockBlob'],
|
||||
prefixMatch: ['entities/nouns/metadata/', 'entities/verbs/metadata/']
|
||||
},
|
||||
actions: {
|
||||
baseBlob: {
|
||||
tierToCool: { daysAfterModificationGreaterThan: 30 },
|
||||
tierToArchive: { daysAfterModificationGreaterThan: 90 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -253,16 +252,16 @@ await storage.setLifecyclePolicy({
|
|||
|
||||
**Vectors (300TB):**
|
||||
```
|
||||
Hot (90TB): 90TB × $0.0184/GB × 12 = $19,354/year
|
||||
Cool (210TB): 210TB × $0.0115/GB × 12 = $28,980/year
|
||||
Hot (90TB): 90TB × $0.0184/GB × 12 = $19,354/year
|
||||
Cool (210TB): 210TB × $0.0115/GB × 12 = $28,980/year
|
||||
Subtotal: $48,334/year
|
||||
```
|
||||
|
||||
**Metadata (200TB):**
|
||||
```
|
||||
Hot (30TB): 30TB × $0.0184/GB × 12 = $6,451/year
|
||||
Cool (70TB): 70TB × $0.0115/GB × 12 = $9,660/year
|
||||
Archive (100TB): 100TB × $0.00099/GB × 12 = $1,158/year
|
||||
Hot (30TB): 30TB × $0.0184/GB × 12 = $6,451/year
|
||||
Cool (70TB): 70TB × $0.0115/GB × 12 = $9,660/year
|
||||
Archive (100TB): 100TB × $0.00099/GB × 12 = $1,158/year
|
||||
Subtotal: $17,269/year
|
||||
```
|
||||
|
||||
|
|
@ -288,8 +287,8 @@ Subtotal: $17,269/year
|
|||
```typescript
|
||||
// Rehydrate blob from Archive to Hot (high priority)
|
||||
await storage.rehydrateBlob(
|
||||
'entities/nouns/vectors/00/00123456-uuid.json',
|
||||
'High' // 'Standard' or 'High' priority
|
||||
'entities/nouns/vectors/00/00123456-uuid.json',
|
||||
'High' // 'Standard' or 'High' priority
|
||||
)
|
||||
|
||||
// Rehydration time:
|
||||
|
|
@ -309,7 +308,7 @@ console.log('Archive status:', metadata.archiveStatus)
|
|||
const blobsToRehydrate = [/* array of blob paths */]
|
||||
|
||||
for (const blobPath of blobsToRehydrate) {
|
||||
await storage.rehydrateBlob(blobPath, 'High')
|
||||
await storage.rehydrateBlob(blobPath, 'High')
|
||||
}
|
||||
|
||||
// Wait for rehydration to complete (1-15 hours)
|
||||
|
|
@ -333,15 +332,15 @@ Examples:
|
|||
### Efficient Bulk Deletions
|
||||
|
||||
```typescript
|
||||
// v4.0.0: Batch delete (256 blobs per request)
|
||||
// Batch delete (256 blobs per request)
|
||||
const idsToDelete = [/* array of entity IDs */]
|
||||
|
||||
const paths = idsToDelete.flatMap(id => {
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
})
|
||||
|
||||
// Batch delete via BlobBatchClient
|
||||
|
|
@ -375,14 +374,14 @@ console.log('Active rules:', policy.rules)
|
|||
|
||||
// Example output:
|
||||
// {
|
||||
// rules: [
|
||||
// {
|
||||
// name: 'optimizeVectors',
|
||||
// enabled: true,
|
||||
// type: 'Lifecycle',
|
||||
// definition: {...}
|
||||
// }
|
||||
// ]
|
||||
// rules: [
|
||||
// {
|
||||
// name: 'optimizeVectors',
|
||||
// enabled: true,
|
||||
// type: 'Lifecycle',
|
||||
// definition: {...}
|
||||
// }
|
||||
// ]
|
||||
// }
|
||||
```
|
||||
|
||||
|
|
@ -452,8 +451,8 @@ Monthly cost trend: Decreasing 5-8% per month as data transitions
|
|||
// Check lifecycle policy status
|
||||
const policy = await storage.getLifecyclePolicy()
|
||||
console.log('Policy rules:', policy.rules.map(r => ({
|
||||
name: r.name,
|
||||
enabled: r.enabled
|
||||
name: r.name,
|
||||
enabled: r.enabled
|
||||
})))
|
||||
|
||||
// Azure lifecycle policies run once per day
|
||||
|
|
@ -471,9 +470,9 @@ await storage.rehydrateBlob(blobPath, 'High')
|
|||
// Wait for rehydration (check status)
|
||||
let status
|
||||
do {
|
||||
await new Promise(resolve => setTimeout(resolve, 60000)) // Wait 1 minute
|
||||
const metadata = await storage.getBlobMetadata(blobPath)
|
||||
status = metadata.archiveStatus
|
||||
await new Promise(resolve => setTimeout(resolve, 60000)) // Wait 1 minute
|
||||
const metadata = await storage.getBlobMetadata(blobPath)
|
||||
status = metadata.archiveStatus
|
||||
} while (status && status.includes('pending'))
|
||||
|
||||
// Now access the blob
|
||||
|
|
@ -494,8 +493,8 @@ const data = await storage.get(blobPath)
|
|||
|
||||
```typescript
|
||||
const storage = new AzureBlobStorage({
|
||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
||||
containerName: 'brainy-data'
|
||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING,
|
||||
containerName: 'brainy-data'
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -503,9 +502,9 @@ const storage = new AzureBlobStorage({
|
|||
|
||||
```typescript
|
||||
const storage = new AzureBlobStorage({
|
||||
accountName: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
accountKey: process.env.AZURE_STORAGE_KEY,
|
||||
containerName: 'brainy-data'
|
||||
accountName: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
accountKey: process.env.AZURE_STORAGE_KEY,
|
||||
containerName: 'brainy-data'
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -513,9 +512,9 @@ const storage = new AzureBlobStorage({
|
|||
|
||||
```typescript
|
||||
const storage = new AzureBlobStorage({
|
||||
sasToken: process.env.AZURE_STORAGE_SAS_TOKEN,
|
||||
accountName: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
containerName: 'brainy-data'
|
||||
sasToken: process.env.AZURE_STORAGE_SAS_TOKEN,
|
||||
accountName: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
containerName: 'brainy-data'
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -549,6 +548,5 @@ const storage = new AzureBlobStorage({
|
|||
|
||||
---
|
||||
|
||||
**Version**: v4.0.0
|
||||
**Last Updated**: 2025-10-17
|
||||
**Cloud Provider**: Azure Blob Storage
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
# Cloudflare R2 Cost Optimization Guide for Brainy v4.0.0
|
||||
|
||||
# Cloudflare R2 Cost Optimization Guide for Brainy
|
||||
> **Cost Impact**: $0 egress fees + 96% storage savings = Lowest cloud storage costs
|
||||
|
||||
## Overview
|
||||
|
||||
Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy v4.0.0 fully supports R2 with lifecycle policies and batch operations.
|
||||
Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy fully supports R2 with lifecycle policies and batch operations.
|
||||
|
||||
## Cost Breakdown
|
||||
|
||||
|
|
@ -49,7 +48,7 @@ Savings vs AWS: $156,000/year (62%)
|
|||
- Cloudflare plans to add infrequent access tiers
|
||||
|
||||
**When lifecycle features arrive:**
|
||||
- Brainy v4.0.0 is already prepared with `setLifecyclePolicy()` support
|
||||
- Brainy is already prepared with `setLifecyclePolicy()` support
|
||||
- Will work seamlessly once Cloudflare enables lifecycle management
|
||||
|
||||
## Strategy 1: Use R2 Standard (Current Best Practice)
|
||||
|
|
@ -62,11 +61,11 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|||
|
||||
// R2 uses S3-compatible API
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'my-brainy-data',
|
||||
region: 'auto', // R2 uses 'auto' region
|
||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
||||
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'my-brainy-data',
|
||||
region: 'auto', // R2 uses 'auto' region
|
||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
|
|
@ -96,23 +95,23 @@ Total: $90,081/year
|
|||
```typescript
|
||||
// Cloudflare Worker (runs at edge)
|
||||
export default {
|
||||
async fetch(request, env) {
|
||||
// Initialize Brainy with R2
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: env.R2_ENDPOINT,
|
||||
bucket: env.R2_BUCKET,
|
||||
accessKeyId: env.R2_ACCESS_KEY,
|
||||
secretAccessKey: env.R2_SECRET_KEY,
|
||||
region: 'auto'
|
||||
})
|
||||
async fetch(request, env) {
|
||||
// Initialize Brainy with R2
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: env.R2_ENDPOINT,
|
||||
bucket: env.R2_BUCKET,
|
||||
accessKeyId: env.R2_ACCESS_KEY,
|
||||
secretAccessKey: env.R2_SECRET_KEY,
|
||||
region: 'auto'
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
const brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
// Process at edge (no origin server needed)
|
||||
const results = await brain.search(request.query)
|
||||
return new Response(JSON.stringify(results))
|
||||
}
|
||||
// Process at edge (no origin server needed)
|
||||
const results = await brain.search(request.query)
|
||||
return new Response(JSON.stringify(results))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -121,18 +120,18 @@ export default {
|
|||
```
|
||||
R2 Storage (500TB): $90,000/year
|
||||
Workers (10M requests/day):
|
||||
- First 100k requests/day: FREE
|
||||
- Additional 350M requests/month: $1,750/year
|
||||
- CPU time (50ms avg): $5,000/year
|
||||
- First 100k requests/day: FREE
|
||||
- Additional 350M requests/month: $1,750/year
|
||||
- CPU time (50ms avg): $5,000/year
|
||||
|
||||
Total: $96,750/year
|
||||
|
||||
vs Traditional Setup (AWS S3 + EC2 + CloudFront):
|
||||
- S3: $138,000/year
|
||||
- EC2 (t3.xlarge × 4): $24,000/year
|
||||
- CloudFront egress: $50,000/year
|
||||
- Load balancer: $3,000/year
|
||||
Total: $215,000/year
|
||||
- S3: $138,000/year
|
||||
- EC2 (t3.xlarge × 4): $24,000/year
|
||||
- CloudFront egress: $50,000/year
|
||||
- Load balancer: $3,000/year
|
||||
Total: $215,000/year
|
||||
|
||||
Savings: $118,250/year (55%)
|
||||
```
|
||||
|
|
@ -144,20 +143,20 @@ Savings: $118,250/year (55%)
|
|||
```typescript
|
||||
// Use R2 for frequently accessed data (zero egress)
|
||||
const hotStorage = new S3CompatibleStorage({
|
||||
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'brainy-hot',
|
||||
region: 'auto',
|
||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
||||
endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'brainy-hot',
|
||||
region: 'auto',
|
||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
// Use AWS S3 with Intelligent-Tiering for cold data
|
||||
const coldStorage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
bucket: 'brainy-archive',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
endpoint: 's3.amazonaws.com',
|
||||
bucket: 'brainy-archive',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
// Initialize separate Brainy instances or implement tiering logic
|
||||
|
|
@ -167,17 +166,17 @@ const coldStorage = new S3CompatibleStorage({
|
|||
|
||||
```
|
||||
R2 Hot Data (300TB):
|
||||
Storage: 300TB × $0.015/GB × 12 = $54,000/year
|
||||
Egress: $0
|
||||
Storage: 300TB × $0.015/GB × 12 = $54,000/year
|
||||
Egress: $0
|
||||
|
||||
S3 Cold Data (200TB with lifecycle → Deep Archive):
|
||||
Storage: 200TB × $0.00099/GB × 12 = $2,376/year
|
||||
Ops: $1,000/year
|
||||
Storage: 200TB × $0.00099/GB × 12 = $2,376/year
|
||||
Ops: $1,000/year
|
||||
|
||||
Total: $57,376/year
|
||||
|
||||
vs All AWS S3:
|
||||
- S3 storage + egress: $251,000/year
|
||||
- S3 storage + egress: $251,000/year
|
||||
|
||||
Savings: $193,624/year (77%)
|
||||
```
|
||||
|
|
@ -187,15 +186,15 @@ Savings: $193,624/year (77%)
|
|||
### Efficient Bulk Deletions
|
||||
|
||||
```typescript
|
||||
// v4.0.0: R2 supports S3 batch delete API
|
||||
// R2 supports S3 batch delete API
|
||||
const idsToDelete = [/* array of entity IDs */]
|
||||
|
||||
const paths = idsToDelete.flatMap(id => {
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
})
|
||||
|
||||
// Batch delete (1000 objects per request)
|
||||
|
|
@ -231,10 +230,10 @@ https://storage.yourdomain.com/entities/nouns/vectors/...
|
|||
```typescript
|
||||
// Worker triggered on R2 object upload
|
||||
export default {
|
||||
async fetch(request, env) {
|
||||
// Process new objects automatically
|
||||
// E.g., index new entities, generate thumbnails, etc.
|
||||
}
|
||||
async fetch(request, env) {
|
||||
// Process new objects automatically
|
||||
// E.g., index new entities, generate thumbnails, etc.
|
||||
}
|
||||
}
|
||||
|
||||
// Cost: Only pay for Worker execution (no polling needed)
|
||||
|
|
@ -244,7 +243,7 @@ export default {
|
|||
|
||||
```typescript
|
||||
// Generate presigned URL for direct browser uploads
|
||||
const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry
|
||||
const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry
|
||||
|
||||
// Client uploads directly to R2 (no server bandwidth used)
|
||||
```
|
||||
|
|
@ -255,7 +254,7 @@ const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry
|
|||
|
||||
```typescript
|
||||
const status = await storage.getStorageStatus()
|
||||
console.log('Storage type:', status.type) // 's3-compatible'
|
||||
console.log('Storage type:', status.type) // 's3-compatible'
|
||||
console.log('Bucket:', status.details.bucket)
|
||||
console.log('Endpoint:', status.details.endpoint)
|
||||
```
|
||||
|
|
@ -318,20 +317,20 @@ Egress: Unlimited (always free)
|
|||
|
||||
```bash
|
||||
# Install rclone
|
||||
brew install rclone # or apt-get install rclone
|
||||
brew install rclone # or apt-get install rclone
|
||||
|
||||
# Configure S3 source
|
||||
rclone config create s3-source s3 \
|
||||
access_key_id=$AWS_ACCESS_KEY \
|
||||
secret_access_key=$AWS_SECRET_KEY \
|
||||
region=us-east-1
|
||||
access_key_id=$AWS_ACCESS_KEY \
|
||||
secret_access_key=$AWS_SECRET_KEY \
|
||||
region=us-east-1
|
||||
|
||||
# Configure R2 destination
|
||||
rclone config create r2-dest s3 \
|
||||
access_key_id=$R2_ACCESS_KEY \
|
||||
secret_access_key=$R2_SECRET_KEY \
|
||||
endpoint=https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com \
|
||||
region=auto
|
||||
access_key_id=$R2_ACCESS_KEY \
|
||||
secret_access_key=$R2_SECRET_KEY \
|
||||
endpoint=https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com \
|
||||
region=auto
|
||||
|
||||
# Copy data
|
||||
rclone copy s3-source:my-bucket r2-dest:my-bucket --progress
|
||||
|
|
@ -361,17 +360,17 @@ ROI: 3.4 months
|
|||
### Prepared for Future Features
|
||||
|
||||
```typescript
|
||||
// Brainy v4.0.0 is ready for R2 lifecycle features
|
||||
// Brainy is ready for R2 lifecycle features
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
id: 'archive-old-data',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'INFREQUENT_ACCESS' }, // When available
|
||||
{ days: 90, storageClass: 'ARCHIVE' }
|
||||
]
|
||||
}]
|
||||
rules: [{
|
||||
id: 'archive-old-data',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'INFREQUENT_ACCESS' }, // When available
|
||||
{ days: 90, storageClass: 'ARCHIVE' }
|
||||
]
|
||||
}]
|
||||
})
|
||||
|
||||
// Expected cost impact (when lifecycle is available):
|
||||
|
|
@ -388,11 +387,11 @@ await storage.setLifecyclePolicy({
|
|||
```typescript
|
||||
// Ensure correct endpoint format
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
|
||||
// NOT: `https://r2.cloudflarestorage.com/${accountId}`
|
||||
endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
|
||||
// NOT: `https://r2.cloudflarestorage.com/${accountId}`
|
||||
|
||||
region: 'auto', // R2 requires 'auto'
|
||||
forcePathStyle: false // R2 uses virtual-hosted-style
|
||||
region: 'auto', // R2 requires 'auto'
|
||||
forcePathStyle: false // R2 uses virtual-hosted-style
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -446,7 +445,6 @@ const storage = new S3CompatibleStorage({
|
|||
|
||||
---
|
||||
|
||||
**Version**: v4.0.0
|
||||
**Last Updated**: 2025-10-17
|
||||
**Cloud Provider**: Cloudflare R2
|
||||
**Key Advantage**: **$0 egress fees forever**
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
# Google Cloud Storage Cost Optimization Guide for Brainy v4.0.0
|
||||
|
||||
# Google Cloud Storage Cost Optimization Guide for Brainy
|
||||
> **Cost Impact**: Reduce storage costs from $138k/year to $8.3k/year at 500TB scale (**94% savings**)
|
||||
|
||||
## Overview
|
||||
|
||||
Brainy v4.0.0 provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management.
|
||||
Brainy provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management.
|
||||
|
||||
## Cost Breakdown (Before Optimization)
|
||||
|
||||
|
|
@ -35,8 +34,8 @@ import { GcsStorage } from '@soulcraft/brainy/storage'
|
|||
|
||||
// Initialize Brainy with GCS storage
|
||||
const storage = new GcsStorage({
|
||||
bucketName: 'my-brainy-data',
|
||||
keyFilename: './service-account.json' // Or use ADC
|
||||
bucketName: 'my-brainy-data',
|
||||
keyFilename: './service-account.json' // Or use ADC
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
|
|
@ -44,16 +43,16 @@ await brain.init()
|
|||
|
||||
// Set lifecycle policy for automatic archival
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
condition: { age: 30 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
||||
}, {
|
||||
condition: { age: 90 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
||||
}, {
|
||||
condition: { age: 365 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
||||
}]
|
||||
rules: [{
|
||||
condition: { age: 30 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
||||
}, {
|
||||
condition: { age: 90 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
||||
}, {
|
||||
condition: { age: 365 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
||||
}]
|
||||
})
|
||||
|
||||
// Verify lifecycle policy
|
||||
|
|
@ -70,10 +69,10 @@ console.log('Lifecycle policy active:', policy.rules.length, 'rules')
|
|||
- 10% of data 365+ days old (Archive)
|
||||
|
||||
```
|
||||
Standard (200TB): 200TB × $0.020/GB × 12 = $48,000/year
|
||||
Nearline (150TB): 150TB × $0.010/GB × 12 = $18,000/year
|
||||
Coldline (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
||||
Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year
|
||||
Standard (200TB): 200TB × $0.020/GB × 12 = $48,000/year
|
||||
Nearline (150TB): 150TB × $0.010/GB × 12 = $18,000/year
|
||||
Coldline (100TB): 100TB × $0.004/GB × 12 = $4,800/year
|
||||
Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year
|
||||
|
||||
Total Storage Cost: $71,520/year
|
||||
Total with Operations: ~$76,500/year
|
||||
|
|
@ -89,7 +88,7 @@ Savings: $66,500/year (46%)
|
|||
```typescript
|
||||
// Enable Autoclass for automatic tier management
|
||||
await storage.enableAutoclass({
|
||||
terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier
|
||||
terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier
|
||||
})
|
||||
|
||||
// Benefits:
|
||||
|
|
@ -116,10 +115,10 @@ await storage.enableAutoclass({
|
|||
- 40% Archive (cold data)
|
||||
|
||||
```
|
||||
Standard (50TB): 50TB × $0.020/GB × 12 = $12,000/year
|
||||
Nearline (75TB): 75TB × $0.010/GB × 12 = $9,000/year
|
||||
Coldline (175TB): 175TB × $0.004/GB × 12 = $8,400/year
|
||||
Archive (200TB): 200TB × $0.0012/GB × 12 = $2,880/year
|
||||
Standard (50TB): 50TB × $0.020/GB × 12 = $12,000/year
|
||||
Nearline (75TB): 75TB × $0.010/GB × 12 = $9,000/year
|
||||
Coldline (175TB): 175TB × $0.004/GB × 12 = $8,400/year
|
||||
Archive (200TB): 200TB × $0.0012/GB × 12 = $2,880/year
|
||||
|
||||
Total Storage Cost: $32,280/year
|
||||
Total with Operations: ~$37,000/year
|
||||
|
|
@ -133,24 +132,24 @@ Savings vs Standard: $106,000/year (74%)
|
|||
```typescript
|
||||
// Enable Autoclass for vectors (frequently searched)
|
||||
await storage.enableAutoclass({
|
||||
terminalStorageClass: 'COLDLINE' // Don't archive vectors deeply
|
||||
terminalStorageClass: 'COLDLINE' // Don't archive vectors deeply
|
||||
})
|
||||
|
||||
// Set lifecycle policy for metadata (less frequently accessed)
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
condition: { age: 30, matchesPrefix: ['entities/nouns/metadata/'] },
|
||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
||||
}, {
|
||||
condition: { age: 60, matchesPrefix: ['entities/nouns/metadata/'] },
|
||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
||||
}, {
|
||||
condition: { age: 180, matchesPrefix: ['entities/nouns/metadata/'] },
|
||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
||||
}, {
|
||||
condition: { age: 730, matchesPrefix: ['_system/'] },
|
||||
action: { type: 'Delete' } // Delete old system files after 2 years
|
||||
}]
|
||||
rules: [{
|
||||
condition: { age: 30, matchesPrefix: ['entities/nouns/metadata/'] },
|
||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
||||
}, {
|
||||
condition: { age: 60, matchesPrefix: ['entities/nouns/metadata/'] },
|
||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
||||
}, {
|
||||
condition: { age: 180, matchesPrefix: ['entities/nouns/metadata/'] },
|
||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
||||
}, {
|
||||
condition: { age: 730, matchesPrefix: ['_system/'] },
|
||||
action: { type: 'Delete' } // Delete old system files after 2 years
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -158,18 +157,18 @@ await storage.setLifecyclePolicy({
|
|||
|
||||
**Vectors (300TB with Autoclass):**
|
||||
```
|
||||
Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year
|
||||
Nearline (45TB): 45TB × $0.010/GB × 12 = $5,400/year
|
||||
Coldline (225TB): 225TB × $0.004/GB × 12 = $10,800/year
|
||||
Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year
|
||||
Nearline (45TB): 45TB × $0.010/GB × 12 = $5,400/year
|
||||
Coldline (225TB): 225TB × $0.004/GB × 12 = $10,800/year
|
||||
Subtotal: $23,400/year
|
||||
```
|
||||
|
||||
**Metadata (200TB with Lifecycle Policy):**
|
||||
```
|
||||
Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year
|
||||
Nearline (40TB): 40TB × $0.010/GB × 12 = $4,800/year
|
||||
Coldline (80TB): 80TB × $0.004/GB × 12 = $3,840/year
|
||||
Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year
|
||||
Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year
|
||||
Nearline (40TB): 40TB × $0.010/GB × 12 = $4,800/year
|
||||
Coldline (80TB): 80TB × $0.004/GB × 12 = $3,840/year
|
||||
Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year
|
||||
Subtotal: $16,560/year
|
||||
```
|
||||
|
||||
|
|
@ -182,16 +181,16 @@ Subtotal: $16,560/year
|
|||
|
||||
```typescript
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
condition: { age: 14 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
||||
}, {
|
||||
condition: { age: 30 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
||||
}, {
|
||||
condition: { age: 90 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
||||
}]
|
||||
rules: [{
|
||||
condition: { age: 14 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'NEARLINE' }
|
||||
}, {
|
||||
condition: { age: 30 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'COLDLINE' }
|
||||
}, {
|
||||
condition: { age: 90 },
|
||||
action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' }
|
||||
}]
|
||||
})
|
||||
|
||||
// Note: Archive class has 365-day minimum storage duration
|
||||
|
|
@ -202,10 +201,10 @@ await storage.setLifecyclePolicy({
|
|||
|
||||
**After 1 year:**
|
||||
```
|
||||
Standard (25TB): 25TB × $0.020/GB × 12 = $6,000/year
|
||||
Nearline (50TB): 50TB × $0.010/GB × 12 = $6,000/year
|
||||
Coldline (75TB): 75TB × $0.004/GB × 12 = $3,600/year
|
||||
Archive (350TB): 350TB × $0.0012/GB × 12 = $5,040/year
|
||||
Standard (25TB): 25TB × $0.020/GB × 12 = $6,000/year
|
||||
Nearline (50TB): 50TB × $0.010/GB × 12 = $6,000/year
|
||||
Coldline (75TB): 75TB × $0.004/GB × 12 = $3,600/year
|
||||
Archive (350TB): 350TB × $0.0012/GB × 12 = $5,040/year
|
||||
|
||||
Total Storage Cost: $20,640/year
|
||||
Total with Operations: ~$25,500/year
|
||||
|
|
@ -241,15 +240,15 @@ Warning: High retrieval costs if archived data is accessed frequently
|
|||
### Efficient Cleanup
|
||||
|
||||
```typescript
|
||||
// v4.0.0: Batch delete (100 objects per request for GCS)
|
||||
// Batch delete (100 objects per request for GCS)
|
||||
const idsToDelete = [/* array of entity IDs */]
|
||||
|
||||
const paths = idsToDelete.flatMap(id => {
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
const shard = id.substring(0, 2)
|
||||
return [
|
||||
`entities/nouns/vectors/${shard}/${id}.json`,
|
||||
`entities/nouns/metadata/${shard}/${id}.json`
|
||||
]
|
||||
})
|
||||
|
||||
// Batch delete
|
||||
|
|
@ -271,9 +270,9 @@ console.log('Terminal class:', status.terminalStorageClass)
|
|||
|
||||
// Example output:
|
||||
// {
|
||||
// enabled: true,
|
||||
// terminalStorageClass: 'ARCHIVE',
|
||||
// toggleTime: '2025-01-15T10:30:00Z'
|
||||
// enabled: true,
|
||||
// terminalStorageClass: 'ARCHIVE',
|
||||
// toggleTime: '2025-01-15T10:30:00Z'
|
||||
// }
|
||||
```
|
||||
|
||||
|
|
@ -336,7 +335,7 @@ Monthly cost trend: Decreasing 8-12% per month as data ages into cheaper classes
|
|||
// Check Autoclass status
|
||||
const status = await storage.getAutoclassStatus()
|
||||
if (!status.enabled) {
|
||||
await storage.enableAutoclass({ terminalStorageClass: 'ARCHIVE' })
|
||||
await storage.enableAutoclass({ terminalStorageClass: 'ARCHIVE' })
|
||||
}
|
||||
|
||||
// Autoclass requires 24-48 hours for initial transitions
|
||||
|
|
@ -365,8 +364,8 @@ if (!status.enabled) {
|
|||
```typescript
|
||||
// Use ADC instead of service account key file
|
||||
const storage = new GcsStorage({
|
||||
bucketName: 'my-brainy-data'
|
||||
// No keyFilename needed - uses ADC automatically
|
||||
bucketName: 'my-brainy-data'
|
||||
// No keyFilename needed - uses ADC automatically
|
||||
})
|
||||
|
||||
// ADC authentication order:
|
||||
|
|
@ -417,6 +416,5 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
|
|||
|
||||
---
|
||||
|
||||
**Version**: v4.0.0
|
||||
**Last Updated**: 2025-10-17
|
||||
**Cloud Provider**: Google Cloud Storage
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue