fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -1,8 +1,8 @@
|
|||
# Brainy v4.0.0 Cloud Deployment Guide
|
||||
# Brainy Cloud Deployment Guide
|
||||
|
||||
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy v4.0.0 codebase.
|
||||
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy codebase.
|
||||
|
||||
## 🆕 v4.0.0 Production Features
|
||||
## 🆕 Production Features
|
||||
|
||||
**Cost Optimization at Scale:**
|
||||
- **Lifecycle Policies**: Automatic tier transitions for 96% cost savings
|
||||
|
|
@ -12,7 +12,7 @@ This guide provides production-ready deployment configurations for Brainy using
|
|||
|
||||
**Example Impact (500TB dataset):**
|
||||
- Before: $138,000/year
|
||||
- With v4.0.0 lifecycle policies: $5,940/year
|
||||
- With lifecycle policies: $5,940/year
|
||||
- **Savings: $132,060/year (96%)**
|
||||
|
||||
See the [Cost Optimization](#cost-optimization-v40) section below for implementation details.
|
||||
|
|
@ -43,53 +43,53 @@ let brain
|
|||
let handler
|
||||
|
||||
exports.handler = async (event) => {
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: process.env.AWS_REGION,
|
||||
bucket: process.env.BRAINY_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
prefix: 'brainy-data/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
auth: {
|
||||
required: true,
|
||||
apiKeys: [process.env.API_KEY]
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
// Get the universal handler from the augmentation
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Lambda event to Request
|
||||
const url = `https://${event.requestContext.domainName}${event.rawPath}`
|
||||
const request = new Request(url, {
|
||||
method: event.requestContext.http.method,
|
||||
headers: event.headers,
|
||||
body: event.body
|
||||
})
|
||||
|
||||
// Use the universal handler
|
||||
const response = await handler(request)
|
||||
|
||||
return {
|
||||
statusCode: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: process.env.AWS_REGION,
|
||||
bucket: process.env.BRAINY_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
prefix: 'brainy-data/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
auth: {
|
||||
required: true,
|
||||
apiKeys: [process.env.API_KEY]
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
// Get the universal handler from the augmentation
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Lambda event to Request
|
||||
const url = `https://${event.requestContext.domainName}${event.rawPath}`
|
||||
const request = new Request(url, {
|
||||
method: event.requestContext.http.method,
|
||||
headers: event.headers,
|
||||
body: event.body
|
||||
})
|
||||
|
||||
// Use the universal handler
|
||||
const response = await handler(request)
|
||||
|
||||
return {
|
||||
statusCode: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -104,36 +104,36 @@ let brain
|
|||
let handler
|
||||
|
||||
exports.brainyAPI = async (req, res) => {
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET,
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Express req/res to Request/Response
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET,
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Express req/res to Request/Response
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -156,39 +156,39 @@ let brain
|
|||
let handler
|
||||
|
||||
async function initBrainy() {
|
||||
// Google Cloud Storage is S3-compatible
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET || 'brainy-data',
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: process.env.GCS_REGION || 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT,
|
||||
cors: {
|
||||
origin: process.env.CORS_ORIGIN || '*'
|
||||
},
|
||||
auth: {
|
||||
required: process.env.AUTH_REQUIRED === 'true',
|
||||
apiKeys: process.env.API_KEY ? [process.env.API_KEY] : []
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
// Google Cloud Storage is S3-compatible
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET || 'brainy-data',
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: process.env.GCS_REGION || 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT,
|
||||
cors: {
|
||||
origin: process.env.CORS_ORIGIN || '*'
|
||||
},
|
||||
auth: {
|
||||
required: process.env.AUTH_REQUIRED === 'true',
|
||||
apiKeys: process.env.API_KEY ? [process.env.API_KEY] : []
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Initialize on startup
|
||||
|
|
@ -196,26 +196,26 @@ await initBrainy()
|
|||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'healthy', service: 'brainy-api' })
|
||||
res.json({ status: 'healthy', service: 'brainy-api' })
|
||||
})
|
||||
|
||||
// Universal handler for all API routes
|
||||
app.use('*', async (req, res) => {
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
})
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -237,43 +237,43 @@ CMD ["node", "server.js"]
|
|||
```yaml
|
||||
# cloudbuild.yaml
|
||||
steps:
|
||||
# Build the container image
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.']
|
||||
|
||||
# Push to Container Registry
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA']
|
||||
|
||||
# Deploy to Cloud Run
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
entrypoint: gcloud
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- 'brainy-api'
|
||||
- '--image'
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
- '--region'
|
||||
- 'us-central1'
|
||||
- '--platform'
|
||||
- 'managed'
|
||||
- '--allow-unauthenticated'
|
||||
- '--set-env-vars'
|
||||
- 'GCS_BUCKET=brainy-storage,GCS_REGION=US'
|
||||
- '--set-secrets'
|
||||
- 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest'
|
||||
- '--memory'
|
||||
- '2Gi'
|
||||
- '--cpu'
|
||||
- '2'
|
||||
- '--max-instances'
|
||||
- '100'
|
||||
- '--min-instances'
|
||||
- '0'
|
||||
# Build the container image
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.']
|
||||
|
||||
# Push to Container Registry
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA']
|
||||
|
||||
# Deploy to Cloud Run
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
entrypoint: gcloud
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- 'brainy-api'
|
||||
- '--image'
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
- '--region'
|
||||
- 'us-central1'
|
||||
- '--platform'
|
||||
- 'managed'
|
||||
- '--allow-unauthenticated'
|
||||
- '--set-env-vars'
|
||||
- 'GCS_BUCKET=brainy-storage,GCS_REGION=US'
|
||||
- '--set-secrets'
|
||||
- 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest'
|
||||
- '--memory'
|
||||
- '2Gi'
|
||||
- '--cpu'
|
||||
- '2'
|
||||
- '--max-instances'
|
||||
- '100'
|
||||
- '--min-instances'
|
||||
- '0'
|
||||
|
||||
images:
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
```
|
||||
|
||||
**Deploy with gcloud CLI:**
|
||||
|
|
@ -283,12 +283,12 @@ gcloud builds submit --config cloudbuild.yaml
|
|||
|
||||
# Or deploy directly
|
||||
gcloud run deploy brainy-api \
|
||||
--source . \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated \
|
||||
--set-env-vars GCS_BUCKET=brainy-storage \
|
||||
--set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest"
|
||||
--source . \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated \
|
||||
--set-env-vars GCS_BUCKET=brainy-storage \
|
||||
--set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest"
|
||||
```
|
||||
|
||||
**Create GCS Bucket with S3-compatible access:**
|
||||
|
|
@ -312,37 +312,37 @@ echo -n "YOUR_SECRET_KEY" | gcloud secrets create gcs-secret-key --data-file=-
|
|||
```javascript
|
||||
// index.js
|
||||
module.exports = async function (context, req) {
|
||||
const { Brainy } = require('@soulcraft/brainy')
|
||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
||||
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
secretAccessKey: process.env.AZURE_STORAGE_KEY,
|
||||
prefix: 'entities/',
|
||||
forcePathStyle: false
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
const handler = apiAugmentation.createUniversalHandler()
|
||||
|
||||
const request = new Request(`https://${context.req.headers.host}${context.req.url}`, {
|
||||
method: context.req.method,
|
||||
headers: context.req.headers,
|
||||
body: JSON.stringify(context.req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
context.res = {
|
||||
status: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
const { Brainy } = require('@soulcraft/brainy')
|
||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
||||
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
secretAccessKey: process.env.AZURE_STORAGE_KEY,
|
||||
prefix: 'entities/',
|
||||
forcePathStyle: false
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
const handler = apiAugmentation.createUniversalHandler()
|
||||
|
||||
const request = new Request(`https://${context.req.headers.host}${context.req.url}`, {
|
||||
method: context.req.method,
|
||||
headers: context.req.headers,
|
||||
body: JSON.stringify(context.req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
context.res = {
|
||||
status: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -356,37 +356,37 @@ import { R2Storage } from '@soulcraft/brainy/storage' // Alias for S3CompatibleS
|
|||
let handler
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx) {
|
||||
if (!handler) {
|
||||
const storage = new R2Storage({
|
||||
endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_SECRET_ACCESS_KEY,
|
||||
region: 'auto',
|
||||
forcePathStyle: true
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
cors: { origin: '*' }
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// The handler works directly with Request/Response!
|
||||
return handler(request)
|
||||
}
|
||||
async fetch(request, env, ctx) {
|
||||
if (!handler) {
|
||||
const storage = new R2Storage({
|
||||
endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_SECRET_ACCESS_KEY,
|
||||
region: 'auto',
|
||||
forcePathStyle: true
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
cors: { origin: '*' }
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// The handler works directly with Request/Response!
|
||||
return handler(request)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -418,52 +418,52 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|||
let handler
|
||||
|
||||
export const config = {
|
||||
runtime: 'edge',
|
||||
runtime: 'edge',
|
||||
}
|
||||
|
||||
export default async (request) => {
|
||||
if (!handler) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
return handler(request)
|
||||
if (!handler) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
return handler(request)
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
// vercel.json
|
||||
{
|
||||
"functions": {
|
||||
"api/brainy.js": {
|
||||
"maxDuration": 30,
|
||||
"memory": 1024
|
||||
}
|
||||
},
|
||||
"rewrites": [
|
||||
{
|
||||
"source": "/api/:path*",
|
||||
"destination": "/api/brainy"
|
||||
}
|
||||
]
|
||||
"functions": {
|
||||
"api/brainy.js": {
|
||||
"maxDuration": 30,
|
||||
"memory": 1024
|
||||
}
|
||||
},
|
||||
"rewrites": [
|
||||
{
|
||||
"source": "/api/:path*",
|
||||
"destination": "/api/brainy"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -482,51 +482,51 @@ let brain
|
|||
let handler
|
||||
|
||||
async function init() {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.S3_ACCESS_KEY,
|
||||
secretAccessKey: process.env.S3_SECRET_KEY,
|
||||
prefix: 'brainy/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.S3_ACCESS_KEY,
|
||||
secretAccessKey: process.env.S3_SECRET_KEY,
|
||||
prefix: 'brainy/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
await init()
|
||||
|
||||
// Universal handler for all routes
|
||||
app.use('*', async (req, res) => {
|
||||
const request = new Request(`http://localhost${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
const request = new Request(`http://localhost${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
})
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -552,26 +552,26 @@ restartPolicyMaxRetries = 3
|
|||
```yaml
|
||||
# render.yaml
|
||||
services:
|
||||
- type: web
|
||||
name: brainy-api
|
||||
runtime: node
|
||||
buildCommand: npm install
|
||||
startCommand: node server.js
|
||||
envVars:
|
||||
- key: S3_BUCKET
|
||||
value: brainy-data
|
||||
- key: S3_ENDPOINT
|
||||
value: s3.amazonaws.com
|
||||
- key: S3_REGION
|
||||
value: us-east-1
|
||||
- key: S3_ACCESS_KEY
|
||||
sync: false
|
||||
- key: S3_SECRET_KEY
|
||||
sync: false
|
||||
- key: API_KEY
|
||||
generateValue: true
|
||||
healthCheckPath: /health
|
||||
autoDeploy: true
|
||||
- type: web
|
||||
name: brainy-api
|
||||
runtime: node
|
||||
buildCommand: npm install
|
||||
startCommand: node server.js
|
||||
envVars:
|
||||
- key: S3_BUCKET
|
||||
value: brainy-data
|
||||
- key: S3_ENDPOINT
|
||||
value: s3.amazonaws.com
|
||||
- key: S3_REGION
|
||||
value: us-east-1
|
||||
- key: S3_ACCESS_KEY
|
||||
sync: false
|
||||
- key: S3_SECRET_KEY
|
||||
sync: false
|
||||
- key: API_KEY
|
||||
generateValue: true
|
||||
healthCheckPath: /health
|
||||
autoDeploy: true
|
||||
```
|
||||
|
||||
### Deno Deploy
|
||||
|
|
@ -582,19 +582,19 @@ import { Brainy } from "npm:@soulcraft/brainy"
|
|||
import { S3CompatibleStorage } from "npm:@soulcraft/brainy/storage"
|
||||
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com",
|
||||
bucket: Deno.env.get("S3_BUCKET")!,
|
||||
accessKeyId: Deno.env.get("S3_ACCESS_KEY")!,
|
||||
secretAccessKey: Deno.env.get("S3_SECRET_KEY")!,
|
||||
region: "auto"
|
||||
endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com",
|
||||
bucket: Deno.env.get("S3_BUCKET")!,
|
||||
accessKeyId: Deno.env.get("S3_ACCESS_KEY")!,
|
||||
secretAccessKey: Deno.env.get("S3_SECRET_KEY")!,
|
||||
region: "auto"
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
|
@ -653,29 +653,29 @@ RATE_LIMIT_MAX=100
|
|||
```javascript
|
||||
// REST API Client
|
||||
const response = await fetch('https://your-api.com/api/brainy/add', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: 'Your content here',
|
||||
metadata: { type: 'document' }
|
||||
})
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: 'Your content here',
|
||||
metadata: { type: 'document' }
|
||||
})
|
||||
})
|
||||
|
||||
const { id } = await response.json()
|
||||
|
||||
// Search
|
||||
const searchResponse = await fetch('https://your-api.com/api/brainy/find', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: 'neural networks'
|
||||
})
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: 'neural networks'
|
||||
})
|
||||
})
|
||||
|
||||
const results = await searchResponse.json()
|
||||
|
|
@ -683,14 +683,14 @@ const results = await searchResponse.json()
|
|||
// WebSocket Client
|
||||
const ws = new WebSocket('wss://your-api.com/ws')
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({
|
||||
type: 'subscribe',
|
||||
pattern: 'technology'
|
||||
}))
|
||||
ws.send(JSON.stringify({
|
||||
type: 'subscribe',
|
||||
pattern: 'technology'
|
||||
}))
|
||||
}
|
||||
ws.onmessage = (event) => {
|
||||
const update = JSON.parse(event.data)
|
||||
console.log('Real-time update:', update)
|
||||
const update = JSON.parse(event.data)
|
||||
console.log('Real-time update:', update)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -700,13 +700,13 @@ S3CompatibleStorage constructor parameters (verified from source):
|
|||
|
||||
```javascript
|
||||
{
|
||||
endpoint: string, // Required (e.g., 's3.amazonaws.com')
|
||||
bucket: string, // Required
|
||||
accessKeyId: string, // Required
|
||||
secretAccessKey: string, // Required
|
||||
region?: string, // Optional (default: 'us-east-1')
|
||||
prefix?: string, // Optional (e.g., 'brainy/')
|
||||
forcePathStyle?: boolean, // Optional (needed for some S3-compatible services)
|
||||
endpoint: string, // Required (e.g., 's3.amazonaws.com')
|
||||
bucket: string, // Required
|
||||
accessKeyId: string, // Required
|
||||
secretAccessKey: string, // Required
|
||||
region?: string, // Optional (default: 'us-east-1')
|
||||
prefix?: string, // Optional (e.g., 'brainy/')
|
||||
forcePathStyle?: boolean, // Optional (needed for some S3-compatible services)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -727,7 +727,7 @@ S3CompatibleStorage constructor parameters (verified from source):
|
|||
4. **Implement rate limiting** to prevent abuse
|
||||
5. **Configure CORS** appropriately for your use case
|
||||
|
||||
## Cost Optimization (v4.0.0)
|
||||
## Cost Optimization
|
||||
|
||||
### Enable Lifecycle Policies
|
||||
|
||||
|
|
@ -738,16 +738,16 @@ const storage = brain.storage
|
|||
|
||||
// Set lifecycle policy for automatic archival
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
id: 'archive-old-data',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days
|
||||
{ days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days
|
||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year
|
||||
]
|
||||
}]
|
||||
rules: [{
|
||||
id: 'archive-old-data',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days
|
||||
{ days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days
|
||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year
|
||||
]
|
||||
}]
|
||||
})
|
||||
|
||||
// Or enable Intelligent-Tiering for hands-off optimization
|
||||
|
|
@ -765,14 +765,14 @@ await storage.enableIntelligentTiering('entities/', 'auto-optimize')
|
|||
|
||||
**Efficient bulk deletions:**
|
||||
```javascript
|
||||
// v4.0.0: Batch delete (1000 objects per request)
|
||||
// Batch delete (1000 objects per request)
|
||||
const idsToDelete = [/* array of entity IDs */]
|
||||
const paths = idsToDelete.flatMap(id => [
|
||||
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
||||
`entities/nouns/metadata/${id.substring(0, 2)}/${id}.json`
|
||||
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
||||
`entities/nouns/metadata/${id.substring(0, 2)}/${id}.json`
|
||||
])
|
||||
|
||||
await storage.batchDelete(paths) // Much faster than individual deletes
|
||||
await storage.batchDelete(paths) // Much faster than individual deletes
|
||||
```
|
||||
|
||||
### Enable Compression (FileSystem)
|
||||
|
|
@ -780,8 +780,8 @@ await storage.batchDelete(paths) // Much faster than individual deletes
|
|||
**For local/server deployments:**
|
||||
```javascript
|
||||
const storage = new FileSystemStorage({
|
||||
path: './data',
|
||||
compression: true // 60-80% space savings
|
||||
path: './data',
|
||||
compression: true // 60-80% space savings
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
|
|
@ -793,8 +793,8 @@ const brain = new Brainy({ storage })
|
|||
2. **Use S3CompatibleStorage** for cloud deployments (better scalability)
|
||||
3. **Enable the cache augmentation** for frequently accessed data
|
||||
4. **Configure appropriate memory limits** for your runtime
|
||||
5. **v4.0.0**: Enable lifecycle policies to reduce storage costs by 96%
|
||||
6. **v4.0.0**: Use batch operations for cleanup tasks
|
||||
5. Enable lifecycle policies to reduce storage costs by 96%
|
||||
6. Use batch operations for cleanup tasks
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
|
@ -810,15 +810,15 @@ const brain = new Brainy({ storage })
|
|||
Enable debug logging by setting:
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
debug: true,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
verbose: true
|
||||
}
|
||||
}]
|
||||
storage,
|
||||
debug: true,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
verbose: true
|
||||
}
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -18,24 +18,24 @@ const { Brainy } = require('@soulcraft/brainy')
|
|||
let brain
|
||||
|
||||
exports.handler = async (event) => {
|
||||
// Brainy auto-detects Lambda environment and configures accordingly
|
||||
if (!brain) {
|
||||
brain = new Brainy() // Zero config - auto-adapts to Lambda
|
||||
await brain.init()
|
||||
}
|
||||
|
||||
const { method, ...params } = JSON.parse(event.body)
|
||||
|
||||
switch(method) {
|
||||
case 'add':
|
||||
const id = await brain.add(params)
|
||||
return { statusCode: 200, body: JSON.stringify({ id }) }
|
||||
case 'find':
|
||||
const results = await brain.find(params)
|
||||
return { statusCode: 200, body: JSON.stringify({ results }) }
|
||||
default:
|
||||
return { statusCode: 400, body: 'Unknown method' }
|
||||
}
|
||||
// Brainy auto-detects Lambda environment and configures accordingly
|
||||
if (!brain) {
|
||||
brain = new Brainy() // Zero config - auto-adapts to Lambda
|
||||
await brain.init()
|
||||
}
|
||||
|
||||
const { method, ...params } = JSON.parse(event.body)
|
||||
|
||||
switch(method) {
|
||||
case 'add':
|
||||
const id = await brain.add(params)
|
||||
return { statusCode: 200, body: JSON.stringify({ id }) }
|
||||
case 'find':
|
||||
const results = await brain.find(params)
|
||||
return { statusCode: 200, body: JSON.stringify({ results }) }
|
||||
default:
|
||||
return { statusCode: 400, body: 'Unknown method' }
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
|
|
@ -56,26 +56,26 @@ docker push $ECR_URI/brainy:latest
|
|||
# Deploy with minimal ECS task definition
|
||||
cat > task-definition.json << 'EOF'
|
||||
{
|
||||
"family": "brainy",
|
||||
"networkMode": "awsvpc",
|
||||
"requiresCompatibilities": ["FARGATE"],
|
||||
"cpu": "256",
|
||||
"memory": "512",
|
||||
"containerDefinitions": [{
|
||||
"name": "brainy",
|
||||
"image": "$ECR_URI/brainy:latest",
|
||||
"environment": [
|
||||
{"name": "NODE_ENV", "value": "production"}
|
||||
],
|
||||
"logConfiguration": {
|
||||
"logDriver": "awslogs",
|
||||
"options": {
|
||||
"awslogs-group": "/ecs/brainy",
|
||||
"awslogs-region": "us-east-1",
|
||||
"awslogs-stream-prefix": "ecs"
|
||||
}
|
||||
}
|
||||
}]
|
||||
"family": "brainy",
|
||||
"networkMode": "awsvpc",
|
||||
"requiresCompatibilities": ["FARGATE"],
|
||||
"cpu": "256",
|
||||
"memory": "512",
|
||||
"containerDefinitions": [{
|
||||
"name": "brainy",
|
||||
"image": "$ECR_URI/brainy:latest",
|
||||
"environment": [
|
||||
{"name": "NODE_ENV", "value": "production"}
|
||||
],
|
||||
"logConfiguration": {
|
||||
"logDriver": "awslogs",
|
||||
"options": {
|
||||
"awslogs-group": "/ecs/brainy",
|
||||
"awslogs-region": "us-east-1",
|
||||
"awslogs-stream-prefix": "ecs"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
EOF
|
||||
|
||||
|
|
@ -138,14 +138,14 @@ const brain = new Brainy()
|
|||
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
options: {
|
||||
bucket: process.env.S3_BUCKET || 'auto', // 'auto' creates bucket
|
||||
region: process.env.AWS_REGION || 'auto', // 'auto' detects region
|
||||
// IAM role provides credentials automatically
|
||||
}
|
||||
}
|
||||
storage: {
|
||||
type: 's3',
|
||||
options: {
|
||||
bucket: process.env.S3_BUCKET || 'auto', // 'auto' creates bucket
|
||||
region: process.env.AWS_REGION || 'auto', // 'auto' detects region
|
||||
// IAM role provides credentials automatically
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -156,23 +156,23 @@ const brain = new Brainy({
|
|||
```yaml
|
||||
# Auto-scaling policy
|
||||
Resources:
|
||||
AutoScalingTarget:
|
||||
Type: AWS::ApplicationAutoScaling::ScalableTarget
|
||||
Properties:
|
||||
ServiceNamespace: ecs
|
||||
ResourceId: service/default/brainy
|
||||
ScalableDimension: ecs:service:DesiredCount
|
||||
MinCapacity: 2
|
||||
MaxCapacity: 100
|
||||
|
||||
AutoScalingPolicy:
|
||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
||||
Properties:
|
||||
PolicyType: TargetTrackingScaling
|
||||
TargetTrackingScalingPolicyConfiguration:
|
||||
TargetValue: 70.0
|
||||
PredefinedMetricSpecification:
|
||||
PredefinedMetricType: ECSServiceAverageCPUUtilization
|
||||
AutoScalingTarget:
|
||||
Type: AWS::ApplicationAutoScaling::ScalableTarget
|
||||
Properties:
|
||||
ServiceNamespace: ecs
|
||||
ResourceId: service/default/brainy
|
||||
ScalableDimension: ecs:service:DesiredCount
|
||||
MinCapacity: 2
|
||||
MaxCapacity: 100
|
||||
|
||||
AutoScalingPolicy:
|
||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
||||
Properties:
|
||||
PolicyType: TargetTrackingScaling
|
||||
TargetTrackingScalingPolicyConfiguration:
|
||||
TargetValue: 70.0
|
||||
PredefinedMetricSpecification:
|
||||
PredefinedMetricType: ECSServiceAverageCPUUtilization
|
||||
```
|
||||
|
||||
### 2. Vertical Scaling
|
||||
|
|
@ -189,10 +189,10 @@ Brainy automatically adapts to available memory:
|
|||
```javascript
|
||||
// Brainy automatically handles multi-AZ with S3
|
||||
const brain = new Brainy({
|
||||
distributed: {
|
||||
enabled: true, // Auto-enables with S3 storage
|
||||
coordinationMethod: 'auto' // Uses S3 for coordination
|
||||
}
|
||||
distributed: {
|
||||
enabled: true, // Auto-enables with S3 storage
|
||||
coordinationMethod: 'auto' // Uses S3 for coordination
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -201,17 +201,17 @@ const brain = new Brainy({
|
|||
```bash
|
||||
# Application Load Balancer with health checks
|
||||
aws elbv2 create-load-balancer \
|
||||
--name brainy-alb \
|
||||
--subnets subnet-xxx subnet-yyy \
|
||||
--security-groups sg-xxx
|
||||
--name brainy-alb \
|
||||
--subnets subnet-xxx subnet-yyy \
|
||||
--security-groups sg-xxx
|
||||
|
||||
aws elbv2 create-target-group \
|
||||
--name brainy-targets \
|
||||
--protocol HTTP \
|
||||
--port 3000 \
|
||||
--vpc-id vpc-xxx \
|
||||
--health-check-path /health \
|
||||
--health-check-interval-seconds 30
|
||||
--name brainy-targets \
|
||||
--protocol HTTP \
|
||||
--port 3000 \
|
||||
--vpc-id vpc-xxx \
|
||||
--health-check-path /health \
|
||||
--health-check-interval-seconds 30
|
||||
```
|
||||
|
||||
## Monitoring & Observability
|
||||
|
|
@ -233,16 +233,16 @@ Brainy automatically sends metrics when running on AWS:
|
|||
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
monitoring: {
|
||||
enabled: true,
|
||||
customMetrics: {
|
||||
namespace: 'Brainy/Production',
|
||||
dimensions: [
|
||||
{ Name: 'Environment', Value: 'production' },
|
||||
{ Name: 'Service', Value: 'api' }
|
||||
]
|
||||
}
|
||||
}
|
||||
monitoring: {
|
||||
enabled: true,
|
||||
customMetrics: {
|
||||
namespace: 'Brainy/Production',
|
||||
dimensions: [
|
||||
{ Name: 'Environment', Value: 'production' },
|
||||
{ Name: 'Service', Value: 'api' }
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -252,22 +252,22 @@ const brain = new Brainy({
|
|||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::brainy-*/*",
|
||||
"arn:aws:s3:::brainy-*"
|
||||
]
|
||||
}
|
||||
]
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::brainy-*/*",
|
||||
"arn:aws:s3:::brainy-*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -285,12 +285,12 @@ aws ec2 create-subnet --vpc-id vpc-xxx --cidr-block 10.0.2.0/24 --availability-z
|
|||
```javascript
|
||||
// Automatic encryption with S3
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
options: {
|
||||
encryption: 'auto' // Uses S3 SSE-S3 by default
|
||||
}
|
||||
}
|
||||
storage: {
|
||||
type: 's3',
|
||||
options: {
|
||||
encryption: 'auto' // Uses S3 SSE-S3 by default
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -300,14 +300,14 @@ const brain = new Brainy({
|
|||
|
||||
```bash
|
||||
aws ec2 request-spot-fleet --spot-fleet-request-config '{
|
||||
"IamFleetRole": "arn:aws:iam::xxx:role/fleet-role",
|
||||
"TargetCapacity": 2,
|
||||
"SpotPrice": "0.05",
|
||||
"LaunchSpecifications": [{
|
||||
"ImageId": "ami-xxx",
|
||||
"InstanceType": "t3.medium",
|
||||
"UserData": "BASE64_ENCODED_STARTUP_SCRIPT"
|
||||
}]
|
||||
"IamFleetRole": "arn:aws:iam::xxx:role/fleet-role",
|
||||
"TargetCapacity": 2,
|
||||
"SpotPrice": "0.05",
|
||||
"LaunchSpecifications": [{
|
||||
"ImageId": "ami-xxx",
|
||||
"InstanceType": "t3.medium",
|
||||
"UserData": "BASE64_ENCODED_STARTUP_SCRIPT"
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
|
|
@ -316,12 +316,12 @@ aws ec2 request-spot-fleet --spot-fleet-request-config '{
|
|||
```javascript
|
||||
// Brainy automatically uses S3 Intelligent-Tiering
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
options: {
|
||||
storageClass: 'INTELLIGENT_TIERING' // Automatic cost optimization
|
||||
}
|
||||
}
|
||||
storage: {
|
||||
type: 's3',
|
||||
options: {
|
||||
storageClass: 'INTELLIGENT_TIERING' // Automatic cost optimization
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -329,8 +329,8 @@ const brain = new Brainy({
|
|||
|
||||
```bash
|
||||
aws lambda put-function-concurrency \
|
||||
--function-name brainy-handler \
|
||||
--reserved-concurrent-executions 10
|
||||
--function-name brainy-handler \
|
||||
--reserved-concurrent-executions 10
|
||||
```
|
||||
|
||||
## Deployment Automation
|
||||
|
|
@ -340,28 +340,28 @@ aws lambda put-function-concurrency \
|
|||
```yaml
|
||||
name: Deploy to AWS
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-east-1
|
||||
|
||||
- name: Deploy to ECS
|
||||
run: |
|
||||
docker build -t brainy .
|
||||
docker tag brainy:latest $ECR_URI/brainy:latest
|
||||
docker push $ECR_URI/brainy:latest
|
||||
aws ecs update-service --cluster default --service brainy --force-new-deployment
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-east-1
|
||||
|
||||
- name: Deploy to ECS
|
||||
run: |
|
||||
docker build -t brainy .
|
||||
docker tag brainy:latest $ECR_URI/brainy:latest
|
||||
docker push $ECR_URI/brainy:latest
|
||||
aws ecs update-service --cluster default --service brainy --force-new-deployment
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
|
@ -369,121 +369,121 @@ jobs:
|
|||
### Common Issues
|
||||
|
||||
1. **Storage Auto-Detection Fails**
|
||||
```javascript
|
||||
// Explicitly specify storage
|
||||
const brain = new Brainy({
|
||||
storage: { type: 's3', options: { bucket: 'my-bucket' } }
|
||||
})
|
||||
```
|
||||
```javascript
|
||||
// Explicitly specify storage
|
||||
const brain = new Brainy({
|
||||
storage: { type: 's3', options: { bucket: 'my-bucket' } }
|
||||
})
|
||||
```
|
||||
|
||||
2. **Memory Issues**
|
||||
```javascript
|
||||
// Optimize for low memory
|
||||
const brain = new Brainy({
|
||||
cache: { maxSize: 100 }, // Reduce cache size
|
||||
index: { M: 8 } // Reduce HNSW connections
|
||||
})
|
||||
```
|
||||
```javascript
|
||||
// Optimize for low memory
|
||||
const brain = new Brainy({
|
||||
cache: { maxSize: 100 }, // Reduce cache size
|
||||
index: { M: 8 } // Reduce HNSW connections
|
||||
})
|
||||
```
|
||||
|
||||
3. **Cold Starts (Lambda)**
|
||||
|
||||
**v7.3.0+ Progressive Initialization (Zero-Config)**
|
||||
**Progressive Initialization (Zero-Config)**
|
||||
|
||||
Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME)
|
||||
and uses progressive initialization for <200ms cold starts:
|
||||
Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME)
|
||||
and uses progressive initialization for <200ms cold starts:
|
||||
|
||||
```javascript
|
||||
// Zero-config - Brainy auto-detects Lambda
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'my-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
}
|
||||
}
|
||||
})
|
||||
await brain.init() // Returns in <200ms
|
||||
```javascript
|
||||
// Zero-config - Brainy auto-detects Lambda
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'my-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
}
|
||||
}
|
||||
})
|
||||
await brain.init() // Returns in <200ms
|
||||
|
||||
// First write validates bucket (lazy validation)
|
||||
await brain.add('noun', { name: 'test' }) // Validates here
|
||||
```
|
||||
// First write validates bucket (lazy validation)
|
||||
await brain.add('noun', { name: 'test' }) // Validates here
|
||||
```
|
||||
|
||||
**Manual Override (if needed)**
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'my-bucket',
|
||||
// Force specific mode
|
||||
initMode: 'progressive' // 'auto' | 'progressive' | 'strict'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
**Manual Override (if needed)**
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3',
|
||||
s3Storage: {
|
||||
bucketName: 'my-bucket',
|
||||
// Force specific mode
|
||||
initMode: 'progressive' // 'auto' | 'progressive' | 'strict'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
| Mode | Cold Start | Best For |
|
||||
|------|------------|----------|
|
||||
| `auto` (default) | <200ms in Lambda | Zero-config, auto-detects |
|
||||
| `progressive` | <200ms always | Force fast init everywhere |
|
||||
| `strict` | 100-500ms+ | Local dev, tests, debugging |
|
||||
| Mode | Cold Start | Best For |
|
||||
|------|------------|----------|
|
||||
| `auto` (default) | <200ms in Lambda | Zero-config, auto-detects |
|
||||
| `progressive` | <200ms always | Force fast init everywhere |
|
||||
| `strict` | 100-500ms+ | Local dev, tests, debugging |
|
||||
|
||||
**Warm-up (Alternative)**
|
||||
```javascript
|
||||
exports.warmup = async () => {
|
||||
if (!brain) {
|
||||
brain = new Brainy({ warmup: true })
|
||||
await brain.init()
|
||||
}
|
||||
}
|
||||
```
|
||||
**Warm-up (Alternative)**
|
||||
```javascript
|
||||
exports.warmup = async () => {
|
||||
if (!brain) {
|
||||
brain = new Brainy({ warmup: true })
|
||||
await brain.init()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Readiness Detection (v7.3.0+)**
|
||||
**Readiness Detection**
|
||||
|
||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
||||
|
||||
```javascript
|
||||
let brain
|
||||
```javascript
|
||||
let brain
|
||||
|
||||
exports.handler = async (event) => {
|
||||
if (!brain) {
|
||||
brain = new Brainy({ storage: { type: 's3', ... } })
|
||||
brain.init() // Fire and forget
|
||||
}
|
||||
exports.handler = async (event) => {
|
||||
if (!brain) {
|
||||
brain = new Brainy({ storage: { type: 's3', ... } })
|
||||
brain.init() // Fire and forget
|
||||
}
|
||||
|
||||
// Wait for initialization to complete
|
||||
await brain.ready
|
||||
// Wait for initialization to complete
|
||||
await brain.ready
|
||||
|
||||
// Now safe to use brain methods
|
||||
const results = await brain.find({ query: event.queryStringParameters.q })
|
||||
return { statusCode: 200, body: JSON.stringify(results) }
|
||||
}
|
||||
```
|
||||
// Now safe to use brain methods
|
||||
const results = await brain.find({ query: event.queryStringParameters.q })
|
||||
return { statusCode: 200, body: JSON.stringify(results) }
|
||||
}
|
||||
```
|
||||
|
||||
For health checks, use `isFullyInitialized()` to verify all background tasks are complete:
|
||||
For health checks, use `isFullyInitialized()` to verify all background tasks are complete:
|
||||
|
||||
```javascript
|
||||
exports.healthCheck = async () => {
|
||||
try {
|
||||
await brain.ready
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: JSON.stringify({
|
||||
status: 'ready',
|
||||
fullyInitialized: brain.isFullyInitialized()
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
statusCode: 503,
|
||||
body: JSON.stringify({ status: 'initializing' })
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
```javascript
|
||||
exports.healthCheck = async () => {
|
||||
try {
|
||||
await brain.ready
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: JSON.stringify({
|
||||
status: 'ready',
|
||||
fullyInitialized: brain.isFullyInitialized()
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
statusCode: 503,
|
||||
body: JSON.stringify({ status: 'initializing' })
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Production Checklist
|
||||
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ Deploy Brainy on GCP with automatic scaling, global distribution, and zero-confi
|
|||
```bash
|
||||
# Build and deploy with one command
|
||||
gcloud run deploy brainy \
|
||||
--source . \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated
|
||||
--source . \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated
|
||||
|
||||
# Brainy auto-detects Cloud Run and configures:
|
||||
# - Memory-optimized caching
|
||||
|
|
@ -30,44 +30,44 @@ const { Brainy } = require('@soulcraft/brainy')
|
|||
let brain
|
||||
|
||||
exports.brainyHandler = async (req, res) => {
|
||||
// Zero-config - auto-adapts to Cloud Functions
|
||||
if (!brain) {
|
||||
brain = new Brainy() // Detects GCP environment automatically
|
||||
await brain.init()
|
||||
}
|
||||
|
||||
const { method, ...params } = req.body
|
||||
|
||||
try {
|
||||
let result
|
||||
switch(method) {
|
||||
case 'add':
|
||||
result = await brain.add(params)
|
||||
break
|
||||
case 'find':
|
||||
result = await brain.find(params)
|
||||
break
|
||||
case 'relate':
|
||||
result = await brain.relate(params)
|
||||
break
|
||||
default:
|
||||
return res.status(400).json({ error: 'Unknown method' })
|
||||
}
|
||||
res.json({ result })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
}
|
||||
// Zero-config - auto-adapts to Cloud Functions
|
||||
if (!brain) {
|
||||
brain = new Brainy() // Detects GCP environment automatically
|
||||
await brain.init()
|
||||
}
|
||||
|
||||
const { method, ...params } = req.body
|
||||
|
||||
try {
|
||||
let result
|
||||
switch(method) {
|
||||
case 'add':
|
||||
result = await brain.add(params)
|
||||
break
|
||||
case 'find':
|
||||
result = await brain.find(params)
|
||||
break
|
||||
case 'relate':
|
||||
result = await brain.relate(params)
|
||||
break
|
||||
default:
|
||||
return res.status(400).json({ error: 'Unknown method' })
|
||||
}
|
||||
res.json({ result })
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message })
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Deploy:
|
||||
```bash
|
||||
gcloud functions deploy brainy \
|
||||
--runtime nodejs20 \
|
||||
--trigger-http \
|
||||
--entry-point brainyHandler \
|
||||
--memory 512MB \
|
||||
--timeout 60s
|
||||
--runtime nodejs20 \
|
||||
--trigger-http \
|
||||
--entry-point brainyHandler \
|
||||
--memory 512MB \
|
||||
--timeout 60s
|
||||
```
|
||||
|
||||
### Option 3: Google Kubernetes Engine (GKE)
|
||||
|
|
@ -75,7 +75,7 @@ gcloud functions deploy brainy \
|
|||
```bash
|
||||
# Create autopilot cluster (fully managed, zero-config)
|
||||
gcloud container clusters create-auto brainy-cluster \
|
||||
--region us-central1
|
||||
--region us-central1
|
||||
|
||||
# Deploy using Cloud Build
|
||||
gcloud builds submit --tag gcr.io/$PROJECT_ID/brainy
|
||||
|
|
@ -85,39 +85,39 @@ kubectl apply -f - <<EOF
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: brainy
|
||||
name: brainy
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: brainy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: brainy
|
||||
spec:
|
||||
containers:
|
||||
- name: brainy
|
||||
image: gcr.io/$PROJECT_ID/brainy
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
env:
|
||||
- name: NODE_ENV
|
||||
value: production
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: brainy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: brainy
|
||||
spec:
|
||||
containers:
|
||||
- name: brainy
|
||||
image: gcr.io/$PROJECT_ID/brainy
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
env:
|
||||
- name: NODE_ENV
|
||||
value: production
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: brainy-service
|
||||
name: brainy-service
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
selector:
|
||||
app: brainy
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 3000
|
||||
type: LoadBalancer
|
||||
selector:
|
||||
app: brainy
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 3000
|
||||
EOF
|
||||
```
|
||||
|
||||
|
|
@ -139,14 +139,14 @@ const brain = new Brainy()
|
|||
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 's3', // GCS is S3-compatible
|
||||
options: {
|
||||
endpoint: 'https://storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET || 'auto', // Auto-creates bucket
|
||||
// Uses Application Default Credentials automatically
|
||||
}
|
||||
}
|
||||
storage: {
|
||||
type: 's3', // GCS is S3-compatible
|
||||
options: {
|
||||
endpoint: 'https://storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET || 'auto', // Auto-creates bucket
|
||||
// Uses Application Default Credentials automatically
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -154,13 +154,13 @@ const brain = new Brainy({
|
|||
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'firestore',
|
||||
options: {
|
||||
projectId: process.env.GCP_PROJECT || 'auto',
|
||||
collection: 'brainy-data'
|
||||
}
|
||||
}
|
||||
storage: {
|
||||
type: 'firestore',
|
||||
options: {
|
||||
projectId: process.env.GCP_PROJECT || 'auto',
|
||||
collection: 'brainy-data'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -173,24 +173,24 @@ const brain = new Brainy({
|
|||
apiVersion: serving.knative.dev/v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: brainy
|
||||
annotations:
|
||||
run.googleapis.com/execution-environment: gen2
|
||||
name: brainy
|
||||
annotations:
|
||||
run.googleapis.com/execution-environment: gen2
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
autoscaling.knative.dev/minScale: "1"
|
||||
autoscaling.knative.dev/maxScale: "1000"
|
||||
autoscaling.knative.dev/target: "80"
|
||||
spec:
|
||||
containerConcurrency: 100
|
||||
containers:
|
||||
- image: gcr.io/PROJECT_ID/brainy
|
||||
resources:
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: "2Gi"
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
autoscaling.knative.dev/minScale: "1"
|
||||
autoscaling.knative.dev/maxScale: "1000"
|
||||
autoscaling.knative.dev/target: "80"
|
||||
spec:
|
||||
containerConcurrency: 100
|
||||
containers:
|
||||
- image: gcr.io/PROJECT_ID/brainy
|
||||
resources:
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: "2Gi"
|
||||
```
|
||||
|
||||
### 2. GKE Horizontal Pod Autoscaling
|
||||
|
|
@ -199,27 +199,27 @@ spec:
|
|||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: brainy-hpa
|
||||
name: brainy-hpa
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: brainy
|
||||
minReplicas: 3
|
||||
maxReplicas: 100
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: brainy
|
||||
minReplicas: 3
|
||||
maxReplicas: 100
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
```
|
||||
|
||||
## Global Distribution
|
||||
|
|
@ -229,11 +229,11 @@ spec:
|
|||
```javascript
|
||||
// Brainy automatically handles multi-region with GCS
|
||||
const brain = new Brainy({
|
||||
distributed: {
|
||||
enabled: true,
|
||||
regions: ['us-central1', 'europe-west1', 'asia-northeast1'],
|
||||
replication: 'auto' // Automatic cross-region replication
|
||||
}
|
||||
distributed: {
|
||||
enabled: true,
|
||||
regions: ['us-central1', 'europe-west1', 'asia-northeast1'],
|
||||
replication: 'auto' // Automatic cross-region replication
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -242,14 +242,14 @@ const brain = new Brainy({
|
|||
```bash
|
||||
# Global load balancing with Traffic Director
|
||||
gcloud compute backend-services create brainy-global \
|
||||
--global \
|
||||
--load-balancing-scheme=INTERNAL_SELF_MANAGED \
|
||||
--protocol=HTTP2
|
||||
--global \
|
||||
--load-balancing-scheme=INTERNAL_SELF_MANAGED \
|
||||
--protocol=HTTP2
|
||||
|
||||
gcloud compute backend-services add-backend brainy-global \
|
||||
--global \
|
||||
--network-endpoint-group=brainy-neg \
|
||||
--network-endpoint-group-region=us-central1
|
||||
--global \
|
||||
--network-endpoint-group=brainy-neg \
|
||||
--network-endpoint-group-region=us-central1
|
||||
```
|
||||
|
||||
## Monitoring & Observability
|
||||
|
|
@ -277,22 +277,22 @@ const { Monitoring } = require('@google-cloud/monitoring')
|
|||
const monitoring = new Monitoring.MetricServiceClient()
|
||||
|
||||
const brain = new Brainy({
|
||||
onMetric: async (metric) => {
|
||||
// Send custom metrics to Cloud Monitoring
|
||||
await monitoring.createTimeSeries({
|
||||
name: monitoring.projectPath(projectId),
|
||||
timeSeries: [{
|
||||
metric: {
|
||||
type: `custom.googleapis.com/brainy/${metric.name}`,
|
||||
labels: metric.labels
|
||||
},
|
||||
points: [{
|
||||
interval: { endTime: { seconds: Date.now() / 1000 } },
|
||||
value: { doubleValue: metric.value }
|
||||
}]
|
||||
}]
|
||||
})
|
||||
}
|
||||
onMetric: async (metric) => {
|
||||
// Send custom metrics to Cloud Monitoring
|
||||
await monitoring.createTimeSeries({
|
||||
name: monitoring.projectPath(projectId),
|
||||
timeSeries: [{
|
||||
metric: {
|
||||
type: `custom.googleapis.com/brainy/${metric.name}`,
|
||||
labels: metric.labels
|
||||
},
|
||||
points: [{
|
||||
interval: { endTime: { seconds: Date.now() / 1000 } },
|
||||
value: { doubleValue: metric.value }
|
||||
}]
|
||||
}]
|
||||
})
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -300,10 +300,10 @@ const brain = new Brainy({
|
|||
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
tracing: {
|
||||
enabled: true,
|
||||
sampleRate: 0.1 // Sample 10% of requests
|
||||
}
|
||||
tracing: {
|
||||
enabled: true,
|
||||
sampleRate: 0.1 // Sample 10% of requests
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -316,9 +316,9 @@ const brain = new Brainy({
|
|||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: brainy-sa
|
||||
annotations:
|
||||
iam.gke.io/gcp-service-account: brainy@PROJECT_ID.iam.gserviceaccount.com
|
||||
name: brainy-sa
|
||||
annotations:
|
||||
iam.gke.io/gcp-service-account: brainy@PROJECT_ID.iam.gserviceaccount.com
|
||||
```
|
||||
|
||||
### 2. Binary Authorization
|
||||
|
|
@ -328,11 +328,11 @@ metadata:
|
|||
apiVersion: binaryauthorization.grafeas.io/v1beta1
|
||||
kind: Policy
|
||||
metadata:
|
||||
name: brainy-policy
|
||||
name: brainy-policy
|
||||
spec:
|
||||
defaultAdmissionRule:
|
||||
requireAttestationsBy:
|
||||
- projects/PROJECT_ID/attestors/prod-attestor
|
||||
defaultAdmissionRule:
|
||||
requireAttestationsBy:
|
||||
- projects/PROJECT_ID/attestors/prod-attestor
|
||||
```
|
||||
|
||||
### 3. VPC Service Controls
|
||||
|
|
@ -340,9 +340,9 @@ spec:
|
|||
```bash
|
||||
# Create VPC Service Perimeter
|
||||
gcloud access-context-manager perimeters create brainy_perimeter \
|
||||
--resources=projects/PROJECT_NUMBER \
|
||||
--restricted-services=storage.googleapis.com \
|
||||
--title="Brainy Security Perimeter"
|
||||
--resources=projects/PROJECT_NUMBER \
|
||||
--restricted-services=storage.googleapis.com \
|
||||
--title="Brainy Security Perimeter"
|
||||
```
|
||||
|
||||
## Cost Optimization
|
||||
|
|
@ -354,16 +354,16 @@ gcloud access-context-manager perimeters create brainy_perimeter \
|
|||
apiVersion: container.cnrm.cloud.google.com/v1beta1
|
||||
kind: ContainerNodePool
|
||||
metadata:
|
||||
name: brainy-preemptible-pool
|
||||
name: brainy-preemptible-pool
|
||||
spec:
|
||||
clusterRef:
|
||||
name: brainy-cluster
|
||||
config:
|
||||
preemptible: true
|
||||
machineType: n2-standard-2
|
||||
autoscaling:
|
||||
minNodeCount: 1
|
||||
maxNodeCount: 10
|
||||
clusterRef:
|
||||
name: brainy-cluster
|
||||
config:
|
||||
preemptible: true
|
||||
machineType: n2-standard-2
|
||||
autoscaling:
|
||||
minNodeCount: 1
|
||||
maxNodeCount: 10
|
||||
```
|
||||
|
||||
### 2. Cloud CDN for Static Assets
|
||||
|
|
@ -371,11 +371,11 @@ spec:
|
|||
```bash
|
||||
# Enable Cloud CDN for frequently accessed data
|
||||
gcloud compute backend-buckets create brainy-assets \
|
||||
--gcs-bucket-name=brainy-static
|
||||
--gcs-bucket-name=brainy-static
|
||||
|
||||
gcloud compute backend-buckets update brainy-assets \
|
||||
--enable-cdn \
|
||||
--cache-mode=CACHE_ALL_STATIC
|
||||
--enable-cdn \
|
||||
--cache-mode=CACHE_ALL_STATIC
|
||||
```
|
||||
|
||||
### 3. Committed Use Discounts
|
||||
|
|
@ -383,8 +383,8 @@ gcloud compute backend-buckets update brainy-assets \
|
|||
```bash
|
||||
# Purchase committed use for predictable workloads
|
||||
gcloud compute commitments create brainy-commitment \
|
||||
--plan=TWELVE_MONTH \
|
||||
--resources=vcpu=100,memory=400
|
||||
--plan=TWELVE_MONTH \
|
||||
--resources=vcpu=100,memory=400
|
||||
```
|
||||
|
||||
## Deployment Automation
|
||||
|
|
@ -394,28 +394,28 @@ gcloud compute commitments create brainy-commitment \
|
|||
```yaml
|
||||
# cloudbuild.yaml
|
||||
steps:
|
||||
# Build container
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA', '.']
|
||||
|
||||
# Push to registry
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA']
|
||||
|
||||
# Deploy to Cloud Run
|
||||
- name: 'gcr.io/cloud-builders/gcloud'
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- 'brainy'
|
||||
- '--image=gcr.io/$PROJECT_ID/brainy:$SHORT_SHA'
|
||||
- '--region=us-central1'
|
||||
- '--platform=managed'
|
||||
# Build container
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA', '.']
|
||||
|
||||
# Push to registry
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA']
|
||||
|
||||
# Deploy to Cloud Run
|
||||
- name: 'gcr.io/cloud-builders/gcloud'
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- 'brainy'
|
||||
- '--image=gcr.io/$PROJECT_ID/brainy:$SHORT_SHA'
|
||||
- '--region=us-central1'
|
||||
- '--platform=managed'
|
||||
|
||||
# Trigger on push to main
|
||||
trigger:
|
||||
branch:
|
||||
name: main
|
||||
branch:
|
||||
name: main
|
||||
```
|
||||
|
||||
### Terraform Infrastructure
|
||||
|
|
@ -423,40 +423,40 @@ trigger:
|
|||
```hcl
|
||||
# main.tf
|
||||
resource "google_cloud_run_service" "brainy" {
|
||||
name = "brainy"
|
||||
location = "us-central1"
|
||||
name = "brainy"
|
||||
location = "us-central1"
|
||||
|
||||
template {
|
||||
spec {
|
||||
containers {
|
||||
image = "gcr.io/${var.project_id}/brainy"
|
||||
|
||||
resources {
|
||||
limits = {
|
||||
cpu = "2"
|
||||
memory = "2Gi"
|
||||
}
|
||||
}
|
||||
|
||||
env {
|
||||
name = "NODE_ENV"
|
||||
value = "production"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
template {
|
||||
spec {
|
||||
containers {
|
||||
image = "gcr.io/${var.project_id}/brainy"
|
||||
|
||||
traffic {
|
||||
percent = 100
|
||||
latest_revision = true
|
||||
}
|
||||
resources {
|
||||
limits = {
|
||||
cpu = "2"
|
||||
memory = "2Gi"
|
||||
}
|
||||
}
|
||||
|
||||
env {
|
||||
name = "NODE_ENV"
|
||||
value = "production"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
traffic {
|
||||
percent = 100
|
||||
latest_revision = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_cloud_run_service_iam_member" "public" {
|
||||
service = google_cloud_run_service.brainy.name
|
||||
location = google_cloud_run_service.brainy.location
|
||||
role = "roles/run.invoker"
|
||||
member = "allUsers"
|
||||
service = google_cloud_run_service.brainy.name
|
||||
location = google_cloud_run_service.brainy.location
|
||||
role = "roles/run.invoker"
|
||||
member = "allUsers"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -467,13 +467,13 @@ resource "google_cloud_run_service_iam_member" "public" {
|
|||
```javascript
|
||||
// Brainy can use Memorystore for caching
|
||||
const brain = new Brainy({
|
||||
cache: {
|
||||
type: 'redis',
|
||||
options: {
|
||||
host: process.env.REDIS_HOST || 'auto-detect',
|
||||
port: 6379
|
||||
}
|
||||
}
|
||||
cache: {
|
||||
type: 'redis',
|
||||
options: {
|
||||
host: process.env.REDIS_HOST || 'auto-detect',
|
||||
port: 6379
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -481,13 +481,13 @@ const brain = new Brainy({
|
|||
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
metadata: {
|
||||
type: 'spanner',
|
||||
options: {
|
||||
instance: 'brainy-instance',
|
||||
database: 'brainy-db'
|
||||
}
|
||||
}
|
||||
metadata: {
|
||||
type: 'spanner',
|
||||
options: {
|
||||
instance: 'brainy-instance',
|
||||
database: 'brainy-db'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -496,116 +496,116 @@ const brain = new Brainy({
|
|||
### Common Issues
|
||||
|
||||
1. **Quota Exceeded**
|
||||
```bash
|
||||
# Check quotas
|
||||
gcloud compute project-info describe --project=$PROJECT_ID
|
||||
|
||||
# Request increase
|
||||
gcloud compute project-info add-metadata \
|
||||
--metadata google-compute-default-region=us-central1
|
||||
```
|
||||
```bash
|
||||
# Check quotas
|
||||
gcloud compute project-info describe --project=$PROJECT_ID
|
||||
|
||||
# Request increase
|
||||
gcloud compute project-info add-metadata \
|
||||
--metadata google-compute-default-region=us-central1
|
||||
```
|
||||
|
||||
2. **Cold Starts**
|
||||
|
||||
**v7.3.0+ Progressive Initialization (Zero-Config)**
|
||||
**Progressive Initialization (Zero-Config)**
|
||||
|
||||
Brainy automatically detects Cloud Run and Cloud Functions environments
|
||||
and uses progressive initialization for <200ms cold starts:
|
||||
Brainy automatically detects Cloud Run and Cloud Functions environments
|
||||
and uses progressive initialization for <200ms cold starts:
|
||||
|
||||
```javascript
|
||||
// Zero-config - Brainy auto-detects Cloud Run (K_SERVICE env var)
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'gcs',
|
||||
gcsNativeStorage: { bucketName: 'my-bucket' }
|
||||
}
|
||||
})
|
||||
await brain.init() // Returns in <200ms
|
||||
```javascript
|
||||
// Zero-config - Brainy auto-detects Cloud Run (K_SERVICE env var)
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'gcs',
|
||||
gcsNativeStorage: { bucketName: 'my-bucket' }
|
||||
}
|
||||
})
|
||||
await brain.init() // Returns in <200ms
|
||||
|
||||
// First write validates bucket (lazy validation)
|
||||
await brain.add('noun', { name: 'test' }) // Validates here
|
||||
```
|
||||
// First write validates bucket (lazy validation)
|
||||
await brain.add('noun', { name: 'test' }) // Validates here
|
||||
```
|
||||
|
||||
**Manual Override (if needed)**
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'gcs',
|
||||
gcsNativeStorage: {
|
||||
bucketName: 'my-bucket',
|
||||
// Force specific mode
|
||||
initMode: 'progressive' // 'auto' | 'progressive' | 'strict'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
**Manual Override (if needed)**
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'gcs',
|
||||
gcsNativeStorage: {
|
||||
bucketName: 'my-bucket',
|
||||
// Force specific mode
|
||||
initMode: 'progressive' // 'auto' | 'progressive' | 'strict'
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
| Mode | Cold Start | Best For |
|
||||
|------|------------|----------|
|
||||
| `auto` (default) | <200ms in cloud | Zero-config, auto-detects |
|
||||
| `progressive` | <200ms always | Force fast init everywhere |
|
||||
| `strict` | 100-500ms+ | Local dev, tests, debugging |
|
||||
| Mode | Cold Start | Best For |
|
||||
|------|------------|----------|
|
||||
| `auto` (default) | <200ms in cloud | Zero-config, auto-detects |
|
||||
| `progressive` | <200ms always | Force fast init everywhere |
|
||||
| `strict` | 100-500ms+ | Local dev, tests, debugging |
|
||||
|
||||
**Keep Warm (Alternative)**
|
||||
```javascript
|
||||
// Keep minimum instances warm
|
||||
const brain = new Brainy({
|
||||
warmup: {
|
||||
enabled: true,
|
||||
interval: 60000 // Ping every minute
|
||||
}
|
||||
})
|
||||
```
|
||||
**Keep Warm (Alternative)**
|
||||
```javascript
|
||||
// Keep minimum instances warm
|
||||
const brain = new Brainy({
|
||||
warmup: {
|
||||
enabled: true,
|
||||
interval: 60000 // Ping every minute
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
**Readiness Detection (v7.3.0+)**
|
||||
**Readiness Detection**
|
||||
|
||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
||||
Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests:
|
||||
|
||||
```javascript
|
||||
let brain
|
||||
```javascript
|
||||
let brain
|
||||
|
||||
async function handleRequest(req, res) {
|
||||
if (!brain) {
|
||||
brain = new Brainy({ storage: { type: 'gcs', ... } })
|
||||
brain.init() // Fire and forget
|
||||
}
|
||||
async function handleRequest(req, res) {
|
||||
if (!brain) {
|
||||
brain = new Brainy({ storage: { type: 'gcs', ... } })
|
||||
brain.init() // Fire and forget
|
||||
}
|
||||
|
||||
// Wait for initialization to complete
|
||||
await brain.ready
|
||||
// Wait for initialization to complete
|
||||
await brain.ready
|
||||
|
||||
// Now safe to use brain methods
|
||||
const results = await brain.find({ query: req.query.q })
|
||||
res.json(results)
|
||||
}
|
||||
```
|
||||
// Now safe to use brain methods
|
||||
const results = await brain.find({ query: req.query.q })
|
||||
res.json(results)
|
||||
}
|
||||
```
|
||||
|
||||
For Cloud Run health checks, use `isFullyInitialized()` to verify all background tasks are complete:
|
||||
For Cloud Run health checks, use `isFullyInitialized()` to verify all background tasks are complete:
|
||||
|
||||
```javascript
|
||||
// Health check endpoint for Cloud Run
|
||||
app.get('/health', async (req, res) => {
|
||||
try {
|
||||
await brain.ready
|
||||
res.json({
|
||||
status: 'ready',
|
||||
fullyInitialized: brain.isFullyInitialized()
|
||||
})
|
||||
} catch (error) {
|
||||
res.status(503).json({ status: 'initializing' })
|
||||
}
|
||||
})
|
||||
```
|
||||
```javascript
|
||||
// Health check endpoint for Cloud Run
|
||||
app.get('/health', async (req, res) => {
|
||||
try {
|
||||
await brain.ready
|
||||
res.json({
|
||||
status: 'ready',
|
||||
fullyInitialized: brain.isFullyInitialized()
|
||||
})
|
||||
} catch (error) {
|
||||
res.status(503).json({ status: 'initializing' })
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
3. **Memory Pressure**
|
||||
```javascript
|
||||
// Optimize for GCP memory constraints
|
||||
const brain = new Brainy({
|
||||
memory: {
|
||||
mode: 'aggressive', // Aggressive garbage collection
|
||||
maxHeap: 0.8 // Use 80% of available memory
|
||||
}
|
||||
})
|
||||
```
|
||||
```javascript
|
||||
// Optimize for GCP memory constraints
|
||||
const brain = new Brainy({
|
||||
memory: {
|
||||
mode: 'aggressive', // Aggressive garbage collection
|
||||
maxHeap: 0.8 // Use 80% of available memory
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## Production Checklist
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue