fix: exclude __words__ keyword index from corruption detection and getStats()
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
This commit is contained in:
parent
32dbdcec61
commit
364360d447
128 changed files with 5637 additions and 5682 deletions
|
|
@ -1,8 +1,8 @@
|
|||
# Brainy v4.0.0 Cloud Deployment Guide
|
||||
# Brainy Cloud Deployment Guide
|
||||
|
||||
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy v4.0.0 codebase.
|
||||
This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy codebase.
|
||||
|
||||
## 🆕 v4.0.0 Production Features
|
||||
## 🆕 Production Features
|
||||
|
||||
**Cost Optimization at Scale:**
|
||||
- **Lifecycle Policies**: Automatic tier transitions for 96% cost savings
|
||||
|
|
@ -12,7 +12,7 @@ This guide provides production-ready deployment configurations for Brainy using
|
|||
|
||||
**Example Impact (500TB dataset):**
|
||||
- Before: $138,000/year
|
||||
- With v4.0.0 lifecycle policies: $5,940/year
|
||||
- With lifecycle policies: $5,940/year
|
||||
- **Savings: $132,060/year (96%)**
|
||||
|
||||
See the [Cost Optimization](#cost-optimization-v40) section below for implementation details.
|
||||
|
|
@ -43,53 +43,53 @@ let brain
|
|||
let handler
|
||||
|
||||
exports.handler = async (event) => {
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: process.env.AWS_REGION,
|
||||
bucket: process.env.BRAINY_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
prefix: 'brainy-data/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
auth: {
|
||||
required: true,
|
||||
apiKeys: [process.env.API_KEY]
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
// Get the universal handler from the augmentation
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Lambda event to Request
|
||||
const url = `https://${event.requestContext.domainName}${event.rawPath}`
|
||||
const request = new Request(url, {
|
||||
method: event.requestContext.http.method,
|
||||
headers: event.headers,
|
||||
body: event.body
|
||||
})
|
||||
|
||||
// Use the universal handler
|
||||
const response = await handler(request)
|
||||
|
||||
return {
|
||||
statusCode: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: process.env.AWS_REGION,
|
||||
bucket: process.env.BRAINY_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
prefix: 'brainy-data/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
auth: {
|
||||
required: true,
|
||||
apiKeys: [process.env.API_KEY]
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
// Get the universal handler from the augmentation
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Lambda event to Request
|
||||
const url = `https://${event.requestContext.domainName}${event.rawPath}`
|
||||
const request = new Request(url, {
|
||||
method: event.requestContext.http.method,
|
||||
headers: event.headers,
|
||||
body: event.body
|
||||
})
|
||||
|
||||
// Use the universal handler
|
||||
const response = await handler(request)
|
||||
|
||||
return {
|
||||
statusCode: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -104,36 +104,36 @@ let brain
|
|||
let handler
|
||||
|
||||
exports.brainyAPI = async (req, res) => {
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET,
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Express req/res to Request/Response
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
if (!brain) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET,
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Convert Express req/res to Request/Response
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -156,39 +156,39 @@ let brain
|
|||
let handler
|
||||
|
||||
async function initBrainy() {
|
||||
// Google Cloud Storage is S3-compatible
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET || 'brainy-data',
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: process.env.GCS_REGION || 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT,
|
||||
cors: {
|
||||
origin: process.env.CORS_ORIGIN || '*'
|
||||
},
|
||||
auth: {
|
||||
required: process.env.AUTH_REQUIRED === 'true',
|
||||
apiKeys: process.env.API_KEY ? [process.env.API_KEY] : []
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
// Google Cloud Storage is S3-compatible
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 'storage.googleapis.com',
|
||||
bucket: process.env.GCS_BUCKET || 'brainy-data',
|
||||
accessKeyId: process.env.GCS_ACCESS_KEY,
|
||||
secretAccessKey: process.env.GCS_SECRET_KEY,
|
||||
prefix: 'brainy/',
|
||||
forcePathStyle: false,
|
||||
region: process.env.GCS_REGION || 'US'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT,
|
||||
cors: {
|
||||
origin: process.env.CORS_ORIGIN || '*'
|
||||
},
|
||||
auth: {
|
||||
required: process.env.AUTH_REQUIRED === 'true',
|
||||
apiKeys: process.env.API_KEY ? [process.env.API_KEY] : []
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// Initialize on startup
|
||||
|
|
@ -196,26 +196,26 @@ await initBrainy()
|
|||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'healthy', service: 'brainy-api' })
|
||||
res.json({ status: 'healthy', service: 'brainy-api' })
|
||||
})
|
||||
|
||||
// Universal handler for all API routes
|
||||
app.use('*', async (req, res) => {
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
const request = new Request(`https://${req.hostname}${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
})
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -237,43 +237,43 @@ CMD ["node", "server.js"]
|
|||
```yaml
|
||||
# cloudbuild.yaml
|
||||
steps:
|
||||
# Build the container image
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.']
|
||||
|
||||
# Push to Container Registry
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA']
|
||||
|
||||
# Deploy to Cloud Run
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
entrypoint: gcloud
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- 'brainy-api'
|
||||
- '--image'
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
- '--region'
|
||||
- 'us-central1'
|
||||
- '--platform'
|
||||
- 'managed'
|
||||
- '--allow-unauthenticated'
|
||||
- '--set-env-vars'
|
||||
- 'GCS_BUCKET=brainy-storage,GCS_REGION=US'
|
||||
- '--set-secrets'
|
||||
- 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest'
|
||||
- '--memory'
|
||||
- '2Gi'
|
||||
- '--cpu'
|
||||
- '2'
|
||||
- '--max-instances'
|
||||
- '100'
|
||||
- '--min-instances'
|
||||
- '0'
|
||||
# Build the container image
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.']
|
||||
|
||||
# Push to Container Registry
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA']
|
||||
|
||||
# Deploy to Cloud Run
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
entrypoint: gcloud
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- 'brainy-api'
|
||||
- '--image'
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
- '--region'
|
||||
- 'us-central1'
|
||||
- '--platform'
|
||||
- 'managed'
|
||||
- '--allow-unauthenticated'
|
||||
- '--set-env-vars'
|
||||
- 'GCS_BUCKET=brainy-storage,GCS_REGION=US'
|
||||
- '--set-secrets'
|
||||
- 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest'
|
||||
- '--memory'
|
||||
- '2Gi'
|
||||
- '--cpu'
|
||||
- '2'
|
||||
- '--max-instances'
|
||||
- '100'
|
||||
- '--min-instances'
|
||||
- '0'
|
||||
|
||||
images:
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
- 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'
|
||||
```
|
||||
|
||||
**Deploy with gcloud CLI:**
|
||||
|
|
@ -283,12 +283,12 @@ gcloud builds submit --config cloudbuild.yaml
|
|||
|
||||
# Or deploy directly
|
||||
gcloud run deploy brainy-api \
|
||||
--source . \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated \
|
||||
--set-env-vars GCS_BUCKET=brainy-storage \
|
||||
--set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest"
|
||||
--source . \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--allow-unauthenticated \
|
||||
--set-env-vars GCS_BUCKET=brainy-storage \
|
||||
--set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest"
|
||||
```
|
||||
|
||||
**Create GCS Bucket with S3-compatible access:**
|
||||
|
|
@ -312,37 +312,37 @@ echo -n "YOUR_SECRET_KEY" | gcloud secrets create gcs-secret-key --data-file=-
|
|||
```javascript
|
||||
// index.js
|
||||
module.exports = async function (context, req) {
|
||||
const { Brainy } = require('@soulcraft/brainy')
|
||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
||||
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
secretAccessKey: process.env.AZURE_STORAGE_KEY,
|
||||
prefix: 'entities/',
|
||||
forcePathStyle: false
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
const handler = apiAugmentation.createUniversalHandler()
|
||||
|
||||
const request = new Request(`https://${context.req.headers.host}${context.req.url}`, {
|
||||
method: context.req.method,
|
||||
headers: context.req.headers,
|
||||
body: JSON.stringify(context.req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
context.res = {
|
||||
status: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
const { Brainy } = require('@soulcraft/brainy')
|
||||
const { S3CompatibleStorage } = require('@soulcraft/brainy/storage')
|
||||
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: process.env.AZURE_STORAGE_ACCOUNT,
|
||||
secretAccessKey: process.env.AZURE_STORAGE_KEY,
|
||||
prefix: 'entities/',
|
||||
forcePathStyle: false
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
const handler = apiAugmentation.createUniversalHandler()
|
||||
|
||||
const request = new Request(`https://${context.req.headers.host}${context.req.url}`, {
|
||||
method: context.req.method,
|
||||
headers: context.req.headers,
|
||||
body: JSON.stringify(context.req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
context.res = {
|
||||
status: response.status,
|
||||
headers: Object.fromEntries(response.headers),
|
||||
body: await response.text()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -356,37 +356,37 @@ import { R2Storage } from '@soulcraft/brainy/storage' // Alias for S3CompatibleS
|
|||
let handler
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx) {
|
||||
if (!handler) {
|
||||
const storage = new R2Storage({
|
||||
endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_SECRET_ACCESS_KEY,
|
||||
region: 'auto',
|
||||
forcePathStyle: true
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
cors: { origin: '*' }
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// The handler works directly with Request/Response!
|
||||
return handler(request)
|
||||
}
|
||||
async fetch(request, env, ctx) {
|
||||
if (!handler) {
|
||||
const storage = new R2Storage({
|
||||
endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`,
|
||||
bucket: 'brainy-data',
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_SECRET_ACCESS_KEY,
|
||||
region: 'auto',
|
||||
forcePathStyle: true
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
cors: { origin: '*' }
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
// The handler works directly with Request/Response!
|
||||
return handler(request)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -418,52 +418,52 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage'
|
|||
let handler
|
||||
|
||||
export const config = {
|
||||
runtime: 'edge',
|
||||
runtime: 'edge',
|
||||
}
|
||||
|
||||
export default async (request) => {
|
||||
if (!handler) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
return handler(request)
|
||||
if (!handler) {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: 's3.amazonaws.com',
|
||||
region: 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
return handler(request)
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
// vercel.json
|
||||
{
|
||||
"functions": {
|
||||
"api/brainy.js": {
|
||||
"maxDuration": 30,
|
||||
"memory": 1024
|
||||
}
|
||||
},
|
||||
"rewrites": [
|
||||
{
|
||||
"source": "/api/:path*",
|
||||
"destination": "/api/brainy"
|
||||
}
|
||||
]
|
||||
"functions": {
|
||||
"api/brainy.js": {
|
||||
"maxDuration": 30,
|
||||
"memory": 1024
|
||||
}
|
||||
},
|
||||
"rewrites": [
|
||||
{
|
||||
"source": "/api/:path*",
|
||||
"destination": "/api/brainy"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -482,51 +482,51 @@ let brain
|
|||
let handler
|
||||
|
||||
async function init() {
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.S3_ACCESS_KEY,
|
||||
secretAccessKey: process.env.S3_SECRET_KEY,
|
||||
prefix: 'brainy/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com',
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.S3_ACCESS_KEY,
|
||||
secretAccessKey: process.env.S3_SECRET_KEY,
|
||||
prefix: 'brainy/'
|
||||
})
|
||||
|
||||
brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
port: PORT
|
||||
}
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
||||
const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server')
|
||||
handler = apiAugmentation.createUniversalHandler()
|
||||
}
|
||||
|
||||
await init()
|
||||
|
||||
// Universal handler for all routes
|
||||
app.use('*', async (req, res) => {
|
||||
const request = new Request(`http://localhost${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
const request = new Request(`http://localhost${req.originalUrl}`, {
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
body: JSON.stringify(req.body)
|
||||
})
|
||||
|
||||
const response = await handler(request)
|
||||
|
||||
res.status(response.status)
|
||||
res.set(Object.fromEntries(response.headers))
|
||||
res.send(await response.text())
|
||||
})
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
console.log(`Brainy API running on port ${PORT}`)
|
||||
})
|
||||
```
|
||||
|
||||
|
|
@ -552,26 +552,26 @@ restartPolicyMaxRetries = 3
|
|||
```yaml
|
||||
# render.yaml
|
||||
services:
|
||||
- type: web
|
||||
name: brainy-api
|
||||
runtime: node
|
||||
buildCommand: npm install
|
||||
startCommand: node server.js
|
||||
envVars:
|
||||
- key: S3_BUCKET
|
||||
value: brainy-data
|
||||
- key: S3_ENDPOINT
|
||||
value: s3.amazonaws.com
|
||||
- key: S3_REGION
|
||||
value: us-east-1
|
||||
- key: S3_ACCESS_KEY
|
||||
sync: false
|
||||
- key: S3_SECRET_KEY
|
||||
sync: false
|
||||
- key: API_KEY
|
||||
generateValue: true
|
||||
healthCheckPath: /health
|
||||
autoDeploy: true
|
||||
- type: web
|
||||
name: brainy-api
|
||||
runtime: node
|
||||
buildCommand: npm install
|
||||
startCommand: node server.js
|
||||
envVars:
|
||||
- key: S3_BUCKET
|
||||
value: brainy-data
|
||||
- key: S3_ENDPOINT
|
||||
value: s3.amazonaws.com
|
||||
- key: S3_REGION
|
||||
value: us-east-1
|
||||
- key: S3_ACCESS_KEY
|
||||
sync: false
|
||||
- key: S3_SECRET_KEY
|
||||
sync: false
|
||||
- key: API_KEY
|
||||
generateValue: true
|
||||
healthCheckPath: /health
|
||||
autoDeploy: true
|
||||
```
|
||||
|
||||
### Deno Deploy
|
||||
|
|
@ -582,19 +582,19 @@ import { Brainy } from "npm:@soulcraft/brainy"
|
|||
import { S3CompatibleStorage } from "npm:@soulcraft/brainy/storage"
|
||||
|
||||
const storage = new S3CompatibleStorage({
|
||||
endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com",
|
||||
bucket: Deno.env.get("S3_BUCKET")!,
|
||||
accessKeyId: Deno.env.get("S3_ACCESS_KEY")!,
|
||||
secretAccessKey: Deno.env.get("S3_SECRET_KEY")!,
|
||||
region: "auto"
|
||||
endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com",
|
||||
bucket: Deno.env.get("S3_BUCKET")!,
|
||||
accessKeyId: Deno.env.get("S3_ACCESS_KEY")!,
|
||||
secretAccessKey: Deno.env.get("S3_SECRET_KEY")!,
|
||||
region: "auto"
|
||||
})
|
||||
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
storage,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: { enabled: true }
|
||||
}]
|
||||
})
|
||||
|
||||
await brain.init()
|
||||
|
|
@ -653,29 +653,29 @@ RATE_LIMIT_MAX=100
|
|||
```javascript
|
||||
// REST API Client
|
||||
const response = await fetch('https://your-api.com/api/brainy/add', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: 'Your content here',
|
||||
metadata: { type: 'document' }
|
||||
})
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: 'Your content here',
|
||||
metadata: { type: 'document' }
|
||||
})
|
||||
})
|
||||
|
||||
const { id } = await response.json()
|
||||
|
||||
// Search
|
||||
const searchResponse = await fetch('https://your-api.com/api/brainy/find', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: 'neural networks'
|
||||
})
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer YOUR_API_KEY'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: 'neural networks'
|
||||
})
|
||||
})
|
||||
|
||||
const results = await searchResponse.json()
|
||||
|
|
@ -683,14 +683,14 @@ const results = await searchResponse.json()
|
|||
// WebSocket Client
|
||||
const ws = new WebSocket('wss://your-api.com/ws')
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({
|
||||
type: 'subscribe',
|
||||
pattern: 'technology'
|
||||
}))
|
||||
ws.send(JSON.stringify({
|
||||
type: 'subscribe',
|
||||
pattern: 'technology'
|
||||
}))
|
||||
}
|
||||
ws.onmessage = (event) => {
|
||||
const update = JSON.parse(event.data)
|
||||
console.log('Real-time update:', update)
|
||||
const update = JSON.parse(event.data)
|
||||
console.log('Real-time update:', update)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -700,13 +700,13 @@ S3CompatibleStorage constructor parameters (verified from source):
|
|||
|
||||
```javascript
|
||||
{
|
||||
endpoint: string, // Required (e.g., 's3.amazonaws.com')
|
||||
bucket: string, // Required
|
||||
accessKeyId: string, // Required
|
||||
secretAccessKey: string, // Required
|
||||
region?: string, // Optional (default: 'us-east-1')
|
||||
prefix?: string, // Optional (e.g., 'brainy/')
|
||||
forcePathStyle?: boolean, // Optional (needed for some S3-compatible services)
|
||||
endpoint: string, // Required (e.g., 's3.amazonaws.com')
|
||||
bucket: string, // Required
|
||||
accessKeyId: string, // Required
|
||||
secretAccessKey: string, // Required
|
||||
region?: string, // Optional (default: 'us-east-1')
|
||||
prefix?: string, // Optional (e.g., 'brainy/')
|
||||
forcePathStyle?: boolean, // Optional (needed for some S3-compatible services)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -727,7 +727,7 @@ S3CompatibleStorage constructor parameters (verified from source):
|
|||
4. **Implement rate limiting** to prevent abuse
|
||||
5. **Configure CORS** appropriately for your use case
|
||||
|
||||
## Cost Optimization (v4.0.0)
|
||||
## Cost Optimization
|
||||
|
||||
### Enable Lifecycle Policies
|
||||
|
||||
|
|
@ -738,16 +738,16 @@ const storage = brain.storage
|
|||
|
||||
// Set lifecycle policy for automatic archival
|
||||
await storage.setLifecyclePolicy({
|
||||
rules: [{
|
||||
id: 'archive-old-data',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days
|
||||
{ days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days
|
||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year
|
||||
]
|
||||
}]
|
||||
rules: [{
|
||||
id: 'archive-old-data',
|
||||
prefix: 'entities/',
|
||||
status: 'Enabled',
|
||||
transitions: [
|
||||
{ days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days
|
||||
{ days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days
|
||||
{ days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year
|
||||
]
|
||||
}]
|
||||
})
|
||||
|
||||
// Or enable Intelligent-Tiering for hands-off optimization
|
||||
|
|
@ -765,14 +765,14 @@ await storage.enableIntelligentTiering('entities/', 'auto-optimize')
|
|||
|
||||
**Efficient bulk deletions:**
|
||||
```javascript
|
||||
// v4.0.0: Batch delete (1000 objects per request)
|
||||
// Batch delete (1000 objects per request)
|
||||
const idsToDelete = [/* array of entity IDs */]
|
||||
const paths = idsToDelete.flatMap(id => [
|
||||
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
||||
`entities/nouns/metadata/${id.substring(0, 2)}/${id}.json`
|
||||
`entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`,
|
||||
`entities/nouns/metadata/${id.substring(0, 2)}/${id}.json`
|
||||
])
|
||||
|
||||
await storage.batchDelete(paths) // Much faster than individual deletes
|
||||
await storage.batchDelete(paths) // Much faster than individual deletes
|
||||
```
|
||||
|
||||
### Enable Compression (FileSystem)
|
||||
|
|
@ -780,8 +780,8 @@ await storage.batchDelete(paths) // Much faster than individual deletes
|
|||
**For local/server deployments:**
|
||||
```javascript
|
||||
const storage = new FileSystemStorage({
|
||||
path: './data',
|
||||
compression: true // 60-80% space savings
|
||||
path: './data',
|
||||
compression: true // 60-80% space savings
|
||||
})
|
||||
|
||||
const brain = new Brainy({ storage })
|
||||
|
|
@ -793,8 +793,8 @@ const brain = new Brainy({ storage })
|
|||
2. **Use S3CompatibleStorage** for cloud deployments (better scalability)
|
||||
3. **Enable the cache augmentation** for frequently accessed data
|
||||
4. **Configure appropriate memory limits** for your runtime
|
||||
5. **v4.0.0**: Enable lifecycle policies to reduce storage costs by 96%
|
||||
6. **v4.0.0**: Use batch operations for cleanup tasks
|
||||
5. Enable lifecycle policies to reduce storage costs by 96%
|
||||
6. Use batch operations for cleanup tasks
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
|
@ -810,15 +810,15 @@ const brain = new Brainy({ storage })
|
|||
Enable debug logging by setting:
|
||||
```javascript
|
||||
const brain = new Brainy({
|
||||
storage,
|
||||
debug: true,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
verbose: true
|
||||
}
|
||||
}]
|
||||
storage,
|
||||
debug: true,
|
||||
augmentations: [{
|
||||
name: 'api-server',
|
||||
config: {
|
||||
enabled: true,
|
||||
verbose: true
|
||||
}
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue