feat(8.0): zero-config finalize + cut JS quantization (config.vector = recall + persistMode)
This commit is contained in:
parent
f4c5d9749f
commit
f8e0079d3f
12 changed files with 348 additions and 1739 deletions
|
|
@ -1206,20 +1206,14 @@ export interface BrainyConfig {
|
|||
}
|
||||
| StorageAdapter
|
||||
|
||||
// Index configuration
|
||||
index?: {
|
||||
m?: number // HNSW M parameter
|
||||
efConstruction?: number // HNSW construction parameter
|
||||
efSearch?: number // HNSW search parameter
|
||||
}
|
||||
|
||||
// Performance options
|
||||
cache?: boolean | { // Enable caching
|
||||
maxSize?: number
|
||||
ttl?: number
|
||||
}
|
||||
|
||||
// Distributed configuration
|
||||
/**
|
||||
* Distributed-cluster configuration. **Explicit opt-in only** — Brainy
|
||||
* never auto-enables cluster mode from environment heuristics (a single
|
||||
* production process is the 90th-percentile deployment). Set
|
||||
* `enabled: true` (or the `BRAINY_DISTRIBUTED=true` env var) to activate;
|
||||
* the remaining fields then default sensibly (64 shards, 3 replicas,
|
||||
* raft consensus, http transport, hostname-derived nodeId).
|
||||
*/
|
||||
distributed?: {
|
||||
enabled: boolean
|
||||
nodeId?: string
|
||||
|
|
@ -1231,23 +1225,18 @@ export interface BrainyConfig {
|
|||
transport?: 'tcp' | 'http' | 'udp'
|
||||
}
|
||||
|
||||
// Advanced options
|
||||
warmup?: boolean // Warm up on init
|
||||
realtime?: boolean // Enable real-time updates
|
||||
multiTenancy?: boolean // Enable service isolation
|
||||
telemetry?: boolean // Send anonymous usage stats
|
||||
|
||||
// Performance tuning options for production
|
||||
disableAutoRebuild?: boolean // Disable automatic index rebuilding on init
|
||||
disableMetrics?: boolean // Completely disable metrics collection
|
||||
disableAutoOptimize?: boolean // Disable automatic index optimization
|
||||
batchWrites?: boolean // Enable write batching for better performance
|
||||
maxConcurrentOperations?: number // Limit concurrent file operations
|
||||
/**
|
||||
* Disable the automatic index rebuild check during `init()`. By default
|
||||
* Brainy auto-decides from dataset size: small datasets rebuild missing
|
||||
* indexes inline, large datasets rebuild lazily on first query. Set `true`
|
||||
* only when an operator wants full manual control via `repairIndex()`.
|
||||
*/
|
||||
disableAutoRebuild?: boolean
|
||||
|
||||
/**
|
||||
* Vector index configuration (Brainy 8.0).
|
||||
*
|
||||
* Three knobs. No escape hatch. The algorithm-internal HNSW knobs
|
||||
* Two knobs. No escape hatch. The algorithm-internal HNSW knobs
|
||||
* (`M`, `efConstruction`, `efSearch`, `ml`, …) and DiskANN knobs
|
||||
* (`pqM`, `searchListSize`, `alpha`, …) are deliberately not exposed —
|
||||
* the `recall` preset covers the legitimate quality/latency tradeoff
|
||||
|
|
@ -1271,26 +1260,15 @@ export interface BrainyConfig {
|
|||
*/
|
||||
recall?: 'fast' | 'balanced' | 'accurate'
|
||||
|
||||
/**
|
||||
* Vector quantization. Independent of `recall` — it trades RAM for a
|
||||
* small recall hit at any preset. `bits: 8` is SQ8 (4× memory reduction,
|
||||
* ~0.4% recall loss). `bits: 4` is SQ4 (8× memory reduction, ~1-3% recall
|
||||
* loss). Both ship in the open-core path; cortex's `distance:sq8` /
|
||||
* `distance:sq4` SIMD providers accelerate them when available.
|
||||
*
|
||||
* Silently ignored on a native DiskANN-style provider path (it uses its
|
||||
* own PQ math instead).
|
||||
*/
|
||||
quantization?: {
|
||||
enabled?: boolean
|
||||
bits?: 8 | 4
|
||||
}
|
||||
|
||||
/**
|
||||
* Vector persistence mode. `'immediate'` writes per-noun graph state on
|
||||
* every `add()`; durable but slower. `'deferred'` writes only on
|
||||
* `flush()` / `close()`; faster bulk ingest. Defaults to `'immediate'`
|
||||
* on filesystem storage (the only persistent backend in 8.0).
|
||||
* `flush()` / `close()`; faster bulk ingest.
|
||||
*
|
||||
* **Auto-selected from the storage adapter when omitted:** `'immediate'`
|
||||
* on filesystem storage (durability is the point of a persistent
|
||||
* backend), `'deferred'` on memory storage (nothing survives the process
|
||||
* anyway, so per-add persistence writes are pure overhead).
|
||||
*
|
||||
* Brainy 7.x called this `hnswPersistMode` at the top level. The 8.0
|
||||
* surface folds it under `config.vector` alongside `recall`.
|
||||
|
|
@ -1298,16 +1276,60 @@ export interface BrainyConfig {
|
|||
persistMode?: 'immediate' | 'deferred'
|
||||
}
|
||||
|
||||
/**
|
||||
* Generational-history retention policy (8.0 MVCC).
|
||||
*
|
||||
* Every `transact()` produces an immutable generation record-set that
|
||||
* serves historical reads (`asOf()`, pinned `Db` values). Without
|
||||
* compaction those record-sets accumulate forever, so Brainy
|
||||
* **auto-compacts on every `flush()` and `close()`** with this policy.
|
||||
* A generation is reclaimed only when it is older than BOTH bounds
|
||||
* (and never while a live `Db` pin protects it):
|
||||
*
|
||||
* - `retainGenerations` — keep at least the N most recent committed
|
||||
* generations (default: `100`).
|
||||
* - `retainMs` — keep everything committed within the window
|
||||
* (default: 7 days).
|
||||
*
|
||||
* Escape hatches: raise either bound for longer time-travel windows,
|
||||
* or set `autoCompact: false` to manage history manually via
|
||||
* `brain.compactHistory()`. Long-term archives belong in
|
||||
* `db.persist(path)` snapshots, which compaction never touches.
|
||||
*/
|
||||
history?: {
|
||||
/** Keep at least this many recent committed generations (default: 100). */
|
||||
retainGenerations?: number
|
||||
/** Keep generations committed within this window in ms (default: 604800000 = 7 days). */
|
||||
retainMs?: number
|
||||
/** Run compaction automatically on flush()/close() (default: true). */
|
||||
autoCompact?: boolean
|
||||
}
|
||||
|
||||
// Memory management options
|
||||
maxQueryLimit?: number // Override auto-detected query result limit (max: 100000)
|
||||
reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB)
|
||||
|
||||
// Embedding initialization
|
||||
// Controls when the WASM embedding engine is initialized
|
||||
// - false (default): Lazy initialization on first embed() call
|
||||
// - true: Eager initialization during brain.init()
|
||||
// Set to true for cloud deployments (Cloud Run, Lambda) where you want
|
||||
// WASM compilation to happen during container startup, not on first request
|
||||
/**
|
||||
* Controls when the WASM embedding engine is initialized.
|
||||
*
|
||||
* **Adaptive default (8.0):** when omitted, the engine eagerly initializes
|
||||
* during `init()` whenever the WASM embedder is the *active* one — i.e. no
|
||||
* native `'embeddings'` provider is registered — and this instance is a
|
||||
* writer (not `mode: 'reader'`) running outside unit tests. The WASM module
|
||||
* (≈93MB with the embedded model) takes 90-140s to compile on throttled
|
||||
* CPUs, so paying that during boot rather than on the first `embed()`-driven
|
||||
* call is the right default for a single-process server.
|
||||
*
|
||||
* The adaptive path skips itself automatically when a native embeddings
|
||||
* provider owns embeddings, in reader-mode (readers query existing vectors
|
||||
* and never embed), and in unit-test mode (kept fast via the mock embedder).
|
||||
*
|
||||
* - `true` — force eager init during `init()` (the adaptive default already
|
||||
* does this for the active-embedder writer case; set it explicitly to be
|
||||
* unambiguous).
|
||||
* - `false` — explicit override to force lazy init (first `embed()` call)
|
||||
* even when this instance is the active embedder.
|
||||
*/
|
||||
eagerEmbeddings?: boolean
|
||||
|
||||
// Plugin configuration
|
||||
|
|
@ -1330,8 +1352,10 @@ export interface BrainyConfig {
|
|||
integrations?: boolean | IntegrationsConfig
|
||||
|
||||
// Migration configuration
|
||||
// - false/undefined (default): Log warning if pending migrations exist, but don't auto-run
|
||||
// - true: Automatically run pending migrations during init() for small datasets (<10K entities)
|
||||
// - true/undefined (default): Automatically run pending data migrations during
|
||||
// init() for small datasets (<10K entities). Larger datasets defer with a
|
||||
// notice — call brain.migrate() explicitly (optionally with backupTo).
|
||||
// - false: Never auto-run; log a notice when pending migrations exist.
|
||||
autoMigrate?: boolean
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue