feat(8.0): zero-config finalize + cut JS quantization (config.vector = recall + persistMode)

This commit is contained in:
David Snelling 2026-06-15 10:08:51 -07:00
parent f4c5d9749f
commit f8e0079d3f
12 changed files with 348 additions and 1739 deletions

View file

@ -1206,20 +1206,14 @@ export interface BrainyConfig {
}
| StorageAdapter
// Index configuration
index?: {
m?: number // HNSW M parameter
efConstruction?: number // HNSW construction parameter
efSearch?: number // HNSW search parameter
}
// Performance options
cache?: boolean | { // Enable caching
maxSize?: number
ttl?: number
}
// Distributed configuration
/**
* Distributed-cluster configuration. **Explicit opt-in only** Brainy
* never auto-enables cluster mode from environment heuristics (a single
* production process is the 90th-percentile deployment). Set
* `enabled: true` (or the `BRAINY_DISTRIBUTED=true` env var) to activate;
* the remaining fields then default sensibly (64 shards, 3 replicas,
* raft consensus, http transport, hostname-derived nodeId).
*/
distributed?: {
enabled: boolean
nodeId?: string
@ -1231,23 +1225,18 @@ export interface BrainyConfig {
transport?: 'tcp' | 'http' | 'udp'
}
// Advanced options
warmup?: boolean // Warm up on init
realtime?: boolean // Enable real-time updates
multiTenancy?: boolean // Enable service isolation
telemetry?: boolean // Send anonymous usage stats
// Performance tuning options for production
disableAutoRebuild?: boolean // Disable automatic index rebuilding on init
disableMetrics?: boolean // Completely disable metrics collection
disableAutoOptimize?: boolean // Disable automatic index optimization
batchWrites?: boolean // Enable write batching for better performance
maxConcurrentOperations?: number // Limit concurrent file operations
/**
* Disable the automatic index rebuild check during `init()`. By default
* Brainy auto-decides from dataset size: small datasets rebuild missing
* indexes inline, large datasets rebuild lazily on first query. Set `true`
* only when an operator wants full manual control via `repairIndex()`.
*/
disableAutoRebuild?: boolean
/**
* Vector index configuration (Brainy 8.0).
*
* Three knobs. No escape hatch. The algorithm-internal HNSW knobs
* Two knobs. No escape hatch. The algorithm-internal HNSW knobs
* (`M`, `efConstruction`, `efSearch`, `ml`, ) and DiskANN knobs
* (`pqM`, `searchListSize`, `alpha`, ) are deliberately not exposed
* the `recall` preset covers the legitimate quality/latency tradeoff
@ -1271,26 +1260,15 @@ export interface BrainyConfig {
*/
recall?: 'fast' | 'balanced' | 'accurate'
/**
* Vector quantization. Independent of `recall` it trades RAM for a
* small recall hit at any preset. `bits: 8` is SQ8 (4× memory reduction,
* ~0.4% recall loss). `bits: 4` is SQ4 (8× memory reduction, ~1-3% recall
* loss). Both ship in the open-core path; cortex's `distance:sq8` /
* `distance:sq4` SIMD providers accelerate them when available.
*
* Silently ignored on a native DiskANN-style provider path (it uses its
* own PQ math instead).
*/
quantization?: {
enabled?: boolean
bits?: 8 | 4
}
/**
* Vector persistence mode. `'immediate'` writes per-noun graph state on
* every `add()`; durable but slower. `'deferred'` writes only on
* `flush()` / `close()`; faster bulk ingest. Defaults to `'immediate'`
* on filesystem storage (the only persistent backend in 8.0).
* `flush()` / `close()`; faster bulk ingest.
*
* **Auto-selected from the storage adapter when omitted:** `'immediate'`
* on filesystem storage (durability is the point of a persistent
* backend), `'deferred'` on memory storage (nothing survives the process
* anyway, so per-add persistence writes are pure overhead).
*
* Brainy 7.x called this `hnswPersistMode` at the top level. The 8.0
* surface folds it under `config.vector` alongside `recall`.
@ -1298,16 +1276,60 @@ export interface BrainyConfig {
persistMode?: 'immediate' | 'deferred'
}
/**
* Generational-history retention policy (8.0 MVCC).
*
* Every `transact()` produces an immutable generation record-set that
* serves historical reads (`asOf()`, pinned `Db` values). Without
* compaction those record-sets accumulate forever, so Brainy
* **auto-compacts on every `flush()` and `close()`** with this policy.
* A generation is reclaimed only when it is older than BOTH bounds
* (and never while a live `Db` pin protects it):
*
* - `retainGenerations` keep at least the N most recent committed
* generations (default: `100`).
* - `retainMs` keep everything committed within the window
* (default: 7 days).
*
* Escape hatches: raise either bound for longer time-travel windows,
* or set `autoCompact: false` to manage history manually via
* `brain.compactHistory()`. Long-term archives belong in
* `db.persist(path)` snapshots, which compaction never touches.
*/
history?: {
/** Keep at least this many recent committed generations (default: 100). */
retainGenerations?: number
/** Keep generations committed within this window in ms (default: 604800000 = 7 days). */
retainMs?: number
/** Run compaction automatically on flush()/close() (default: true). */
autoCompact?: boolean
}
// Memory management options
maxQueryLimit?: number // Override auto-detected query result limit (max: 100000)
reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB)
// Embedding initialization
// Controls when the WASM embedding engine is initialized
// - false (default): Lazy initialization on first embed() call
// - true: Eager initialization during brain.init()
// Set to true for cloud deployments (Cloud Run, Lambda) where you want
// WASM compilation to happen during container startup, not on first request
/**
* Controls when the WASM embedding engine is initialized.
*
* **Adaptive default (8.0):** when omitted, the engine eagerly initializes
* during `init()` whenever the WASM embedder is the *active* one i.e. no
* native `'embeddings'` provider is registered and this instance is a
* writer (not `mode: 'reader'`) running outside unit tests. The WASM module
* (93MB with the embedded model) takes 90-140s to compile on throttled
* CPUs, so paying that during boot rather than on the first `embed()`-driven
* call is the right default for a single-process server.
*
* The adaptive path skips itself automatically when a native embeddings
* provider owns embeddings, in reader-mode (readers query existing vectors
* and never embed), and in unit-test mode (kept fast via the mock embedder).
*
* - `true` force eager init during `init()` (the adaptive default already
* does this for the active-embedder writer case; set it explicitly to be
* unambiguous).
* - `false` explicit override to force lazy init (first `embed()` call)
* even when this instance is the active embedder.
*/
eagerEmbeddings?: boolean
// Plugin configuration
@ -1330,8 +1352,10 @@ export interface BrainyConfig {
integrations?: boolean | IntegrationsConfig
// Migration configuration
// - false/undefined (default): Log warning if pending migrations exist, but don't auto-run
// - true: Automatically run pending migrations during init() for small datasets (<10K entities)
// - true/undefined (default): Automatically run pending data migrations during
// init() for small datasets (<10K entities). Larger datasets defer with a
// notice — call brain.migrate() explicitly (optionally with backupTo).
// - false: Never auto-run; log a notice when pending migrations exist.
autoMigrate?: boolean
/**