Merge branch 'release/8.10.1'

# Conflicts:
#	.forgejo/workflows/ci.yml

src/brainy.ts

@@ -65,7 +65,8 @@ import type {
   OpaqueIdSet,
   AtGenerationVectors,
   VectorIndexProvider,
-  GraphIndexProvider
+  GraphIndexProvider,
+  ProviderMaintenanceDebt
 } from './plugin.js'
 import type {
   BrainyPlugin,
@@ -424,6 +425,30 @@ export interface WarmReport {
   totalDurationMs: number
 }
 
+/**
+ * @description Result of {@link Brainy.maintenanceDebt}: one outcome per
+ * index surface, mirroring {@link WarmReport}'s shape.
+ * - `'reported'` — the active provider for this surface implements
+ *   `maintenanceDebt?()` and its {@link ProviderMaintenanceDebt} payload is
+ *   attached verbatim under `debt`.
+ * - `'unavailable'` — the active provider does not implement the hook, so
+ *   nothing is known; brainy never estimates or infers a payload on its
+ *   behalf.
+ */
+export type MaintenanceDebtOutcome = 'reported' | 'unavailable'
+
+/**
+ * @description Per-surface result of {@link Brainy.maintenanceDebt}. Brainy
+ * performs no thresholding, polling, or estimation over this data — it is a
+ * pure passthrough of each active provider's own self-report (the provider
+ * owns the numbers; the operator owns the policy).
+ */
+export interface MaintenanceDebtReport {
+  vector: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt }
+  metadata: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt }
+  graph: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt }
+}
+
 /**
  * How long a failed aggregation-backfill walk suppresses fresh walk attempts.
  * Within the window, queries rethrow the recorded failure instantly (loud,
@@ -14352,9 +14377,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    *   *some* backing storage as a side effect but is reported honestly as
    *   `'probed'`, never `'warmed'`. An empty index or unknown dimension has
    *   nothing to probe (`'unavailable'`).
-   * - **Metadata**: full hydration — every persisted field's sparse index is
-   *   loaded from storage (`MetadataIndexManager.hydrateAll()`), not just the
-   *   heuristic common-fields subset `init()` warms.
+   * - **Metadata**: calls the provider's own `warm?()` when the active
+   *   `'metadataIndex'` provider implements it (`'warmed'`) — the seam a
+   *   native metadata provider lights up so it is not duck-typed against the
+   *   JS manager's method. Otherwise falls back to full hydration on the
+   *   built-in JS manager — every persisted field's sparse index is loaded
+   *   from storage (`MetadataIndexManager.hydrateAll()`), not just the
+   *   heuristic common-fields subset `init()` warms — and reports `'warmed'`.
+   *   Neither seam present → `'unavailable'` (honest: `init()` is never used
+   *   as a substitute here, since a native provider's `init()` may be a
+   *   cheap verify rather than a real warm).
    * - **Graph**: calls the provider's own `warm?()` when the active graph
    *   provider implements it; otherwise re-runs its existing eager cold-load
    *   `init()` seam (idempotent — the JS adjacency index's `init()` already
@@ -14410,12 +14442,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // --- Metadata --------------------------------------------------------
     const metadataStart = Date.now()
     let metadataOutcome: WarmOutcome
+    const metadataProvider = this.metadataIndex as unknown as MetadataIndexProvider
     const metadataWithHydrate = this.metadataIndex as unknown as { hydrateAll?: () => Promise<void> }
-    if (typeof metadataWithHydrate.hydrateAll === 'function') {
+    if (typeof metadataProvider.warm === 'function') {
+      // Active provider (e.g. a native metadata index) declares its own warm
+      // seam — route through it FIRST so a native provider's warmth is
+      // reported honestly instead of being duck-typed against the JS
+      // manager's hydrateAll(), which a native provider does not implement.
+      await metadataProvider.warm()
+      metadataOutcome = 'warmed'
+    } else if (typeof metadataWithHydrate.hydrateAll === 'function') {
+      // Built-in JS manager path — full sparse-index hydration.
       await metadataWithHydrate.hydrateAll()
       metadataOutcome = 'warmed'
     } else {
-      // No hydration seam on this metadata provider — nothing to run.
+      // No hydration seam on this metadata provider — nothing to run. (No
+      // init() fallback here: init() on a native provider may be a cheap
+      // verify, and reporting that as warmth would lie.)
       metadataOutcome = 'unavailable'
     }
     const metadataDurationMs = Date.now() - metadataStart
@@ -14449,6 +14492,63 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
   }
 
+  /**
+   * Read each index surface's self-reported outstanding maintenance work —
+   * the observability seam so an operator sees a grind coming (rising
+   * pending bytes/items, a stalled background pass) instead of discovering
+   * it as a CPU storm or a transaction blowing its budget mid-flight (see
+   * {@link TransactionTimeoutError}).
+   *
+   * PURE PASSTHROUGH: for each of vector/metadata/graph, this calls ONLY the
+   * ACTIVE provider's own `maintenanceDebt?()` hook (the same per-surface
+   * provider resolution {@link Brainy.warm} uses) and reports its
+   * {@link ProviderMaintenanceDebt} payload verbatim. There is no JS-side
+   * fallback computation, no threshold evaluation, and no polling — brainy
+   * surfaces the truth the provider measured; the provider owns the numbers
+   * and the operator owns the policy (what threshold matters, what action to
+   * take). A surface whose active provider does not implement the hook
+   * reports `'unavailable'` — never a guessed or zeroed payload.
+   *
+   * @returns A {@link MaintenanceDebtReport}: per-surface outcome + payload.
+   * @example
+   * ```typescript
+   * const debt = await brain.maintenanceDebt()
+   * if (debt.metadata.outcome === 'reported' && debt.metadata.debt?.pendingBytes) {
+   *   console.log('metadata pending bytes:', debt.metadata.debt.pendingBytes)
+   * }
+   * ```
+   */
+  async maintenanceDebt(): Promise<MaintenanceDebtReport> {
+    await this.ensureInitialized({ needs: ['vector', 'metadata', 'graph'] })
+
+    // --- Vector ---------------------------------------------------------
+    const vectorProvider = this.index as VectorIndexProvider & {
+      maintenanceDebt?: () => Promise<ProviderMaintenanceDebt>
+    }
+    const vector =
+      typeof vectorProvider.maintenanceDebt === 'function'
+        ? { outcome: 'reported' as const, debt: await vectorProvider.maintenanceDebt() }
+        : { outcome: 'unavailable' as const }
+
+    // --- Metadata --------------------------------------------------------
+    const metadataProvider = this.metadataIndex as unknown as MetadataIndexProvider
+    const metadata =
+      typeof metadataProvider.maintenanceDebt === 'function'
+        ? { outcome: 'reported' as const, debt: await metadataProvider.maintenanceDebt() }
+        : { outcome: 'unavailable' as const }
+
+    // --- Graph -------------------------------------------------------------
+    const graphProvider = this.graphIndex as GraphIndexProvider & {
+      maintenanceDebt?: () => Promise<ProviderMaintenanceDebt>
+    }
+    const graph =
+      typeof graphProvider.maintenanceDebt === 'function'
+        ? { outcome: 'reported' as const, debt: await graphProvider.maintenanceDebt() }
+        : { outcome: 'unavailable' as const }
+
+    return { vector, metadata, graph }
+  }
+
   /**
    * Explicitly warm up the embedding engine
    *

src/db/types.ts

@@ -121,9 +121,13 @@ export interface TransactOptions {
    * with the batch: `max(30 000, opCount × 2 000)` — production imports on
    * network-attached disks measure ~2 s per operation, so a flat 30 s budget
    * silently capped honest bulk work at ~15 operations. A tripped budget
-   * rolls the whole batch back and throws a retryable
-   * `TransactionTimeoutError` naming the operation it stopped at, the batch
-   * size, and the elapsed/budget times.
+   * rolls the whole batch back and throws a `TransactionTimeoutError` naming
+   * the operation it stopped at, the batch size, and the elapsed/budget
+   * times. That error is retryable-with-latch, never hot-retry: its
+   * `retryable` field says a later attempt may succeed, its
+   * `hotRetryUnsafe` field says an immediate identical retry re-pays the
+   * full cost that just timed out — callers must latch and back off, never
+   * loop.
    */
   timeoutMs?: number
 }

src/index.ts

@@ -31,6 +31,11 @@ export type { DiagnosticsResult } from './brainy.js'
 // brain.warm() — eager index/storage readiness report (per-surface honest
 // outcome + timing). See the WarmReport JSDoc in brainy.ts.
 export type { WarmReport, WarmOutcome } from './brainy.js'
+// brain.maintenanceDebt() — per-surface passthrough of each active
+// provider's self-reported background maintenance debt. See the
+// MaintenanceDebtReport JSDoc in brainy.ts and ProviderMaintenanceDebt in
+// plugin.ts for the measure-only-what-you-track contract.
+export type { MaintenanceDebtReport, MaintenanceDebtOutcome } from './brainy.js'
 export type {
   GraphAuditReport,
   GraphAuditDiscrepancy
@@ -228,6 +233,11 @@ export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.j
 export { isVersionedIndexProvider } from './plugin.js'
 export type { VersionedIndexProvider } from './plugin.js'
 export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js'
+// Optional provider self-report of outstanding background maintenance work
+// (compaction, deferred writes, etc.) — the payload type for
+// brain.maintenanceDebt(). See the measure-only-what-you-track contract on
+// ProviderMaintenanceDebt in plugin.ts.
+export type { ProviderMaintenanceDebt } from './plugin.js'
 // Optional native graph-acceleration engine (cor 3.0) — the published provider
 // contract + its columnar wire types. Brainy feature-detects an implementation
 // and falls back to its pure-TS adjacency when absent.

src/plugin.ts

@@ -171,6 +171,37 @@ export interface ProviderInvariantReport {
   durationMs: number
 }
 
+/**
+ * @description A provider's self-report of its own outstanding background
+ * maintenance work (compaction, deferred writes, a build-new→verify→swap in
+ * flight, etc.) — the observability seam so an operator sees a grind coming
+ * (rising pending bytes/items, a stalled pass) instead of discovering it as a
+ * CPU storm or a timeout under transaction budget pressure. Every field is
+ * OPTIONAL and every field is a MEASUREMENT: a provider reports ONLY what it
+ * actually tracks, never an estimate dressed up as a fact. Absence of the
+ * {@link VectorIndexProvider.maintenanceDebt} /
+ * {@link GraphIndexProvider.maintenanceDebt} /
+ * {@link MetadataIndexProvider.maintenanceDebt} hook itself means the
+ * provider does not track debt at all — brainy reports that surface
+ * `'unavailable'` rather than inventing zeros. Brainy performs NO threshold
+ * checks, NO polling, and NO JS-side estimation over this payload — it is a
+ * pure passthrough via {@link Brainy.maintenanceDebt}; the provider owns the
+ * numbers and the operator owns the policy (what threshold matters, what to
+ * do about it).
+ */
+export interface ProviderMaintenanceDebt {
+  /** Bytes of outstanding/unmerged work, if the provider measures it (e.g. unflushed writes, unmerged segments). */
+  pendingBytes?: number
+  /** Count of outstanding items (records, segments, nodes) awaiting the provider's background pass. */
+  pendingItems?: number
+  /** Epoch millis when the provider's last maintenance pass finished, if it tracks one. */
+  lastPassCompletedAt?: number
+  /** How the last pass ended, if the provider tracks pass outcomes. */
+  lastPassOutcome?: 'completed' | 'partial' | 'failed'
+  /** `true` if the provider's own measurements show debt trending down (making progress); `false` if flat or growing; omitted if the provider can't tell. */
+  converging?: boolean
+}
+
 /**
  * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`.
  * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and
@@ -181,6 +212,34 @@ export interface MetadataIndexProvider {
   flush(): Promise<void>
   rebuild(): Promise<void>
 
+  /**
+   * @description OPTIONAL. Eagerly load/fault-in backing storage (e.g. mmap
+   * pretouch, full sparse-index hydration) so first queries run at
+   * steady-state cost. Optional; absence means the provider demand-loads.
+   * Mirrors {@link GraphIndexProvider.warm} / the vector provider's `warm?()`
+   * (`src/plugin.ts` VectorIndexProvider). Distinct from `init()`: `init` is
+   * required and runs once automatically during brain startup; `warm` is a
+   * separate, explicit step a caller opts into via `brain.warm()` (or
+   * `warmOnOpen`) to pre-pay demand-load cost `init` left lazy. Idempotent —
+   * calling it more than once must be safe and cheap on a brain that is
+   * already warm. A provider that already loads everything eagerly in
+   * `init()` may implement `warm` as a no-op or omit it — `brain.warm()`
+   * falls back to the built-in JS manager's `hydrateAll()` duck-type when
+   * absent, and to an honest `'unavailable'` when neither exists.
+   */
+  warm?(): Promise<void>
+
+  /**
+   * @description OPTIONAL self-reported {@link ProviderMaintenanceDebt} —
+   * the observability seam so an operator sees outstanding background
+   * maintenance work (e.g. unmerged postings) BEFORE it grinds a transaction
+   * into a budget-busting op. Absence means this provider does not track
+   * debt; `brain.maintenanceDebt()` reports this surface `'unavailable'`
+   * rather than guessing. See {@link ProviderMaintenanceDebt} for the
+   * measure-only-what-you-track contract.
+   */
+  maintenanceDebt?(): Promise<ProviderMaintenanceDebt>
+
   /**
    * @description OPTIONAL honest durability signal (readiness contract,
    * mirrors `isReady?()` on the graph and vector providers). `true` ⇔ the
@@ -395,6 +454,18 @@ export interface GraphIndexProvider {
    */
   warm?(): Promise<void>
 
+  /**
+   * @description OPTIONAL self-reported {@link ProviderMaintenanceDebt} —
+   * the observability seam so an operator sees outstanding background
+   * maintenance work (e.g. a build-new→verify→swap in flight, unmerged
+   * adjacency segments) BEFORE it grinds a transaction into a
+   * budget-busting op. Absence means this provider does not track debt;
+   * `brain.maintenanceDebt()` reports this surface `'unavailable'` rather
+   * than guessing. See {@link ProviderMaintenanceDebt} for the
+   * measure-only-what-you-track contract.
+   */
+  maintenanceDebt?(): Promise<ProviderMaintenanceDebt>
+
   /**
    * @description OPTIONAL. A native provider returns true from the moment its
    * `init()` detects a large epoch-drift until its background
@@ -1057,6 +1128,18 @@ export interface VectorIndexProvider {
    */
   warm?(): Promise<void>
 
+  /**
+   * @description OPTIONAL self-reported {@link ProviderMaintenanceDebt} —
+   * the observability seam so an operator sees outstanding background
+   * maintenance work (e.g. unflushed writes, a pending rebuild) BEFORE it
+   * grinds a transaction into a budget-busting op. Absence means this
+   * provider does not track debt; `brain.maintenanceDebt()` reports this
+   * surface `'unavailable'` rather than guessing. See
+   * {@link ProviderMaintenanceDebt} for the measure-only-what-you-track
+   * contract.
+   */
+  maintenanceDebt?(): Promise<ProviderMaintenanceDebt>
+
   /**
    * @description OPTIONAL honest durability signal (readiness contract,
    * mirrors {@link GraphIndexProvider.isReady}). `true` ⇔ the persisted

src/transaction/Transaction.ts

@@ -62,9 +62,10 @@ const DEFAULT_BUDGET_FLOOR_MS = 30_000
  * NEXT operation may start (see {@link Transaction.execute}), never whether
  * already-completed work is rolled back after the fact. A trip mid-batch
  * still rolls back every operation applied so far, atomically, and throws a
- * retryable, fully-labeled TransactionTimeoutError — that zero-loss guarantee
- * doesn't change; only the point at which the clock stops mattering does (at
- * the last operation, not one check later).
+ * fully-labeled `TransactionTimeoutError` — retryable-with-latch, never
+ * hot-retry (see its `retryable` and `hotRetryUnsafe` fields) — that
+ * zero-loss guarantee doesn't change; only the point at which the clock
+ * stops mattering does (at the last operation, not one check later).
  *
  * @param opCount - Number of operations in the batch.
  * @param override - A full override for this call; wins over everything else.

src/transaction/TransactionManager.ts

@@ -19,7 +19,6 @@
 import { Transaction } from './Transaction.js'
 import {
   TransactionFunction,
-  TransactionResult,
   TransactionOptions
 } from './types.js'
 import { TransactionError } from './errors.js'
@@ -105,34 +104,6 @@ export class TransactionManager {
     }
   }
 
-  /**
-   * Execute a transaction and return detailed result
-   */
-  async executeTransactionWithResult<T>(
-    fn: TransactionFunction<T>,
-    options?: TransactionOptions
-  ): Promise<TransactionResult<T>> {
-    const startTime = Date.now()
-    const transaction = new Transaction(options)
-
-    try {
-      const value = await fn(transaction)
-      await transaction.execute()
-
-      const executionTimeMs = Date.now() - startTime
-
-      return {
-        value,
-        operationCount: transaction.getOperationCount(),
-        executionTimeMs
-      }
-
-    } catch (error) {
-      // Transaction failed
-      throw error
-    }
-  }
-
   /**
    * Get transaction statistics
    */

src/transaction/errors.ts

@@ -73,14 +73,47 @@ export class InvalidTransactionStateError extends TransactionError {
 
 /**
  * Error for transaction timeout
+ *
+ * Machine-readable no-hot-retry contract: {@link retryable} and
+ * {@link hotRetryUnsafe} are both always `true` on this class — they exist
+ * so a caller can branch on the *shape* of the error instead of parsing
+ * message text. Read them together: the operation may eventually succeed,
+ * but never by looping on it immediately.
+ *
+ * `context` (inherited from {@link TransactionError}) carries the caller's
+ * backoff inputs — see the field docs below.
  */
 export class TransactionTimeoutError extends TransactionError {
+  /**
+   * The failed operation MAY succeed on a later attempt — once the
+   * underlying slowness resolves (e.g. a cold page cache warms up) or the
+   * budget is deliberately raised (`transactionBudgetFloorMs`, or a larger
+   * `timeoutMs` override on the batch). This is a statement about eventual
+   * retryability, not a license to retry now — see {@link hotRetryUnsafe}.
+   */
+  public readonly retryable = true
+
+  /**
+   * An immediate, identical retry re-pays the FULL cost of the work that
+   * just timed out — it does not resume partway. Looping on this error
+   * (hot-retrying) repeats that full cost every attempt and can cascade
+   * into a CPU/resource storm on the caller's side. Callers MUST latch: on
+   * this error, record `{ at: Date.now(), error }`, surface one loud
+   * failure to their own caller, and hold a cooldown window before any
+   * re-attempt (clearing the latch only on success). Never retry this error
+   * in a tight loop.
+   */
+  public readonly hotRetryUnsafe = true
+
   constructor(
     timeoutMs: number,
     operationIndex: number,
     telemetry?: {
+      /** Milliseconds elapsed in the transaction when the budget tripped. */
       elapsedMs?: number
+      /** Total number of operations in the batch that timed out. */
       totalOperations?: number
+      /** Name of the operation the batch was about to start when it tripped, if named. */
       operationName?: string
     }
   ) {
@@ -93,8 +126,16 @@ export class TransactionTimeoutError extends TransactionError {
       telemetry?.elapsedMs !== undefined ? `${telemetry.elapsedMs}ms elapsed, ` : ''
     super(
       `Transaction timed out at operation ${progress}${name} — ${elapsed}budget ${timeoutMs}ms. ` +
-        `The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.`,
-      { timeoutMs, operationIndex, ...telemetry }
+        `The batch rolled back atomically; retryable after the underlying slowness resolves or ` +
+        `the budget is raised, but hot-retry-unsafe — latch and back off, never loop.`,
+      {
+        // Caller backoff inputs — all present on every instance:
+        /** Configured budget (ms) that was exceeded. */
+        timeoutMs,
+        /** Index of the operation the batch was about to start when it tripped. */
+        operationIndex,
+        ...telemetry
+      }
     )
     this.name = 'TransactionTimeoutError'
   }

src/transaction/types.ts

@@ -66,26 +66,6 @@ export interface TransactionContext {
  */
 export type TransactionFunction<T> = (ctx: TransactionContext) => Promise<T>
 
-/**
- * Transaction execution result
- */
-export interface TransactionResult<T> {
-  /**
-   * Result value from user function
-   */
-  value: T
-
-  /**
-   * Number of operations executed
-   */
-  operationCount: number
-
-  /**
-   * Execution time in milliseconds
-   */
-  executionTimeMs: number
-}
-
 /**
  * Transaction execution options
  */

src/types/brainy.types.ts

@@ -1658,8 +1658,10 @@ export interface BrainyConfig {
    * **start** — never whether already-completed work gets rolled back after
    * the fact (a single-op write can never time out post-hoc: it either runs
    * or it commits). A trip mid-batch still rolls back every applied operation
-   * atomically and throws a retryable `TransactionTimeoutError`; only the
-   * floor of the formula is configurable here.
+   * atomically and throws a `TransactionTimeoutError` that is
+   * retryable-with-latch, never hot-retry (see its `retryable` and
+   * `hotRetryUnsafe` fields); only the floor of the formula is configurable
+   * here.
    *
    * Raise this when a cold store's first writes after a restart legitimately
    * take longer than 30s per operation (e.g. page-cache-cold canonical writes