brainy/src/storage/storageFactory.ts
David Snelling 606445cd61 feat(8.0): API simplification — remove neural()/Db.search, one storage path key, integration→0
8.0 RC cleanup toward "one place per thing, zero-config, no deprecation":

- Remove the `brain.neural()` clustering namespace (ImprovedNeuralAPI + the dead
  legacy NeuralAPI + the neural CLI + neural-only types). Similarity is `find({vector})`
  / `similar({to})`; attribute grouping is the aggregation `GROUP BY` engine. The separate
  entity-extraction / smart-import feature (NeuralImport, NeuralEntityExtractor, SmartExtractor,
  NaturalLanguageProcessor, `brain.extract()`/`brain.nlp()`) is kept.
- Remove `Db.search()`; `find()` is the one query verb (accepts a bare string or FindParams).
  Fix the bundled MCP client, which called a non-existent `brain.search(query, limit)` →
  now `find({ query, limit })`.
- Storage config: collapse to one canonical top-level `path` key. The pre-8.0 aliases
  (`rootDirectory`, `options.*`, `fileSystemStorage.*`) are removed and now THROW with the
  exact rename instead of silently defaulting to `./brainy-data` on upgrade. A single resolver
  feeds createStorage, the 7.x→8.0 migration probe, and the plugin-factory handoff, so a native
  storage provider resolves the identical root (no split-brain).
- Fix `similar({ threshold })`: the min-similarity filter was silently dropped; it is now
  applied as a post-filter on `result.score` (the documented way to bound semantic results).
- Fix `vfs.rename()` on a directory: child path updates spread the entity vector into `update()`
  and failed dimension validation; they are metadata-only updates now.
- Fix `vfs.move()`: copy+delete orphaned the content-addressed content blob (the destination
  shared the source hash, then unlink removed it). `move()` now delegates to `rename()` — an
  in-place path change that preserves the blob and the entity id, for files and directories.
- Fix streaming import: the bulk fast path never flushed mid-import nor signalled queryability.
  Entity writes are now chunked by a progressive flush interval (100 → 1000 → 5000); each chunk
  flushes and emits `progress.queryable`, so imported data is queryable during the import.
- Sweep all docs, comments, and JSDoc for the removed/changed APIs.

Integration suite: 49 files / 588 passed / 0 failed. Unit: 80 files / 1456 passed, no type errors.
2026-06-20 13:31:11 -07:00

241 lines
9 KiB
TypeScript

/**
* @module storage/storageFactory
* @description Storage adapter factory for Brainy 8.0.
*
* Brainy 8.0 ships **two storage adapters**:
* - `'filesystem'` (default) — Node.js + Bun + Deno; persistent on disk.
* - `'memory'` — in-memory; ephemeral; the right choice for tests + ephemeral
* workloads.
*
* Cloud-storage adapters (GCS, S3, R2, Azure) and the browser-only OPFS
* adapter were removed in 8.0. The path
* forward for cloud backup is operator tooling: persist locally with
* `db.persist()`, sync the resulting on-disk artefact with `gsutil` /
* `aws s3 cp` / `rclone` / `azcopy`. This is the standard pattern every
* production database uses; bundling cloud SDKs into the library buys
* nothing and ships ~13 K LOC of code we don't maintain well.
*/
import type { StorageAdapter } from '../coreTypes.js'
import { MemoryStorage } from './adapters/memoryStorage.js'
import type { OperationConfig } from '../utils/operationUtils.js'
/**
* Options for creating a storage adapter (Brainy 8.0).
*/
export interface StorageOptions {
/**
* Storage backend to use.
* - `'auto'` (default) — `'filesystem'` on Node-like runtimes, `'memory'`
* in a browser.
* - `'memory'` — in-memory; ephemeral.
* - `'filesystem'` — persistent disk storage; Node-like runtimes only.
*
* A top-level `path` implies `'filesystem'`, so `{ path: '/data' }` works
* without an explicit `type`.
*/
type?: 'auto' | 'memory' | 'filesystem'
/** Force memory storage regardless of environment. */
forceMemoryStorage?: boolean
/** Force filesystem storage. Throws in a browser environment. */
forceFileSystemStorage?: boolean
/**
* **Canonical** directory for filesystem storage. This is the one key the
* rest of the API already speaks (`persist(path)`, `Brainy.load(path)`,
* `asOf(path)`, `restore(path)`). Specifying it implies `type: 'filesystem'`.
*
* @example
* new Brainy({ storage: { path: '/var/lib/app/data' } })
*/
path?: string
/**
* @deprecated REMOVED in 8.0 — passing `rootDirectory` THROWS. Use the
* canonical {@link StorageOptions.path} (`storage: { path: '/data' }`).
*/
rootDirectory?: string
/**
* @deprecated REMOVED in 8.0 — a nested `options.path` / `options.rootDirectory`
* THROWS. Use the top-level {@link StorageOptions.path}.
*/
options?: {
rootDirectory?: string
path?: string
[key: string]: any
}
/**
* @deprecated REMOVED in 8.0 — `fileSystemStorage.path` / `.rootDirectory`
* THROWS. Use the top-level {@link StorageOptions.path}.
*/
fileSystemStorage?: {
rootDirectory?: string
path?: string
[key: string]: any
}
/** Operation tuning (timeouts, retry budgets) passed through to the adapter. */
operationConfig?: OperationConfig
}
/** Default on-disk root when a filesystem store is requested without a path. */
export const DEFAULT_FILESYSTEM_ROOT = './brainy-data'
/**
* @description Throw a clear migration error for a removed legacy storage-config
* key. 8.0 is a clean break: the pre-8.0 aliases were REMOVED (not deprecated),
* so a stale config fails loudly with the exact rename rather than silently
* writing to the wrong directory.
* @param key - The removed key path (e.g. `'rootDirectory'`, `'options.path'`).
* @throws Always — naming the canonical `path` replacement.
*/
function throwRemovedStorageKey(key: string): never {
throw new Error(
`[brainy] storage config '${key}' was removed in 8.0 — use the top-level ` +
`'path' instead (e.g. storage: { path: '/data' }).`
)
}
/**
* @description Resolve any supported filesystem storage config shape to the ONE
* canonical on-disk root, encoding the full precedence in a single place so the
* factory, the 7.x→8.0 migration probe, and any plugin storage factory all agree
* on the IDENTICAL directory (no `./brainy-data` split-brain).
*
* Behavior:
* 1. `path` — the one supported key. Returned as-is.
* 2. A removed pre-8.0 alias (`rootDirectory`, `options.*`, `fileSystemStorage.*`) →
* THROW with the exact rename (never silently fall through to the default,
* which would misplace a 7.x consumer's data on upgrade).
* 3. No path at all → `./brainy-data` (the zero-config default).
*
* @param config - A storage config object (`StorageOptions`-shaped; tolerant of
* extra keys for the plugin-factory `Record<string, unknown>` contract).
* @returns The resolved directory string.
* @throws If a removed pre-8.0 alias is present (message names the `path` rename).
* @example
* resolveFilesystemRoot({ path: '/data' }) // → '/data'
* resolveFilesystemRoot({ rootDirectory: '/data' }) // → throws (use `path`)
* resolveFilesystemRoot({ type: 'filesystem' }) // → './brainy-data'
*/
export function resolveFilesystemRoot(
config: StorageOptions & Record<string, unknown> = {}
): string {
// 1. Canonical top-level path — the one and only supported key.
if (typeof config.path === 'string' && config.path.length > 0) {
return config.path
}
// 2. Removed pre-8.0 aliases → throw with the exact rename. Detect them even
// though they're no longer the path, so a stale config fails loudly
// instead of silently landing on the default and misplacing data.
if (typeof config.rootDirectory === 'string' && config.rootDirectory.length > 0) {
throwRemovedStorageKey('rootDirectory')
}
const opts = config.options
if (
opts &&
typeof opts === 'object' &&
((typeof opts.path === 'string' && opts.path.length > 0) ||
(typeof opts.rootDirectory === 'string' && opts.rootDirectory.length > 0))
) {
throwRemovedStorageKey('options.path')
}
const fss = config.fileSystemStorage
if (
fss &&
typeof fss === 'object' &&
((typeof fss.path === 'string' && fss.path.length > 0) ||
(typeof fss.rootDirectory === 'string' && fss.rootDirectory.length > 0))
) {
throwRemovedStorageKey('fileSystemStorage.path')
}
// 3. Zero-config default. A `type: 'filesystem'` with no path lands here
// intentionally ("persist, default location").
return DEFAULT_FILESYSTEM_ROOT
}
/**
* @description Whether a storage config (no explicit/`'auto'` type) names a
* filesystem directory through ANY supported shape. Used by `pickAdapter` so
* `{ path: '/data' }` (or a deprecated alias) implies filesystem on a Node
* runtime without the caller writing `type: 'filesystem'`.
* @param config - A storage config object.
* @returns `true` if a directory was specified through any recognized key.
*/
function hasExplicitFilesystemPath(
config: StorageOptions & Record<string, unknown>
): boolean {
const nonEmpty = (v: unknown): boolean => typeof v === 'string' && v.length > 0
return (
nonEmpty(config.path) ||
nonEmpty(config.rootDirectory) ||
nonEmpty(config.options?.path) ||
nonEmpty(config.options?.rootDirectory) ||
nonEmpty(config.fileSystemStorage?.path) ||
nonEmpty(config.fileSystemStorage?.rootDirectory)
)
}
/**
* Resolve `StorageOptions` to a concrete storage adapter.
*
* - `'auto'` (default) picks `FileSystemStorage`, falling back to
* `MemoryStorage` only if filesystem init fails (e.g. no write permission).
* - `forceMemoryStorage: true` returns `MemoryStorage` regardless.
* - `forceFileSystemStorage: true` returns `FileSystemStorage`.
*/
export async function createStorage(options: StorageOptions = {}): Promise<StorageAdapter> {
return pickAdapter(options)
}
async function pickAdapter(options: StorageOptions): Promise<StorageAdapter> {
if (options.forceMemoryStorage) {
return new MemoryStorage()
}
const requestedType = options.type ?? 'auto'
if (requestedType === 'memory') {
return new MemoryStorage()
}
if (requestedType === 'filesystem' || options.forceFileSystemStorage) {
return await createFilesystemStorage(options)
}
// 'auto' (or no type) with an explicit filesystem path → filesystem. A naked
// `{ path: '/data' }` should land on disk at that path, not silently fall to
// memory on a runtime where the `auto` filesystem attempt happens to fail.
if (
requestedType === 'auto' &&
hasExplicitFilesystemPath(options as StorageOptions & Record<string, unknown>)
) {
return await createFilesystemStorage(options)
}
// 'auto' with no path: prefer filesystem, fall back to memory if init fails.
try {
return await createFilesystemStorage(options)
} catch {
return new MemoryStorage()
}
}
async function createFilesystemStorage(options: StorageOptions): Promise<StorageAdapter> {
// Single source of truth for the on-disk root across every config shape.
const rootDir = resolveFilesystemRoot(options as StorageOptions & Record<string, unknown>)
// Dynamic import so browser bundles don't pull node:fs.
const { FileSystemStorage } = await import('./adapters/fileSystemStorage.js')
return new FileSystemStorage(rootDir) as unknown as StorageAdapter
}
// Re-export the surviving adapters for direct construction by callers
// that want to skip the factory.
export { MemoryStorage } from './adapters/memoryStorage.js'