8.0 RC cleanup toward "one place per thing, zero-config, no deprecation":
- Remove the `brain.neural()` clustering namespace (ImprovedNeuralAPI + the dead
legacy NeuralAPI + the neural CLI + neural-only types). Similarity is `find({vector})`
/ `similar({to})`; attribute grouping is the aggregation `GROUP BY` engine. The separate
entity-extraction / smart-import feature (NeuralImport, NeuralEntityExtractor, SmartExtractor,
NaturalLanguageProcessor, `brain.extract()`/`brain.nlp()`) is kept.
- Remove `Db.search()`; `find()` is the one query verb (accepts a bare string or FindParams).
Fix the bundled MCP client, which called a non-existent `brain.search(query, limit)` →
now `find({ query, limit })`.
- Storage config: collapse to one canonical top-level `path` key. The pre-8.0 aliases
(`rootDirectory`, `options.*`, `fileSystemStorage.*`) are removed and now THROW with the
exact rename instead of silently defaulting to `./brainy-data` on upgrade. A single resolver
feeds createStorage, the 7.x→8.0 migration probe, and the plugin-factory handoff, so a native
storage provider resolves the identical root (no split-brain).
- Fix `similar({ threshold })`: the min-similarity filter was silently dropped; it is now
applied as a post-filter on `result.score` (the documented way to bound semantic results).
- Fix `vfs.rename()` on a directory: child path updates spread the entity vector into `update()`
and failed dimension validation; they are metadata-only updates now.
- Fix `vfs.move()`: copy+delete orphaned the content-addressed content blob (the destination
shared the source hash, then unlink removed it). `move()` now delegates to `rename()` — an
in-place path change that preserves the blob and the entity id, for files and directories.
- Fix streaming import: the bulk fast path never flushed mid-import nor signalled queryability.
Entity writes are now chunked by a progressive flush interval (100 → 1000 → 5000); each chunk
flushes and emits `progress.queryable`, so imported data is queryable during the import.
- Sweep all docs, comments, and JSDoc for the removed/changed APIs.
Integration suite: 49 files / 588 passed / 0 failed. Unit: 80 files / 1456 passed, no type errors.
241 lines
9 KiB
TypeScript
241 lines
9 KiB
TypeScript
/**
|
|
* @module storage/storageFactory
|
|
* @description Storage adapter factory for Brainy 8.0.
|
|
*
|
|
* Brainy 8.0 ships **two storage adapters**:
|
|
* - `'filesystem'` (default) — Node.js + Bun + Deno; persistent on disk.
|
|
* - `'memory'` — in-memory; ephemeral; the right choice for tests + ephemeral
|
|
* workloads.
|
|
*
|
|
* Cloud-storage adapters (GCS, S3, R2, Azure) and the browser-only OPFS
|
|
* adapter were removed in 8.0. The path
|
|
* forward for cloud backup is operator tooling: persist locally with
|
|
* `db.persist()`, sync the resulting on-disk artefact with `gsutil` /
|
|
* `aws s3 cp` / `rclone` / `azcopy`. This is the standard pattern every
|
|
* production database uses; bundling cloud SDKs into the library buys
|
|
* nothing and ships ~13 K LOC of code we don't maintain well.
|
|
*/
|
|
|
|
import type { StorageAdapter } from '../coreTypes.js'
|
|
import { MemoryStorage } from './adapters/memoryStorage.js'
|
|
import type { OperationConfig } from '../utils/operationUtils.js'
|
|
|
|
/**
|
|
* Options for creating a storage adapter (Brainy 8.0).
|
|
*/
|
|
export interface StorageOptions {
|
|
/**
|
|
* Storage backend to use.
|
|
* - `'auto'` (default) — `'filesystem'` on Node-like runtimes, `'memory'`
|
|
* in a browser.
|
|
* - `'memory'` — in-memory; ephemeral.
|
|
* - `'filesystem'` — persistent disk storage; Node-like runtimes only.
|
|
*
|
|
* A top-level `path` implies `'filesystem'`, so `{ path: '/data' }` works
|
|
* without an explicit `type`.
|
|
*/
|
|
type?: 'auto' | 'memory' | 'filesystem'
|
|
|
|
/** Force memory storage regardless of environment. */
|
|
forceMemoryStorage?: boolean
|
|
|
|
/** Force filesystem storage. Throws in a browser environment. */
|
|
forceFileSystemStorage?: boolean
|
|
|
|
/**
|
|
* **Canonical** directory for filesystem storage. This is the one key the
|
|
* rest of the API already speaks (`persist(path)`, `Brainy.load(path)`,
|
|
* `asOf(path)`, `restore(path)`). Specifying it implies `type: 'filesystem'`.
|
|
*
|
|
* @example
|
|
* new Brainy({ storage: { path: '/var/lib/app/data' } })
|
|
*/
|
|
path?: string
|
|
|
|
/**
|
|
* @deprecated REMOVED in 8.0 — passing `rootDirectory` THROWS. Use the
|
|
* canonical {@link StorageOptions.path} (`storage: { path: '/data' }`).
|
|
*/
|
|
rootDirectory?: string
|
|
|
|
/**
|
|
* @deprecated REMOVED in 8.0 — a nested `options.path` / `options.rootDirectory`
|
|
* THROWS. Use the top-level {@link StorageOptions.path}.
|
|
*/
|
|
options?: {
|
|
rootDirectory?: string
|
|
path?: string
|
|
[key: string]: any
|
|
}
|
|
|
|
/**
|
|
* @deprecated REMOVED in 8.0 — `fileSystemStorage.path` / `.rootDirectory`
|
|
* THROWS. Use the top-level {@link StorageOptions.path}.
|
|
*/
|
|
fileSystemStorage?: {
|
|
rootDirectory?: string
|
|
path?: string
|
|
[key: string]: any
|
|
}
|
|
|
|
/** Operation tuning (timeouts, retry budgets) passed through to the adapter. */
|
|
operationConfig?: OperationConfig
|
|
}
|
|
|
|
/** Default on-disk root when a filesystem store is requested without a path. */
|
|
export const DEFAULT_FILESYSTEM_ROOT = './brainy-data'
|
|
|
|
/**
|
|
* @description Throw a clear migration error for a removed legacy storage-config
|
|
* key. 8.0 is a clean break: the pre-8.0 aliases were REMOVED (not deprecated),
|
|
* so a stale config fails loudly with the exact rename rather than silently
|
|
* writing to the wrong directory.
|
|
* @param key - The removed key path (e.g. `'rootDirectory'`, `'options.path'`).
|
|
* @throws Always — naming the canonical `path` replacement.
|
|
*/
|
|
function throwRemovedStorageKey(key: string): never {
|
|
throw new Error(
|
|
`[brainy] storage config '${key}' was removed in 8.0 — use the top-level ` +
|
|
`'path' instead (e.g. storage: { path: '/data' }).`
|
|
)
|
|
}
|
|
|
|
/**
|
|
* @description Resolve any supported filesystem storage config shape to the ONE
|
|
* canonical on-disk root, encoding the full precedence in a single place so the
|
|
* factory, the 7.x→8.0 migration probe, and any plugin storage factory all agree
|
|
* on the IDENTICAL directory (no `./brainy-data` split-brain).
|
|
*
|
|
* Behavior:
|
|
* 1. `path` — the one supported key. Returned as-is.
|
|
* 2. A removed pre-8.0 alias (`rootDirectory`, `options.*`, `fileSystemStorage.*`) →
|
|
* THROW with the exact rename (never silently fall through to the default,
|
|
* which would misplace a 7.x consumer's data on upgrade).
|
|
* 3. No path at all → `./brainy-data` (the zero-config default).
|
|
*
|
|
* @param config - A storage config object (`StorageOptions`-shaped; tolerant of
|
|
* extra keys for the plugin-factory `Record<string, unknown>` contract).
|
|
* @returns The resolved directory string.
|
|
* @throws If a removed pre-8.0 alias is present (message names the `path` rename).
|
|
* @example
|
|
* resolveFilesystemRoot({ path: '/data' }) // → '/data'
|
|
* resolveFilesystemRoot({ rootDirectory: '/data' }) // → throws (use `path`)
|
|
* resolveFilesystemRoot({ type: 'filesystem' }) // → './brainy-data'
|
|
*/
|
|
export function resolveFilesystemRoot(
|
|
config: StorageOptions & Record<string, unknown> = {}
|
|
): string {
|
|
// 1. Canonical top-level path — the one and only supported key.
|
|
if (typeof config.path === 'string' && config.path.length > 0) {
|
|
return config.path
|
|
}
|
|
|
|
// 2. Removed pre-8.0 aliases → throw with the exact rename. Detect them even
|
|
// though they're no longer the path, so a stale config fails loudly
|
|
// instead of silently landing on the default and misplacing data.
|
|
if (typeof config.rootDirectory === 'string' && config.rootDirectory.length > 0) {
|
|
throwRemovedStorageKey('rootDirectory')
|
|
}
|
|
const opts = config.options
|
|
if (
|
|
opts &&
|
|
typeof opts === 'object' &&
|
|
((typeof opts.path === 'string' && opts.path.length > 0) ||
|
|
(typeof opts.rootDirectory === 'string' && opts.rootDirectory.length > 0))
|
|
) {
|
|
throwRemovedStorageKey('options.path')
|
|
}
|
|
const fss = config.fileSystemStorage
|
|
if (
|
|
fss &&
|
|
typeof fss === 'object' &&
|
|
((typeof fss.path === 'string' && fss.path.length > 0) ||
|
|
(typeof fss.rootDirectory === 'string' && fss.rootDirectory.length > 0))
|
|
) {
|
|
throwRemovedStorageKey('fileSystemStorage.path')
|
|
}
|
|
|
|
// 3. Zero-config default. A `type: 'filesystem'` with no path lands here
|
|
// intentionally ("persist, default location").
|
|
return DEFAULT_FILESYSTEM_ROOT
|
|
}
|
|
|
|
/**
|
|
* @description Whether a storage config (no explicit/`'auto'` type) names a
|
|
* filesystem directory through ANY supported shape. Used by `pickAdapter` so
|
|
* `{ path: '/data' }` (or a deprecated alias) implies filesystem on a Node
|
|
* runtime without the caller writing `type: 'filesystem'`.
|
|
* @param config - A storage config object.
|
|
* @returns `true` if a directory was specified through any recognized key.
|
|
*/
|
|
function hasExplicitFilesystemPath(
|
|
config: StorageOptions & Record<string, unknown>
|
|
): boolean {
|
|
const nonEmpty = (v: unknown): boolean => typeof v === 'string' && v.length > 0
|
|
return (
|
|
nonEmpty(config.path) ||
|
|
nonEmpty(config.rootDirectory) ||
|
|
nonEmpty(config.options?.path) ||
|
|
nonEmpty(config.options?.rootDirectory) ||
|
|
nonEmpty(config.fileSystemStorage?.path) ||
|
|
nonEmpty(config.fileSystemStorage?.rootDirectory)
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Resolve `StorageOptions` to a concrete storage adapter.
|
|
*
|
|
* - `'auto'` (default) picks `FileSystemStorage`, falling back to
|
|
* `MemoryStorage` only if filesystem init fails (e.g. no write permission).
|
|
* - `forceMemoryStorage: true` returns `MemoryStorage` regardless.
|
|
* - `forceFileSystemStorage: true` returns `FileSystemStorage`.
|
|
*/
|
|
export async function createStorage(options: StorageOptions = {}): Promise<StorageAdapter> {
|
|
return pickAdapter(options)
|
|
}
|
|
|
|
async function pickAdapter(options: StorageOptions): Promise<StorageAdapter> {
|
|
if (options.forceMemoryStorage) {
|
|
return new MemoryStorage()
|
|
}
|
|
|
|
const requestedType = options.type ?? 'auto'
|
|
|
|
if (requestedType === 'memory') {
|
|
return new MemoryStorage()
|
|
}
|
|
|
|
if (requestedType === 'filesystem' || options.forceFileSystemStorage) {
|
|
return await createFilesystemStorage(options)
|
|
}
|
|
|
|
// 'auto' (or no type) with an explicit filesystem path → filesystem. A naked
|
|
// `{ path: '/data' }` should land on disk at that path, not silently fall to
|
|
// memory on a runtime where the `auto` filesystem attempt happens to fail.
|
|
if (
|
|
requestedType === 'auto' &&
|
|
hasExplicitFilesystemPath(options as StorageOptions & Record<string, unknown>)
|
|
) {
|
|
return await createFilesystemStorage(options)
|
|
}
|
|
|
|
// 'auto' with no path: prefer filesystem, fall back to memory if init fails.
|
|
try {
|
|
return await createFilesystemStorage(options)
|
|
} catch {
|
|
return new MemoryStorage()
|
|
}
|
|
}
|
|
|
|
async function createFilesystemStorage(options: StorageOptions): Promise<StorageAdapter> {
|
|
// Single source of truth for the on-disk root across every config shape.
|
|
const rootDir = resolveFilesystemRoot(options as StorageOptions & Record<string, unknown>)
|
|
|
|
// Dynamic import so browser bundles don't pull node:fs.
|
|
const { FileSystemStorage } = await import('./adapters/fileSystemStorage.js')
|
|
return new FileSystemStorage(rootDir) as unknown as StorageAdapter
|
|
}
|
|
|
|
// Re-export the surviving adapters for direct construction by callers
|
|
// that want to skip the factory.
|
|
export { MemoryStorage } from './adapters/memoryStorage.js'
|