David Snelling
16d2e1a97e
Arming is asynchronous — the request directory is created before it can be watched — so during that window neither the watcher nor the sweep interval exists yet and the guard let a second call through, leaving two watchers and two sweeps for the life of the store. The callback is the flag that covers the window.
3613 lines
147 KiB
TypeScript
3613 lines
147 KiB
TypeScript
/**
|
||
* File System Storage Adapter
|
||
* File system storage adapter for Node.js environments
|
||
*/
|
||
|
||
import {
|
||
HNSWNoun,
|
||
HNSWNounWithMetadata,
|
||
StatisticsData,
|
||
NounType
|
||
} from '../../coreTypes.js'
|
||
import {
|
||
BaseStorage,
|
||
StorageBatchConfig,
|
||
SYSTEM_DIR,
|
||
STATISTICS_KEY,
|
||
WriterLockInfo,
|
||
WriterCloseRecord
|
||
} from '../baseStorage.js'
|
||
import { getBrainyVersion } from '../../utils/index.js'
|
||
import { isAbsentError } from '../../utils/errorClassification.js'
|
||
import { prodLog } from '../../utils/logger.js'
|
||
import { isZeroNormVector } from '../../utils/distance.js'
|
||
import {
|
||
TornRecordError,
|
||
isUnparseablePayloadError,
|
||
registerTornRecordEncounter
|
||
} from '../tornRecordError.js'
|
||
|
||
// Node.js modules - dynamically imported to avoid issues in browser environments
|
||
let fs: any
|
||
let path: any
|
||
let zlib: any
|
||
let moduleLoadingPromise: Promise<void> | null = null
|
||
|
||
// Try to load Node.js modules
|
||
try {
|
||
// Using dynamic imports to avoid issues in browser environments
|
||
const fsPromise = import('node:fs')
|
||
const pathPromise = import('node:path')
|
||
const zlibPromise = import('node:zlib')
|
||
|
||
moduleLoadingPromise = Promise.all([fsPromise, pathPromise, zlibPromise])
|
||
.then(([fsModule, pathModule, zlibModule]) => {
|
||
fs = fsModule
|
||
path = pathModule.default
|
||
zlib = zlibModule
|
||
})
|
||
.catch((error) => {
|
||
console.error('Failed to load Node.js modules:', error)
|
||
throw error
|
||
})
|
||
} catch (error) {
|
||
console.error(
|
||
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
||
error
|
||
)
|
||
}
|
||
|
||
/**
|
||
* File system storage adapter for Node.js environments
|
||
* Uses the file system to store data in the specified directory structure
|
||
*
|
||
* Type-aware storage now built into BaseStorage
|
||
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
||
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
||
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
||
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
||
*/
|
||
export class FileSystemStorage extends BaseStorage {
|
||
// FileSystem-specific count persistence
|
||
private countsFilePath?: string // Will be set after init
|
||
|
||
// Fixed sharding configuration for optimal balance of simplicity and performance
|
||
// Single-level sharding (depth=1) provides excellent performance for 1-2.5M entities
|
||
// Structure: nouns/ab/uuid.json where 'ab' = first 2 hex chars of UUID
|
||
// - 256 shard directories (00-ff)
|
||
// - Handles 2.5M+ entities with < 10K files per shard
|
||
// - Eliminates dynamic depth changes that cause path mismatch bugs
|
||
private readonly SHARDING_DEPTH = 1 as const
|
||
protected rootDir: string
|
||
private nounsDir!: string
|
||
private verbsDir!: string
|
||
private metadataDir!: string
|
||
private nounMetadataDir!: string
|
||
private verbMetadataDir!: string
|
||
private indexDir!: string // Legacy - for backward compatibility
|
||
private systemDir!: string
|
||
private lockDir!: string
|
||
// Root for raw binary blobs (`<rootDir>/_blobs`). Blobs are stored verbatim
|
||
// (no JSON envelope, no compression) so native code can mmap them directly via
|
||
// getBinaryBlobPath(). Set in init() once the path module is loaded.
|
||
private blobsDir!: string
|
||
private activeLocks: Set<string> = new Set()
|
||
private lockTimers: Map<string, NodeJS.Timeout> = new Map() // Track timers for cleanup
|
||
private allTimers: Set<NodeJS.Timeout> = new Set() // Track all timers for cleanup
|
||
|
||
// Writer-lock state. The writer lock at `locks/_writer.lock` is acquired
|
||
// at Brainy.init() in writer mode and released at close(). A heartbeat
|
||
// timer rewrites the lock every 10s so stale-lock detection can tell a dead
|
||
// writer from a slow one. The constant name matches the file path used.
|
||
private static readonly WRITER_LOCK_FILE = '_writer.lock'
|
||
/**
|
||
* The clean-close record at `locks/_writer.close` (see
|
||
* {@link WriterCloseRecord}). Written when the lock is released, consumed by
|
||
* the next claim, so an open can distinguish "the previous writer left" from
|
||
* "the previous writer died" without inferring either from a pid.
|
||
*/
|
||
private static readonly WRITER_CLOSE_FILE = '_writer.close'
|
||
/**
|
||
* How often the lock file's `lastHeartbeat` is rewritten.
|
||
*
|
||
* THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness
|
||
* is decided by PID LIVENESS alone (see isWriterLockStale) and the fence
|
||
* compares pid + hostname — no decision anywhere reads this timestamp. It
|
||
* exists so an operator inspecting a lock file, or reading the
|
||
* BRAINY_WRITER_LOCKED error, can judge liveness themselves.
|
||
*
|
||
* At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1
|
||
* writes/s across a production process holding 21 idle brains, for a
|
||
* human-readable timestamp nothing computes with. At 60s an operator still
|
||
* sees a heartbeat inside the minute, at a sixth of the cost. With the
|
||
* clean-close record now recording orderly releases explicitly, the
|
||
* heartbeat carries even less weight than it did.
|
||
*/
|
||
private static readonly WRITER_HEARTBEAT_MS = 60_000
|
||
private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
|
||
private writerLockHeartbeat?: NodeJS.Timeout
|
||
private writerLockInfo?: WriterLockInfo
|
||
/**
|
||
* The currently-executing heartbeat refresh, if any. `releaseWriterLock()`
|
||
* awaits it before unlinking: clearInterval() stops FUTURE ticks but not a
|
||
* tick already in flight, and a straggler landing after the unlink would
|
||
* RE-CREATE the lock file — a phantom lock blocking the next writer until
|
||
* the stale TTL expires (the pool-eviction reopen case).
|
||
*/
|
||
private writerHeartbeatInFlight?: Promise<void>
|
||
|
||
/**
|
||
* The in-flight background count-ledger derivation, if one was needed at
|
||
* open. See {@link scheduleCountLedgerDerivation} — awaited only by
|
||
* {@link whenCountLedgerSettled}, never by a read.
|
||
*/
|
||
private countLedgerDerivation?: Promise<void>
|
||
|
||
// Flush-request RPC state. The writer polls `locks/_flush_requests/` for
|
||
// new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
|
||
// flushing. Inspectors call `requestFlushOverFilesystem` to drop a request
|
||
// and wait for the ack. Polling interval is short enough to feel synchronous
|
||
// for operator workflows but doesn't pressure the FS.
|
||
private static readonly FLUSH_REQUEST_DIR = '_flush_requests'
|
||
private static readonly FLUSH_RESPONSE_DIR = '_flush_responses'
|
||
private static readonly FLUSH_WATCH_INTERVAL_MS = 500
|
||
/**
|
||
* The safety sweep behind the fs.watch: catches events an exotic filesystem
|
||
* dropped, and runs the stale-request GC. See startFlushRequestWatcher.
|
||
*/
|
||
private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000
|
||
private static readonly FLUSH_POLL_INTERVAL_MS = 100
|
||
private static readonly FLUSH_REQUEST_TTL_MS = 60_000
|
||
private flushWatcherInterval?: NodeJS.Timeout
|
||
/** The inotify-backed watch on the request directory, when the FS supports one. */
|
||
private flushWatcher?: import('node:fs').FSWatcher
|
||
private flushWatcherInFlight = false
|
||
private flushWatcherOnRequest?: () => Promise<void>
|
||
|
||
// CRITICAL FIX: Mutex locks for HNSW concurrency control
|
||
// Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops)
|
||
// Matches MemoryStorage and OPFSStorage behavior (tested in production)
|
||
private hnswLocks = new Map<string, Promise<void>>()
|
||
|
||
// Compression configuration
|
||
private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings
|
||
private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced)
|
||
|
||
// Transaction durability barrier (see GenerationStorage.beginWriteBarrier).
|
||
// Non-null ONLY between beginWriteBarrier() and flushWriteBarrier(). Because
|
||
// canonical writes are tmp+rename (durable only in the page cache until an
|
||
// fsync), the generation store fsyncs everything a transaction wrote before
|
||
// it advances the generation counter. `writeBarrierPaths` collects the
|
||
// root-relative object paths written; `writeBarrierDeleteDirs` the parent
|
||
// dirs of deleted objects (an unlink is durable only once its directory is
|
||
// fsync'd). Both are null outside a transaction, so the tracking `add`s below
|
||
// are free on the single-op and non-transactional write paths.
|
||
private writeBarrierPaths: Set<string> | null = null
|
||
private writeBarrierDeleteDirs: Set<string> | null = null
|
||
|
||
/**
|
||
* Initialize the storage adapter
|
||
* @param rootDirectory The root directory for storage
|
||
* @param options Optional configuration
|
||
*/
|
||
constructor(
|
||
rootDirectory: string,
|
||
options?: {
|
||
compression?: boolean // Enable gzip compression (default: true)
|
||
compressionLevel?: number // Compression level 1-9 (default: 6)
|
||
}
|
||
) {
|
||
super()
|
||
this.rootDir = rootDirectory
|
||
|
||
// Configure compression
|
||
if (options?.compression !== undefined) {
|
||
this.compressionEnabled = options.compression
|
||
}
|
||
if (options?.compressionLevel !== undefined) {
|
||
this.compressionLevel = Math.min(9, Math.max(1, options.compressionLevel))
|
||
}
|
||
|
||
// Defer path operations until init() when path module is guaranteed to be loaded
|
||
}
|
||
|
||
/**
|
||
* Get FileSystem-optimized batch configuration
|
||
*
|
||
* File system storage is I/O bound but not rate limited:
|
||
* - Large batch sizes (500 items)
|
||
* - No delays needed (0ms)
|
||
* - Moderate concurrency (100 operations) - limited by I/O threads
|
||
* - Parallel processing supported
|
||
*
|
||
* @returns FileSystem-optimized batch configuration
|
||
*/
|
||
public override getBatchConfig(): StorageBatchConfig {
|
||
return {
|
||
maxBatchSize: 500,
|
||
batchDelayMs: 0,
|
||
maxConcurrent: 100,
|
||
supportsParallelWrites: true, // Filesystem handles parallel I/O
|
||
rateLimit: {
|
||
operationsPerSecond: 5000, // Depends on disk speed
|
||
burstCapacity: 2000
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Initialize the storage adapter
|
||
*/
|
||
public override async init(): Promise<void> {
|
||
if (this.isInitialized) {
|
||
return
|
||
}
|
||
|
||
// Wait for module loading to complete
|
||
if (moduleLoadingPromise) {
|
||
try {
|
||
await moduleLoadingPromise
|
||
} catch (error) {
|
||
throw new Error(
|
||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||
)
|
||
}
|
||
}
|
||
|
||
// Check if Node.js modules are available
|
||
if (!fs || !path) {
|
||
throw new Error(
|
||
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
||
)
|
||
}
|
||
|
||
try {
|
||
// Initialize directory paths now that path module is loaded
|
||
// Clean directory structure
|
||
this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw')
|
||
this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw')
|
||
this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference
|
||
this.nounMetadataDir = path.join(this.rootDir, 'entities/nouns/metadata')
|
||
this.verbMetadataDir = path.join(this.rootDir, 'entities/verbs/metadata')
|
||
this.indexDir = path.join(this.rootDir, 'indexes')
|
||
this.systemDir = path.join(this.rootDir, SYSTEM_DIR)
|
||
this.lockDir = path.join(this.rootDir, 'locks')
|
||
this.blobsDir = path.join(this.rootDir, '_blobs')
|
||
|
||
// Create the root directory if it doesn't exist
|
||
await this.ensureDirectoryExists(this.rootDir)
|
||
|
||
// Finish any restore interrupted by a crash (resume the staged swap, or
|
||
// discard an uncommitted staging area) BEFORE counts/derived state load,
|
||
// so the rest of startup sees the completed store. ORDER-DEPENDENT:
|
||
// `swapStagedRestoreIn()` reads `fs.readdir(rootDir)` and then
|
||
// removes/renames rootDir's own TOP-LEVEL entries to place the staged
|
||
// copy — racing that against the directory-creation batch below (which
|
||
// also touches rootDir's children) could see a half-created directory
|
||
// mid-swap or a mkdir racing a concurrent rm/rename on the same path.
|
||
// Stays strictly sequential, never folded into the OPEN-PATH batch.
|
||
await this.completeInterruptedRestore()
|
||
|
||
// OPEN-PATH FIX: the remaining bootstrap directories are mutually
|
||
// independent — each is its own subtree under rootDir, and
|
||
// `fs.mkdir(dir, { recursive: true })` creates every intermediate
|
||
// segment of ITS OWN path in one call, so it never depends on any
|
||
// sibling here existing first. Nothing between here and
|
||
// `initializeCounts()` reads any of them, so batching collapses what
|
||
// was up to 8 sequential mkdir round-trips (each a real syscall+await)
|
||
// into one wave — this is what serialized an N-writer restart storm on
|
||
// filesystem I/O it never structurally needed. `initializeCounts()`
|
||
// right after DOES depend on `systemDir` (which the batch creates), so
|
||
// it stays outside, awaited only once every directory has landed.
|
||
await Promise.all([
|
||
// Create the nouns directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.nounsDir),
|
||
// Create the verbs directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.verbsDir),
|
||
// Create the metadata directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.metadataDir),
|
||
// Create the noun metadata directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.nounMetadataDir),
|
||
// Create the verb metadata directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.verbMetadataDir),
|
||
// Create both directories for backward compatibility
|
||
this.ensureDirectoryExists(this.systemDir),
|
||
// Only create legacy directory if it exists (don't create new legacy
|
||
// dirs) — a read-then-maybe-write, but on its own subtree, so it's
|
||
// still independent of every other entry in this batch.
|
||
(async () => {
|
||
if (await this.directoryExists(this.indexDir)) {
|
||
await this.ensureDirectoryExists(this.indexDir)
|
||
}
|
||
})(),
|
||
// Create the locks directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.lockDir),
|
||
// Create the binary blobs directory if it doesn't exist
|
||
this.ensureDirectoryExists(this.blobsDir)
|
||
])
|
||
|
||
// Initialize count management — depends on systemDir, created above.
|
||
this.countsFilePath = path.join(this.systemDir, 'counts.json')
|
||
await this.initializeCounts()
|
||
|
||
// Boot log: new-vs-established, decided from the canonical layout the
|
||
// database actually reads and writes (`entities/nouns/<shard>/<id>/`)
|
||
// plus the known noun count. The legacy hnsw sharding-depth probe and
|
||
// its depth-migration machinery are gone: the 8.0 write path never
|
||
// populated the directory they inspected, so the probe concluded "new
|
||
// installation" for every store on every boot and the migration branch
|
||
// was unreachable.
|
||
const established =
|
||
this.totalNounCount > 0 || (await this.hasCanonicalEntities())
|
||
console.log(
|
||
established
|
||
? `📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)`
|
||
: `📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)`
|
||
)
|
||
|
||
// Initialize GraphAdjacencyIndex and type statistics
|
||
await super.init()
|
||
} catch (error) {
|
||
console.error('Error initializing FileSystemStorage:', error)
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if a directory exists
|
||
*/
|
||
private async directoryExists(dirPath: string): Promise<boolean> {
|
||
try {
|
||
const stats = await fs.promises.stat(dirPath)
|
||
return stats.isDirectory()
|
||
} catch (error) {
|
||
return false
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Ensure a directory exists, creating it if necessary
|
||
*/
|
||
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
||
try {
|
||
await fs.promises.mkdir(dirPath, { recursive: true })
|
||
} catch (error: any) {
|
||
// Ignore EEXIST error, which means the directory already exists
|
||
if (error.code !== 'EEXIST') {
|
||
throw error
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
/**
|
||
* Primitive operation: Write object to path
|
||
* All metadata operations use this internally via base class routing
|
||
* Supports gzip compression for 60-80% disk savings
|
||
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
|
||
*/
|
||
protected async writeObjectToPath(pathStr: string, data: any): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
const fullPath = path.join(this.rootDir, pathStr)
|
||
await this.ensureDirectoryExists(path.dirname(fullPath))
|
||
|
||
if (this.compressionEnabled) {
|
||
// Write compressed data with .gz extension using atomic pattern
|
||
const compressedPath = `${fullPath}.gz`
|
||
const tempPath = `${compressedPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||
|
||
try {
|
||
// ATOMIC WRITE SEQUENCE:
|
||
// 1. Compress and write to temp file
|
||
const jsonString = JSON.stringify(data, null, 2)
|
||
const compressed = await new Promise<Buffer>((resolve, reject) => {
|
||
zlib.gzip(Buffer.from(jsonString, 'utf-8'), { level: this.compressionLevel }, (err: any, result: Buffer) => {
|
||
if (err) reject(err)
|
||
else resolve(result)
|
||
})
|
||
})
|
||
await fs.promises.writeFile(tempPath, compressed)
|
||
|
||
// 2. Atomic rename temp → final (crash-safe, prevents truncation during concurrent writes)
|
||
await fs.promises.rename(tempPath, compressedPath)
|
||
} catch (error: any) {
|
||
// Clean up temp file on any error
|
||
try {
|
||
await fs.promises.unlink(tempPath)
|
||
} catch (cleanupError) {
|
||
// Ignore cleanup errors
|
||
}
|
||
throw error
|
||
}
|
||
|
||
// Clean up uncompressed file if it exists (migration from uncompressed)
|
||
try {
|
||
await fs.promises.unlink(fullPath)
|
||
} catch (error: any) {
|
||
// Ignore if file doesn't exist
|
||
if (error.code !== 'ENOENT') {
|
||
console.warn(`Failed to remove uncompressed file ${fullPath}:`, error)
|
||
}
|
||
}
|
||
} else {
|
||
// Write uncompressed data using atomic pattern
|
||
const tempPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||
|
||
try {
|
||
// ATOMIC WRITE SEQUENCE:
|
||
// 1. Write to temp file
|
||
await fs.promises.writeFile(tempPath, JSON.stringify(data, null, 2))
|
||
|
||
// 2. Atomic rename temp → final (crash-safe, prevents truncation during concurrent writes)
|
||
await fs.promises.rename(tempPath, fullPath)
|
||
} catch (error: any) {
|
||
// Clean up temp file on any error
|
||
try {
|
||
await fs.promises.unlink(tempPath)
|
||
} catch (cleanupError) {
|
||
// Ignore cleanup errors
|
||
}
|
||
throw error
|
||
}
|
||
}
|
||
|
||
// Transaction durability barrier: record the write so the generation store
|
||
// can fsync it before advancing the counter. Reached only on a successful
|
||
// rename; the logical (non-`.gz`) path is stored — flushWriteBarrier's
|
||
// syncRawObjects resolves the compressed variant.
|
||
this.writeBarrierPaths?.add(pathStr)
|
||
}
|
||
|
||
/**
|
||
* Primitive operation: Read object from path
|
||
* All metadata operations use this internally via base class routing
|
||
* Supports reading both compressed (.gz) and uncompressed files for backward compatibility
|
||
*
|
||
* Read contract (loud errors, never quiet losses):
|
||
* - Genuine absence (ENOENT on every variant) → `null`. Only a missing file
|
||
* is "not found".
|
||
* - TORN record (a file EXISTS but its bytes cannot be decoded — invalid
|
||
* JSON, truncated/garbled gzip) → the encounter is registered (production
|
||
* ERROR log + per-process gauge) and a typed {@link TornRecordError} is
|
||
* thrown. Corruption must NEVER read as absence: callers that can degrade
|
||
* (manifest recovery, rebuildable statistics) catch the typed error at
|
||
* their sites; entity reads surface it.
|
||
* Legacy dual-format exception: when the `.gz` variant is torn but the
|
||
* uncompressed fallback decodes, the recovered object is returned — AFTER
|
||
* the torn `.gz` was logged and counted (loud recovery, not a silent skip).
|
||
* - Real storage fault (EIO/EACCES/EMFILE/…) → propagates as itself; a
|
||
* fault is neither absence nor corruption and must not be reshaped.
|
||
*/
|
||
protected async readObjectFromPath(pathStr: string): Promise<any | null> {
|
||
await this.ensureInitialized()
|
||
|
||
const fullPath = path.join(this.rootDir, pathStr)
|
||
const compressedPath = `${fullPath}.gz`
|
||
|
||
// Try reading compressed file first (if compression is enabled or file exists).
|
||
// A torn .gz is remembered so the uncompressed fallback can either recover
|
||
// (legacy dual-format installs) or surface the corruption typed.
|
||
let tornCompressed: TornRecordError | null = null
|
||
try {
|
||
const compressedData = await fs.promises.readFile(compressedPath)
|
||
const decompressed = await new Promise<Buffer>((resolve, reject) => {
|
||
zlib.gunzip(compressedData, (err: any, result: Buffer) => {
|
||
if (err) reject(err)
|
||
else resolve(result)
|
||
})
|
||
})
|
||
return JSON.parse(decompressed.toString('utf-8'))
|
||
} catch (error: any) {
|
||
if (error.code === 'ENOENT') {
|
||
// No compressed variant — fall through to the uncompressed path.
|
||
} else if (isUnparseablePayloadError(error)) {
|
||
// The .gz EXISTS but cannot be decoded (zlib Z_* error or JSON
|
||
// SyntaxError after gunzip): torn record. Register NOW (log + gauge),
|
||
// then attempt the uncompressed fallback as a recovery read.
|
||
tornCompressed = registerTornRecordEncounter(`${pathStr}.gz`, error)
|
||
} else {
|
||
// Real storage fault on an existing .gz (EIO/EACCES/…): propagate.
|
||
throw error
|
||
}
|
||
}
|
||
|
||
// Fall back to reading uncompressed file (for backward compatibility)
|
||
try {
|
||
const data = await fs.promises.readFile(fullPath, 'utf-8')
|
||
return JSON.parse(data)
|
||
} catch (error: any) {
|
||
if (error.code === 'ENOENT') {
|
||
// No uncompressed file. If the .gz variant existed but was torn, the
|
||
// object EXISTS and is unreadable — that must surface typed, never as
|
||
// "absent". Otherwise this is genuine absence.
|
||
if (tornCompressed !== null) {
|
||
throw tornCompressed
|
||
}
|
||
return null
|
||
}
|
||
|
||
// The file EXISTS but its content cannot be parsed: torn record.
|
||
// Register (production ERROR + gauge) and throw typed — a corrupt row
|
||
// must be distinguishable from a missing row, or nothing ever heals it.
|
||
if (isUnparseablePayloadError(error)) {
|
||
throw registerTornRecordEncounter(pathStr, error)
|
||
}
|
||
|
||
// A real storage fault (EIO/EACCES/EMFILE/…) is NOT "object absent". The
|
||
// ENOENT branch (above) already returns null; a genuine fault reaching
|
||
// here must propagate loudly rather than masquerade as a missing object
|
||
// — which would corrupt reads and drive needless rebuilds.
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Primitive operation: Delete object from path
|
||
* All metadata operations use this internally via base class routing
|
||
* Deletes both compressed and uncompressed versions (for cleanup)
|
||
*/
|
||
protected async deleteObjectFromPath(pathStr: string): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
const fullPath = path.join(this.rootDir, pathStr)
|
||
const compressedPath = `${fullPath}.gz`
|
||
|
||
// Try deleting both compressed and uncompressed files (for cleanup during migration)
|
||
let deletedCount = 0
|
||
|
||
// Delete compressed file
|
||
try {
|
||
await fs.promises.unlink(compressedPath)
|
||
deletedCount++
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.warn(`Error deleting compressed file ${compressedPath}:`, error)
|
||
}
|
||
}
|
||
|
||
// Delete uncompressed file
|
||
try {
|
||
await fs.promises.unlink(fullPath)
|
||
deletedCount++
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.error(`Error deleting uncompressed file ${pathStr}:`, error)
|
||
throw error
|
||
}
|
||
}
|
||
|
||
// If neither file existed, it's not an error (already deleted)
|
||
if (deletedCount === 0) {
|
||
// File doesn't exist - this is fine
|
||
}
|
||
|
||
// Transaction durability barrier: an unlink is durable only once its parent
|
||
// directory is fsync'd. Record the dir (root-relative; '.' for a top-level
|
||
// object) so flushWriteBarrier can sync it before the counter advances.
|
||
this.writeBarrierDeleteDirs?.add(path.dirname(pathStr))
|
||
}
|
||
|
||
/**
|
||
* @description Remove the entity container directory after both canonical legs
|
||
* were deleted (full-removal delete). `objectLegPath` is one leg's
|
||
* storage-root-relative path (e.g. `entities/nouns/<shard>/<id>/vectors.json`);
|
||
* its parent is the `<id>/` entity directory. `rmdir` removes it ONLY if empty
|
||
* — both legs are gone by this point, so it should be. A non-empty dir means an
|
||
* unexpected leftover (a leg whose delete faulted — already surfaced upstream —
|
||
* or foreign data): we do NOT recursively nuke it (loud errors, never quiet
|
||
* losses); we log and leave it for the orphan repair sweep (`repairIndex`).
|
||
*/
|
||
protected override async removeCanonicalContainer(objectLegPath: string): Promise<void> {
|
||
const relDir = path.dirname(objectLegPath)
|
||
const absDir = path.join(this.rootDir, relDir)
|
||
try {
|
||
await fs.promises.rmdir(absDir)
|
||
// The dir removal is durable once its PARENT (the shard dir) is fsync'd.
|
||
this.writeBarrierDeleteDirs?.add(path.dirname(relDir))
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') return // container already gone — fine
|
||
if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') {
|
||
console.warn(
|
||
`[FileSystemStorage] entity container ${relDir} not empty after delete — ` +
|
||
`leaving it for the orphan repair sweep (brain.repairIndex()).`
|
||
)
|
||
return
|
||
}
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @description Prune orphaned entity directories left by the pre-8.3.1
|
||
* partial-delete defect. A delete used to remove the metadata (content) leg
|
||
* but leave the `vectors.json` leg + the `<id>/` directory (a "ghost"), or —
|
||
* on the direct storage path — remove both legs but leave an empty directory
|
||
* (a "scar"). Neither is a live entity (`getNoun` needs the metadata content
|
||
* leg) yet each lingers on disk, inflating enumerated counts and confusing
|
||
* locator resolution.
|
||
*
|
||
* CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg — a
|
||
* directory that still holds its content is left untouched. LOUD: logs every
|
||
* removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs
|
||
* automatically. Returns the pruned container ids so the caller can recompute
|
||
* counts.
|
||
*/
|
||
/**
|
||
* @description Whether an id directory's file legs include the metadata
|
||
* CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single
|
||
* test that decides whether an `entities/<kind>/<shard>/<id>/` container is
|
||
* a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete
|
||
* defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune
|
||
* and {@link scanCanonicalEntities} so the two agree by construction — one
|
||
* counted entity per identity record, never per bare container.
|
||
* @param legs - File names in one `entities/<kind>/<shard>/<id>/` directory.
|
||
*/
|
||
private hasMetadataContentLeg(legs: string[]): boolean {
|
||
return legs.some((f) => f.startsWith('metadata.json'))
|
||
}
|
||
|
||
public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
|
||
await this.ensureInitialized()
|
||
const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
|
||
|
||
for (const [kind, root] of [
|
||
['nouns', 'entities/nouns'],
|
||
['verbs', 'entities/verbs']
|
||
] as const) {
|
||
const rootAbs = path.join(this.rootDir, root)
|
||
let shards: string[]
|
||
try {
|
||
shards = await fs.promises.readdir(rootAbs)
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') continue // no entities of this kind yet
|
||
throw error
|
||
}
|
||
|
||
for (const shard of shards) {
|
||
const shardAbs = path.join(rootAbs, shard)
|
||
let entries: import('fs').Dirent[]
|
||
try {
|
||
entries = await fs.promises.readdir(shardAbs, { withFileTypes: true })
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') continue
|
||
throw error
|
||
}
|
||
|
||
for (const entry of entries) {
|
||
if (!entry.isDirectory()) continue
|
||
const idAbs = path.join(shardAbs, entry.name)
|
||
let legs: string[]
|
||
try {
|
||
legs = await fs.promises.readdir(idAbs)
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') continue
|
||
throw error
|
||
}
|
||
// A live entity has its metadata content leg. No content leg → a
|
||
// vector-only ghost or an empty scar → prune the whole container.
|
||
if (this.hasMetadataContentLeg(legs)) continue
|
||
await fs.promises.rm(idAbs, { recursive: true, force: true })
|
||
pruned[kind].push(entry.name)
|
||
console.warn(
|
||
`[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` +
|
||
`container ${root}/${shard}/${entry.name} (no metadata content leg)`
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
return pruned
|
||
}
|
||
|
||
/**
|
||
* @description The IMMEDIATE child directory names under a prefix — ONE
|
||
* `readdir`, no recursion, no file paths. See the seam's JSDoc
|
||
* (`src/db/types.ts`) for what this replaced: discovering the generations on
|
||
* disk walked the entire generation log on every open, reading out every
|
||
* file in every generation, to learn the set of integers the top-level
|
||
* directory names already spell.
|
||
* @param prefix - Storage-root-relative directory prefix.
|
||
* @returns The child directory names (not paths); empty when the prefix does
|
||
* not exist.
|
||
*/
|
||
public override async listRawPrefixes(prefix: string): Promise<string[]> {
|
||
await this.ensureInitialized()
|
||
const fullPath = path.join(this.rootDir, prefix)
|
||
try {
|
||
const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
|
||
return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory())
|
||
.map((e: { name: string }) => e.name)
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') return []
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Primitive operation: List objects under path prefix
|
||
* All metadata operations use this internally via base class routing
|
||
* Handles both .json and .json.gz files, normalizes paths
|
||
*/
|
||
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
|
||
await this.ensureInitialized()
|
||
|
||
const fullPath = path.join(this.rootDir, prefix)
|
||
const paths: string[] = []
|
||
const seen = new Set<string>() // Track files to avoid duplicates (both .json and .json.gz)
|
||
|
||
try {
|
||
const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
|
||
|
||
for (const entry of entries) {
|
||
if (entry.isFile()) {
|
||
// Handle multiple compression formats for broad compatibility
|
||
// - .json.gz: Standard entity/metadata files (JSON compressed)
|
||
// - .gz: Raw compressed payloads (e.g. blob-store binary values)
|
||
// - .json: Uncompressed JSON files
|
||
if (entry.name.endsWith('.json.gz')) {
|
||
// Strip .gz extension and add the .json path
|
||
const normalizedName = entry.name.slice(0, -3) // Remove .gz
|
||
const normalizedPath = path.join(prefix, normalizedName)
|
||
if (!seen.has(normalizedPath)) {
|
||
paths.push(normalizedPath)
|
||
seen.add(normalizedPath)
|
||
}
|
||
} else if (entry.name.endsWith('.gz')) {
|
||
// Raw payloads stored as .gz (not .json.gz)
|
||
// Strip .gz extension and return path
|
||
const normalizedName = entry.name.slice(0, -3) // Remove .gz
|
||
const normalizedPath = path.join(prefix, normalizedName)
|
||
if (!seen.has(normalizedPath)) {
|
||
paths.push(normalizedPath)
|
||
seen.add(normalizedPath)
|
||
}
|
||
} else if (entry.name.endsWith('.json')) {
|
||
const filePath = path.join(prefix, entry.name)
|
||
if (!seen.has(filePath)) {
|
||
paths.push(filePath)
|
||
seen.add(filePath)
|
||
}
|
||
}
|
||
} else if (entry.isDirectory()) {
|
||
const subpath = path.join(prefix, entry.name)
|
||
const subdirPaths = await this.listObjectsUnderPath(subpath)
|
||
paths.push(...subdirPaths)
|
||
}
|
||
}
|
||
|
||
return paths.sort()
|
||
} catch (error: any) {
|
||
if (error.code === 'ENOENT') {
|
||
return []
|
||
}
|
||
throw error
|
||
}
|
||
}
|
||
|
||
// ===========================================================================
|
||
// Generational record layer (8.0 MVCC) — durability + snapshot primitives
|
||
// ===========================================================================
|
||
|
||
/**
|
||
* Storage-root-relative paths that are mutated **in place** (appended to)
|
||
* rather than replaced via atomic tmp+rename. `snapshotToDirectory()` must
|
||
* byte-copy these instead of hard-linking them: a hard link shares the
|
||
* inode, so a post-snapshot append to the live file would silently mutate
|
||
* the snapshot. Every other persisted file in this adapter is written via
|
||
* tmp+rename (objects, blobs, counts, locks are excluded entirely), which
|
||
* makes hard links safe — a rewrite swaps in a new inode and the snapshot
|
||
* keeps the old one.
|
||
*/
|
||
private static readonly SNAPSHOT_BYTE_COPY_PATHS = new Set<string>([
|
||
`${SYSTEM_DIR}/tx-log.jsonl`
|
||
])
|
||
|
||
/**
|
||
* Top-level directories whose EVERY file is mutated in place (not tmp+rename)
|
||
* and must therefore be byte-copied into a snapshot, not hard-linked — the
|
||
* directory analogue of {@link SNAPSHOT_BYTE_COPY_PATHS}.
|
||
*
|
||
* `_id_mapper` holds the native provider's shared mmap `BinaryIdMapper`, which
|
||
* `flush()` msyncs and a migration rebuild can truncate+re-inject IN PLACE
|
||
* (confirmed by the native side). A hard link shares the inode, so an in-place
|
||
* msync/truncate on the live file would reach through into the pre-upgrade
|
||
* backup — byte-copy keeps the snapshot a faithful, independent copy. It is
|
||
* bounded (the id map), not the large index files, so the copy cost is small.
|
||
*/
|
||
private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set<string>(['_id_mapper'])
|
||
|
||
/**
|
||
* Nested path PREFIXES whose files are byte-copied into snapshots, not
|
||
* hard-linked — for append-in-place files below the top level. The
|
||
* generation fact log's tail segment is appended in place between rotations;
|
||
* a hard-linked tail would let post-snapshot appends reach through into the
|
||
* snapshot. (Sealed segments are immutable and would be link-safe, but the
|
||
* prefix rule keeps the discipline simple; segments are bounded by the
|
||
* rotation threshold, so the copy cost is small.)
|
||
*/
|
||
private static readonly SNAPSHOT_BYTE_COPY_PREFIXES: string[] = ['_generations/facts/']
|
||
|
||
/**
|
||
* Top-level directories excluded from snapshots: process-local lock state
|
||
* (writer lock, flush-request RPC files) must never travel with the data, and
|
||
* the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch
|
||
* that must never be captured or restored.
|
||
*/
|
||
private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks', '_restore_staging'])
|
||
|
||
/**
|
||
* Transient top-level directory holding a restore-in-progress: the snapshot is
|
||
* fully copied here (sparse-aware) BEFORE any live data is touched, then an
|
||
* atomic per-entry swap moves it into place. Its presence + the completion
|
||
* marker let {@link completeInterruptedRestore} resume a crashed restore.
|
||
*/
|
||
private static readonly RESTORE_STAGING_DIR = '_restore_staging'
|
||
/**
|
||
* Written+fsync'd inside the staging dir ONLY after the whole snapshot has
|
||
* copied successfully. Its presence authorizes the swap (and its resume): a
|
||
* staging dir WITHOUT this marker is an interrupted copy — discardable debris,
|
||
* live data still authoritative.
|
||
*/
|
||
private static readonly RESTORE_MARKER = '.restore-manifest.json'
|
||
/** Chunk size for sparse-aware copying (holes are preserved at this grain). */
|
||
private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024
|
||
/** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */
|
||
private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024)
|
||
|
||
/**
|
||
* Remove every object under a storage-root-relative prefix — one recursive
|
||
* directory removal instead of the base class's list+delete loop.
|
||
*
|
||
* @param prefix - Storage-root-relative directory prefix to remove.
|
||
*/
|
||
public override async removeRawPrefix(prefix: string): Promise<void> {
|
||
await this.ensureInitialized()
|
||
// Registered-blob contract: a prefix-nuke must not take out a protected
|
||
// family member. Throws if the prefix intersects one.
|
||
await this.assertPrefixNotProtected(prefix)
|
||
await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true })
|
||
}
|
||
|
||
/**
|
||
* Durability barrier: `fsync` each listed object file (resolving the
|
||
* compressed `.gz` variant when present) and then the set of parent
|
||
* directories, so both the file contents and the rename directory entries
|
||
* are durable before the commit protocol proceeds. Paths whose file no
|
||
* longer exists are skipped (a later step may have replaced them).
|
||
*
|
||
* @param paths - Storage-root-relative object paths previously written.
|
||
*/
|
||
public override async syncRawObjects(paths: string[]): Promise<void> {
|
||
await this.ensureInitialized()
|
||
const parentDirs = new Set<string>()
|
||
|
||
for (const objectPath of paths) {
|
||
const fullPath = path.join(this.rootDir, objectPath)
|
||
let synced = false
|
||
for (const candidate of [`${fullPath}.gz`, fullPath]) {
|
||
let handle: any
|
||
try {
|
||
handle = await fs.promises.open(candidate, 'r')
|
||
} catch (error: any) {
|
||
if (error.code === 'ENOENT') continue
|
||
throw error
|
||
}
|
||
try {
|
||
await handle.sync()
|
||
} finally {
|
||
await handle.close()
|
||
}
|
||
parentDirs.add(path.dirname(fullPath))
|
||
synced = true
|
||
break
|
||
}
|
||
// An absent path is a state too: fsync the parent directory so a
|
||
// completed unlink is durable (a delete must survive power loss as
|
||
// surely as a write — otherwise a bounded log fold could let a
|
||
// tombstoned record resurrect from a lost directory update).
|
||
if (!synced) parentDirs.add(path.dirname(fullPath))
|
||
}
|
||
|
||
for (const dir of parentDirs) {
|
||
let handle: any
|
||
try {
|
||
handle = await fs.promises.open(dir, 'r')
|
||
} catch {
|
||
continue // directory vanished or platform disallows opening dirs
|
||
}
|
||
try {
|
||
await handle.sync()
|
||
} catch {
|
||
// Some platforms (and some filesystems) reject directory fsync —
|
||
// file-level fsync above already covers the data itself.
|
||
} finally {
|
||
await handle.close()
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Begin a transaction durability barrier: start recording every canonical
|
||
* object write and delete so {@link flushWriteBarrier} can fsync them before
|
||
* the generation counter advances. Resets unconditionally, discarding any
|
||
* tracking left by a transaction that aborted without flushing.
|
||
*
|
||
* @see GenerationStorage.beginWriteBarrier
|
||
*/
|
||
public beginWriteBarrier(): void {
|
||
this.writeBarrierPaths = new Set()
|
||
this.writeBarrierDeleteDirs = new Set()
|
||
}
|
||
|
||
/**
|
||
* Flush the transaction durability barrier: fsync every canonical write since
|
||
* {@link beginWriteBarrier} (file contents AND the rename directory entries,
|
||
* via {@link syncRawObjects}), then fsync the parent directory of every
|
||
* canonical delete so the unlinks are durable too. Clears the tracking. After
|
||
* this resolves, the transaction's entire canonical footprint is on disk, so
|
||
* the generation counter/manifest can be advanced without risking a
|
||
* counter-ahead-of-state torn store on a hard kill.
|
||
*
|
||
* @see GenerationStorage.flushWriteBarrier
|
||
*/
|
||
public async flushWriteBarrier(): Promise<void> {
|
||
const paths = this.writeBarrierPaths
|
||
const deleteDirs = this.writeBarrierDeleteDirs
|
||
this.writeBarrierPaths = null
|
||
this.writeBarrierDeleteDirs = null
|
||
|
||
if (paths && paths.size > 0) {
|
||
// syncRawObjects fsyncs each file and its parent directory.
|
||
await this.syncRawObjects([...paths])
|
||
}
|
||
|
||
if (deleteDirs && deleteDirs.size > 0) {
|
||
for (const relDir of deleteDirs) {
|
||
const dirPath = path.join(this.rootDir, relDir)
|
||
let handle: any
|
||
try {
|
||
handle = await fs.promises.open(dirPath, 'r')
|
||
} catch {
|
||
continue // directory vanished or platform disallows opening dirs
|
||
}
|
||
try {
|
||
await handle.sync()
|
||
} catch {
|
||
// Some platforms/filesystems reject directory fsync — best effort.
|
||
} finally {
|
||
await handle.close()
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Append one line to `_system/tx-log.jsonl`. Plain `appendFile` — the
|
||
* tx-log is the one append-in-place file in the store (and is byte-copied,
|
||
* never hard-linked, by {@link FileSystemStorage.snapshotToDirectory}).
|
||
*
|
||
* @param line - One complete JSON document, without trailing newline.
|
||
*/
|
||
public async appendTxLogLine(line: string): Promise<void> {
|
||
await this.ensureInitialized()
|
||
const logPath = path.join(this.systemDir, 'tx-log.jsonl')
|
||
await fs.promises.appendFile(logPath, `${line}\n`, 'utf-8')
|
||
}
|
||
|
||
/**
|
||
* Read all tx-log lines, oldest first (empty array when no log exists).
|
||
* Torn trailing lines from a crashed append are returned as-is — callers
|
||
* tolerate unparseable lines.
|
||
*/
|
||
public async readTxLogLines(): Promise<string[]> {
|
||
await this.ensureInitialized()
|
||
const logPath = path.join(this.systemDir, 'tx-log.jsonl')
|
||
try {
|
||
const content: string = await fs.promises.readFile(logPath, 'utf-8')
|
||
return content.split('\n').filter((l: string) => l.length > 0)
|
||
} catch (error: any) {
|
||
if (error.code === 'ENOENT') return []
|
||
throw error
|
||
}
|
||
}
|
||
|
||
// ==========================================================================
|
||
// Binary raw-byte primitives — the substrate for append-only log-structured
|
||
// files (the generation fact log's CRC-framed segments). Paths are used
|
||
// VERBATIM (no .gz/.bin suffixing). Append durability rides syncRawObjects
|
||
// at the commit barrier, like every other staged write.
|
||
// ==========================================================================
|
||
|
||
/**
|
||
* Append bytes to a raw binary file, creating it (and parent directories)
|
||
* when absent. NOT fsync'd here — the caller batches durability via
|
||
* `syncRawObjects` at its commit barrier.
|
||
*/
|
||
public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
|
||
await this.ensureInitialized()
|
||
const fullPath = path.join(this.rootDir, rawPath)
|
||
await fs.promises.mkdir(path.dirname(fullPath), { recursive: true })
|
||
await fs.promises.appendFile(fullPath, bytes)
|
||
}
|
||
|
||
/**
|
||
* Read a raw binary file whole. Absent → `null`; a real IO fault throws —
|
||
* a present-but-unreadable log segment must never read as "no facts".
|
||
*/
|
||
public async readRawBytes(rawPath: string): Promise<Uint8Array | null> {
|
||
await this.ensureInitialized()
|
||
try {
|
||
const buf: Buffer = await fs.promises.readFile(path.join(this.rootDir, rawPath))
|
||
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
||
} catch (error: any) {
|
||
if (isAbsentError(error)) return null
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Replace a raw binary file atomically: write-new → fsync → rename. The
|
||
* reconcile primitive (e.g. truncating a fact-log tail back to committed
|
||
* truth after a crash) — a crash mid-replace leaves either the old file or
|
||
* the new one, never a mix.
|
||
*/
|
||
public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
|
||
await this.ensureInitialized()
|
||
const fullPath = path.join(this.rootDir, rawPath)
|
||
await fs.promises.mkdir(path.dirname(fullPath), { recursive: true })
|
||
const tmpPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`
|
||
const handle = await fs.promises.open(tmpPath, 'w')
|
||
try {
|
||
await handle.writeFile(bytes)
|
||
await handle.sync()
|
||
} finally {
|
||
await handle.close()
|
||
}
|
||
await fs.promises.rename(tmpPath, fullPath)
|
||
}
|
||
|
||
/**
|
||
* Byte size of a raw binary file, or `null` when absent.
|
||
*/
|
||
public async rawByteSize(rawPath: string): Promise<number | null> {
|
||
await this.ensureInitialized()
|
||
try {
|
||
const stat = await fs.promises.stat(path.join(this.rootDir, rawPath))
|
||
return stat.size
|
||
} catch (error: any) {
|
||
if (isAbsentError(error)) return null
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Snapshot the entire store into `targetPath` as a hard-link farm
|
||
* (Cassandra-style: instant, space-shared). Safe because every data file
|
||
* is immutable-by-rename — rewrites swap in new inodes, leaving the
|
||
* snapshot's links pointing at the old bytes. The two exceptions are
|
||
* handled explicitly: append-in-place files
|
||
* ({@link FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS}) are byte-copied, and
|
||
* process-local lock state
|
||
* ({@link FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS}) is excluded.
|
||
* Cross-device targets (where `link(2)` fails with `EXDEV`) fall back to
|
||
* byte copies per file.
|
||
*
|
||
* @param targetPath - Absolute directory for the snapshot. Created if
|
||
* missing; must be empty or absent (refuses to overwrite).
|
||
*/
|
||
public async snapshotToDirectory(targetPath: string): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
try {
|
||
const existing = await fs.promises.readdir(targetPath)
|
||
if (existing.length > 0) {
|
||
throw new Error(
|
||
`snapshotToDirectory: target ${targetPath} already exists and is not empty. ` +
|
||
`Choose a fresh directory per snapshot.`
|
||
)
|
||
}
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') throw error
|
||
}
|
||
await fs.promises.mkdir(targetPath, { recursive: true })
|
||
|
||
const files: string[] = []
|
||
await this.collectSnapshotFiles(this.rootDir, '', files)
|
||
|
||
for (const relPath of files) {
|
||
const sourceFile = path.join(this.rootDir, relPath)
|
||
const targetFile = path.join(targetPath, relPath)
|
||
await fs.promises.mkdir(path.dirname(targetFile), { recursive: true })
|
||
|
||
// Byte-copy list: compare against the normalized (extension-preserving)
|
||
// relative path with separators unified, so `_system/tx-log.jsonl`
|
||
// matches on every platform.
|
||
const normalized = relPath.split(path.sep).join('/')
|
||
// Byte-copy (never hard-link) files that are mutated in place: the exact
|
||
// append-in-place paths, and every file under a mmap-mutated directory
|
||
// (e.g. the native `_id_mapper/*`). A shared inode would let a live
|
||
// msync/truncate reach through into the snapshot.
|
||
if (
|
||
FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) ||
|
||
FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) ||
|
||
FileSystemStorage.SNAPSHOT_BYTE_COPY_PREFIXES.some((p) => normalized.startsWith(p))
|
||
) {
|
||
await fs.promises.copyFile(sourceFile, targetFile)
|
||
continue
|
||
}
|
||
|
||
try {
|
||
await fs.promises.link(sourceFile, targetFile)
|
||
} catch (error: any) {
|
||
// EXDEV: cross-device; EPERM/ENOTSUP: filesystem forbids links.
|
||
// ENOENT: the live file was atomically replaced mid-walk — retry as
|
||
// a copy of whatever is current (single-writer discipline means this
|
||
// only happens for derived files being flushed concurrently).
|
||
if (['EXDEV', 'EPERM', 'ENOTSUP', 'ENOENT'].includes(error.code)) {
|
||
try {
|
||
await fs.promises.copyFile(sourceFile, targetFile)
|
||
} catch (copyError: any) {
|
||
if (copyError.code !== 'ENOENT') throw copyError
|
||
}
|
||
} else {
|
||
throw error
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @description Pre-upgrade backup: a hard-link snapshot of the whole store into
|
||
* a SIBLING directory (`<rootDir>.migration-backup`, outside `rootDir` so the
|
||
* snapshot never recurses into itself), taken before a 7.x → 8.0 upgrade
|
||
* rebuilds the derived indexes. Zero-copy (shared inodes; the store is
|
||
* immutable-by-rename) and instant even at scale. Returns the backup path, or
|
||
* `null` when the store holds no canonical nouns (nothing to protect). If a
|
||
* backup already exists from a prior FAILED upgrade, it is REUSED as-is (that
|
||
* is the true pre-upgrade state; a retry must not overwrite it).
|
||
*/
|
||
public async createMigrationBackup(): Promise<string | null> {
|
||
await this.ensureInitialized()
|
||
|
||
// Nothing to protect if the store has no canonical nouns (e.g. a brand-new
|
||
// brain whose marker is simply absent — `epochStale` is true but there is no
|
||
// 7.x data to migrate).
|
||
const probe = await this.getNouns({ pagination: { limit: 1 } })
|
||
if ((probe.totalCount || 0) === 0 && probe.items.length === 0) {
|
||
return null
|
||
}
|
||
|
||
const backupPath = `${this.rootDir}.migration-backup`
|
||
|
||
// Reuse an existing backup (a prior failed upgrade's pre-state) rather than
|
||
// overwrite it — snapshotToDirectory also refuses a non-empty target.
|
||
try {
|
||
const existing = await fs.promises.readdir(backupPath)
|
||
if (existing.length > 0) return backupPath
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') throw error
|
||
}
|
||
|
||
await this.snapshotToDirectory(backupPath)
|
||
return backupPath
|
||
}
|
||
|
||
/**
|
||
* @description Remove a {@link createMigrationBackup} snapshot. Best-effort: a
|
||
* missing path is a no-op, and any error is swallowed (a leftover backup dir is
|
||
* harmless — the operator can delete it). Removing the hard-links never touches
|
||
* the live store's bytes (shared inodes; only the extra links go away).
|
||
*/
|
||
public async removeMigrationBackup(location: string): Promise<void> {
|
||
try {
|
||
await fs.promises.rm(location, { recursive: true, force: true })
|
||
} catch {
|
||
// best-effort — leaving the backup behind is safe
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recursively collect snapshot-eligible files under `dirAbs`, excluding the
|
||
* lock directory and in-flight `*.tmp.*` write files.
|
||
*/
|
||
private async collectSnapshotFiles(dirAbs: string, relPrefix: string, out: string[]): Promise<void> {
|
||
let entries: any[]
|
||
try {
|
||
entries = await fs.promises.readdir(dirAbs, { withFileTypes: true })
|
||
} catch (error: any) {
|
||
if (error.code === 'ENOENT') return
|
||
throw error
|
||
}
|
||
|
||
for (const entry of entries) {
|
||
const rel = relPrefix ? path.join(relPrefix, entry.name) : entry.name
|
||
if (entry.isDirectory()) {
|
||
if (relPrefix === '' && FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry.name)) {
|
||
continue
|
||
}
|
||
await this.collectSnapshotFiles(path.join(dirAbs, entry.name), rel, out)
|
||
} else if (entry.isFile()) {
|
||
if (entry.name.includes('.tmp.')) continue // in-flight atomic write
|
||
out.push(rel)
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Replace the store's contents from a snapshot directory: every current
|
||
* top-level entry except `locks/` (the live writer lock must survive) is
|
||
* removed, the snapshot is byte-copied in (`fs.cp` — never hard-linked, so
|
||
* the snapshot stays independent of the restored store), and all
|
||
* adapter-internal derived state is reloaded.
|
||
*
|
||
* @param sourcePath - Absolute path of a directory produced by
|
||
* {@link FileSystemStorage.snapshotToDirectory}.
|
||
*
|
||
* Non-destructive: the snapshot is copied into a staging area (sparse-aware,
|
||
* so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any
|
||
* live data is touched. Only once the full copy has succeeded and a completion
|
||
* marker is fsync'd does an atomic per-entry swap move it into place. A copy
|
||
* failure (including ENOSPC) leaves the live store exactly as it was; a crash
|
||
* mid-swap is resumed forward on the next {@link init} by
|
||
* {@link completeInterruptedRestore}. The previous implementation removed the
|
||
* live store first and then `fs.cp`'d — a copy failure destroyed the brain it
|
||
* was meant to recover.
|
||
*/
|
||
public async restoreFromDirectory(sourcePath: string): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
const sourceStat = await fs.promises.stat(sourcePath).catch(() => null)
|
||
if (!sourceStat || !sourceStat.isDirectory()) {
|
||
throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`)
|
||
}
|
||
|
||
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
|
||
|
||
// Discard any staging left by a prior aborted restore, then stage fresh.
|
||
await fs.promises.rm(staging, { recursive: true, force: true })
|
||
await fs.promises.mkdir(staging, { recursive: true })
|
||
|
||
// Copy the snapshot into staging (sparse-aware). ANY failure here — most
|
||
// importantly ENOSPC — leaves the live store untouched: we remove only the
|
||
// half-written staging area and re-throw.
|
||
const staged: string[] = []
|
||
try {
|
||
const sourceEntries = await fs.promises.readdir(sourcePath)
|
||
for (const entry of sourceEntries) {
|
||
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
|
||
await this.copyTreeSparse(
|
||
path.join(sourcePath, entry),
|
||
path.join(staging, entry)
|
||
)
|
||
staged.push(entry)
|
||
}
|
||
// Commit point: fsync a marker naming the fully-staged entries. Only after
|
||
// this does the swap (and its resume) become authorized.
|
||
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
|
||
await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged }))
|
||
const mfh = await fs.promises.open(markerPath, 'r')
|
||
try {
|
||
await mfh.sync()
|
||
} finally {
|
||
await mfh.close()
|
||
}
|
||
const dfh = await fs.promises.open(staging, 'r').catch(() => null)
|
||
if (dfh) {
|
||
try {
|
||
await dfh.sync()
|
||
} catch {
|
||
// platform may reject directory fsync — best effort
|
||
} finally {
|
||
await dfh.close()
|
||
}
|
||
}
|
||
} catch (error: any) {
|
||
await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
|
||
throw new Error(
|
||
`restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}`
|
||
)
|
||
}
|
||
|
||
// Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC).
|
||
await this.swapStagedRestoreIn()
|
||
await this.reloadDerivedState()
|
||
}
|
||
|
||
/**
|
||
* Move a fully-staged, marker-committed restore into place, then clear the
|
||
* staging area. Idempotent and resumable: driven by the marker's entry list
|
||
* and by which staged entries remain, so a crash at any point is completed by
|
||
* simply calling it again (from {@link completeInterruptedRestore} on the next
|
||
* open). Every step is a same-filesystem rename or a remove — no operation can
|
||
* fail for disk space, so once the marker exists the store is guaranteed to
|
||
* reach the restored state.
|
||
*/
|
||
private async swapStagedRestoreIn(): Promise<void> {
|
||
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
|
||
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
|
||
|
||
let staged: string[]
|
||
try {
|
||
const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8'))
|
||
staged = Array.isArray(marker?.entries) ? marker.entries : []
|
||
} catch {
|
||
return // no committed marker — nothing to swap
|
||
}
|
||
const stagedSet = new Set(staged)
|
||
|
||
// 1. Remove stale live entries the snapshot does not contain (excluding
|
||
// process-local dirs and the staging area itself). An already-placed
|
||
// staged entry is in stagedSet, so it is kept.
|
||
for (const entry of await fs.promises.readdir(this.rootDir)) {
|
||
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
|
||
if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue
|
||
if (stagedSet.has(entry)) continue
|
||
await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true })
|
||
}
|
||
|
||
// 2. Place each staged entry (idempotent: a prior attempt that already moved
|
||
// it leaves staging/entry absent, so we skip). `rm` the old first — a
|
||
// rename onto an existing non-empty directory is not allowed; the staged
|
||
// copy is the durable source until it is placed, so a crash between the
|
||
// rm and the rename is recovered forward on the next call.
|
||
for (const entry of staged) {
|
||
const from = path.join(staging, entry)
|
||
const to = path.join(this.rootDir, entry)
|
||
const exists = await fs.promises.lstat(from).then(() => true, () => false)
|
||
if (!exists) continue
|
||
await fs.promises.rm(to, { recursive: true, force: true })
|
||
await fs.promises.rename(from, to)
|
||
}
|
||
|
||
// 3. Clear the staging area (marker last-standing entry).
|
||
await fs.promises.rm(staging, { recursive: true, force: true })
|
||
}
|
||
|
||
/**
|
||
* On open, finish any restore interrupted by a crash. A staging dir WITH the
|
||
* completion marker means the copy had succeeded — resume the swap forward
|
||
* (loudly). A staging dir WITHOUT the marker is an interrupted copy — pure
|
||
* debris; the live store is authoritative, so discard it. Called from
|
||
* {@link init} before counts/derived state load, so recovery is invisible to
|
||
* the rest of startup. Returns `true` if a swap was resumed.
|
||
*/
|
||
private async completeInterruptedRestore(): Promise<boolean> {
|
||
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
|
||
const stagingStat = await fs.promises.stat(staging).catch(() => null)
|
||
if (!stagingStat || !stagingStat.isDirectory()) return false
|
||
|
||
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
|
||
const hasMarker = await fs.promises
|
||
.stat(markerPath)
|
||
.then(() => true, () => false)
|
||
|
||
if (!hasMarker) {
|
||
// Interrupted before the copy committed — live data untouched, discard.
|
||
await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
|
||
return false
|
||
}
|
||
|
||
console.log('♻️ Resuming an interrupted restore (completing the staged swap)')
|
||
await this.swapStagedRestoreIn()
|
||
return true
|
||
}
|
||
|
||
/**
|
||
* Recursively copy `src` to `dest`, preserving holes (sparse regions). Files
|
||
* are copied chunk-by-chunk skipping all-zero chunks, so a store of
|
||
* mostly-hole mmap blobs restores at its true allocated size instead of
|
||
* materializing every hole (the failure that made `fs.cp` ENOSPC a restore
|
||
* that would otherwise fit). Directories recurse; symlinks are recreated.
|
||
*/
|
||
private async copyTreeSparse(src: string, dest: string): Promise<void> {
|
||
const stat = await fs.promises.lstat(src)
|
||
if (stat.isDirectory()) {
|
||
await fs.promises.mkdir(dest, { recursive: true })
|
||
for (const child of await fs.promises.readdir(src)) {
|
||
await this.copyTreeSparse(path.join(src, child), path.join(dest, child))
|
||
}
|
||
} else if (stat.isSymbolicLink()) {
|
||
await fs.promises.symlink(await fs.promises.readlink(src), dest)
|
||
} else if (stat.isFile()) {
|
||
await this.copyFileSparse(src, dest, stat.size, stat.mode)
|
||
}
|
||
// Other node types (sockets, devices) do not occur in a brain store.
|
||
}
|
||
|
||
/** Sparse-aware single-file copy — see {@link copyTreeSparse}. */
|
||
private async copyFileSparse(
|
||
src: string,
|
||
dest: string,
|
||
size: number,
|
||
mode: number
|
||
): Promise<void> {
|
||
const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES
|
||
const srcFh = await fs.promises.open(src, 'r')
|
||
try {
|
||
const destFh = await fs.promises.open(dest, 'w', mode)
|
||
try {
|
||
// Pre-size the destination so unwritten regions are holes.
|
||
await destFh.truncate(size)
|
||
const buf = Buffer.allocUnsafe(CHUNK)
|
||
let pos = 0
|
||
while (pos < size) {
|
||
const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos)
|
||
if (bytesRead === 0) break
|
||
const chunk = buf.subarray(0, bytesRead)
|
||
// Skip all-zero chunks: leaving them unwritten preserves the hole.
|
||
const isHole = chunk.equals(
|
||
FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead)
|
||
)
|
||
if (!isHole) {
|
||
await destFh.write(chunk, 0, bytesRead, pos)
|
||
}
|
||
pos += bytesRead
|
||
}
|
||
} finally {
|
||
await destFh.close()
|
||
}
|
||
} finally {
|
||
await srcFh.close()
|
||
}
|
||
}
|
||
|
||
// ===========================================================================
|
||
// Raw binary-blob primitive (mmap-friendly)
|
||
// ===========================================================================
|
||
|
||
/**
|
||
* Resolve a blob key to its on-disk path under `<rootDir>/_blobs`.
|
||
*
|
||
* The key's "/"-separated segments become nested directories and the file is
|
||
* suffixed with `.bin`, e.g. `"graph-lsm/source/sstable-123"` →
|
||
* `<rootDir>/_blobs/graph-lsm/source/sstable-123.bin`. This convention is
|
||
* shared with cor's `MmapFileSystemStorage` so native code and the TS layer
|
||
* agree on exactly where each blob lives.
|
||
*
|
||
* @param key - The blob key.
|
||
* @returns The absolute on-disk path for the blob.
|
||
* @private
|
||
*/
|
||
private blobPath(key: string): string {
|
||
// `path` and `blobsDir` are populated in init(). If a caller resolves a blob
|
||
// path before init() (e.g. native code probing for a mmap target), fall back
|
||
// to a POSIX-style join off rootDir so this synchronous method never throws.
|
||
if (path && this.blobsDir) {
|
||
return path.join(this.blobsDir, ...key.split('/')) + '.bin'
|
||
}
|
||
return `${this.rootDir}/_blobs/${key}.bin`
|
||
}
|
||
|
||
/**
|
||
* Persist a raw binary blob verbatim under `key`, using an atomic
|
||
* temp-file + rename so concurrent readers never observe a torn write.
|
||
* Parent directories are created on demand.
|
||
*
|
||
* @param key - The blob key (see {@link getBinaryBlobPath} for the convention).
|
||
* @param data - The exact bytes to store.
|
||
*/
|
||
public async saveBinaryBlob(key: string, data: Buffer): Promise<void> {
|
||
await this.ensureInitialized()
|
||
const filePath = this.blobPath(key)
|
||
await this.ensureDirectoryExists(path.dirname(filePath))
|
||
|
||
// Atomic write via a UNIQUE per-writer temp suffix — matches the pattern
|
||
// used at every other atomic-write site in this file (lines 336, 551, 744,
|
||
// 781, 1529, 2908). The unique suffix (pid+time+random) means NO other
|
||
// writer ever touches this temp, so two concurrent saveBinaryBlob() calls
|
||
// for the same key can never collide on the temp path (the shared-`.tmp`
|
||
// race that once threw ENOENT — column-store compaction vs an explicit
|
||
// flush() on `_column_index/<field>/DELETED.bin` — is gone).
|
||
//
|
||
// Crucially, BECAUSE the temp is unique, a rename ENOENT can no longer mean
|
||
// "a concurrent idempotent writer already renamed it": nobody else has this
|
||
// temp. It means OUR just-written temp vanished before the rename, so the
|
||
// bytes did NOT land — returning success would acknowledge a write that
|
||
// stored nothing (the native provider mmaps these blobs; a phantom-acked
|
||
// blob is exactly the silent-loss class). The only way that happens with a
|
||
// unique temp is an external sweeper / crash-cleanup removing it mid-write,
|
||
// so retry ONCE with a fresh temp; if it vanishes again, FAIL LOUD.
|
||
const writeOnce = async (): Promise<'ok' | 'temp-vanished'> => {
|
||
const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`
|
||
await fs.promises.writeFile(tmpPath, data)
|
||
try {
|
||
await fs.promises.rename(tmpPath, filePath)
|
||
return 'ok'
|
||
} catch (err) {
|
||
// Clean up our own temp (best-effort) so no failure path orphans it.
|
||
await fs.promises.unlink(tmpPath).catch(() => {})
|
||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return 'temp-vanished'
|
||
throw err
|
||
}
|
||
}
|
||
|
||
if ((await writeOnce()) === 'temp-vanished' && (await writeOnce()) === 'temp-vanished') {
|
||
throw new Error(
|
||
`saveBinaryBlob('${key}'): the temp file was removed before rename on two ` +
|
||
`successive attempts — the blob did NOT persist. Some external process is ` +
|
||
`deleting files under ${this.blobsDir} mid-write. Failing loud rather than ` +
|
||
`acknowledging a durable write that stored nothing.`
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Load the raw bytes stored under `key`, or `null` if the blob does not exist.
|
||
*
|
||
* @param key - The blob key.
|
||
* @returns The blob bytes, or `null` if absent.
|
||
*/
|
||
public async loadBinaryBlob(key: string): Promise<Buffer | null> {
|
||
await this.ensureInitialized()
|
||
try {
|
||
return await fs.promises.readFile(this.blobPath(key))
|
||
} catch (err) {
|
||
// Absent blob → null (the documented contract). A real fault
|
||
// (EIO/EACCES/EMFILE/…) must NOT be masked as "absent": doing so makes a
|
||
// present-but-unreadable blob look missing and drives a needless rebuild
|
||
// or an empty read (the native provider consumes this). Mandate: loud
|
||
// errors, never quiet losses. Fault-propagation restored lockstep with
|
||
// cortex 3.0.13, whose two column-store call sites now handle the throw
|
||
// (they mark the field unavailable + throw a named error) instead of
|
||
// relying on null-on-error.
|
||
if (isAbsentError(err)) return null
|
||
throw err
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Delete the blob stored under `key`. Missing blobs are ignored.
|
||
*
|
||
* @param key - The blob key.
|
||
*/
|
||
public async deleteBinaryBlob(key: string): Promise<void> {
|
||
await this.ensureInitialized()
|
||
// Registered-blob contract: refuse to delete a declared
|
||
// derived-index family member — an in-process GC/sweeper cannot remove a
|
||
// load-bearing index file. Throws ProtectedArtifactError; no-op when no
|
||
// families are registered.
|
||
await this.assertBlobKeyDeletable(key)
|
||
try {
|
||
await fs.promises.unlink(this.blobPath(key))
|
||
} catch {
|
||
/* ignore missing files */
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Return the real on-disk path for `key` so native code can mmap the file
|
||
* directly. The path is returned whether or not the file currently exists —
|
||
* callers are expected to write before mapping.
|
||
*
|
||
* @param key - The blob key.
|
||
* @returns The absolute on-disk path (never `null` for filesystem storage).
|
||
*/
|
||
public getBinaryBlobPath(key: string): string | null {
|
||
return this.blobPath(key)
|
||
}
|
||
|
||
/**
|
||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||
* FileSystem implementation uses controlled concurrency to prevent too many file reads
|
||
*/
|
||
public async getMetadataBatch(ids: string[]): Promise<Map<string, any>> {
|
||
await this.ensureInitialized()
|
||
|
||
const results = new Map<string, any>()
|
||
const batchSize = 10 // Process 10 files at a time
|
||
|
||
// Process in batches to avoid overwhelming the filesystem
|
||
for (let i = 0; i < ids.length; i += batchSize) {
|
||
const batch = ids.slice(i, i + batchSize)
|
||
|
||
const batchPromises = batch.map(async (id) => {
|
||
try {
|
||
// CRITICAL: Use getNounMetadata() instead of deprecated getMetadata()
|
||
// This ensures we fetch from the correct noun metadata store (2-file system)
|
||
const metadata = await this.getNounMetadata(id)
|
||
return { id, metadata }
|
||
} catch (error) {
|
||
console.debug(`Failed to read metadata for ${id}:`, error)
|
||
return { id, metadata: null }
|
||
}
|
||
})
|
||
|
||
const batchResults = await Promise.all(batchPromises)
|
||
|
||
for (const { id, metadata } of batchResults) {
|
||
if (metadata !== null) {
|
||
results.set(id, metadata)
|
||
}
|
||
}
|
||
|
||
// Small yield between batches
|
||
await new Promise(resolve => setImmediate(resolve))
|
||
}
|
||
|
||
return results
|
||
}
|
||
|
||
/**
|
||
* Get nouns with pagination support
|
||
* @param options Pagination options
|
||
*/
|
||
// Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
|
||
|
||
/**
|
||
* Clear all data from storage
|
||
*/
|
||
public async clear(): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
// Check if fs module is available
|
||
if (!fs || !fs.promises) {
|
||
console.warn('FileSystemStorage.clear: fs module not available, skipping clear operation')
|
||
return
|
||
}
|
||
|
||
// Helper function to remove all files in a directory
|
||
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
||
try {
|
||
const files = await fs.promises.readdir(dirPath)
|
||
for (const file of files) {
|
||
const filePath = path.join(dirPath, file)
|
||
const stats = await fs.promises.stat(filePath)
|
||
if (stats.isDirectory()) {
|
||
await removeDirectoryContents(filePath)
|
||
await fs.promises.rmdir(filePath)
|
||
} else {
|
||
await fs.promises.unlink(filePath)
|
||
}
|
||
}
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.error(`Error removing directory contents ${dirPath}:`, error)
|
||
throw error
|
||
}
|
||
}
|
||
}
|
||
|
||
// Clear the canonical entity/verb data area
|
||
const entitiesDir = path.join(this.rootDir, 'entities')
|
||
if (await this.directoryExists(entitiesDir)) {
|
||
await removeDirectoryContents(entitiesDir)
|
||
}
|
||
|
||
// Remove all files in both system directories
|
||
await removeDirectoryContents(this.systemDir)
|
||
if (await this.directoryExists(this.indexDir)) {
|
||
await removeDirectoryContents(this.indexDir)
|
||
}
|
||
|
||
// Remove the content-addressed blob store (VFS file content)
|
||
const casDir = path.join(this.rootDir, '_cas')
|
||
if (await this.directoryExists(casDir)) {
|
||
// Delete the entire _cas/ directory (not just contents)
|
||
await fs.promises.rm(casDir, { recursive: true, force: true })
|
||
}
|
||
|
||
// Remove the raw-blob + native-shared + column-index footprint. These
|
||
// top-level trees were NOT wiped before, so a cleared brain re-read stale
|
||
// native blobs (HNSW/LSM segments, the native dkann index), a stale native
|
||
// id-mapper, and — worst — orphaned column manifests. `_column_index` holds
|
||
// the column-store MANIFEST.json files while their segment bytes live under
|
||
// `_blobs/_column_index/...`; removing one without the other would strand a
|
||
// manifest listing segments that no longer exist, which the load path now
|
||
// (correctly) refuses with ColumnSegmentLoadError. They must fall together,
|
||
// as a set, exactly as `_cas` does — the complete derived footprint, not a
|
||
// subset. (`locks/` is deliberately left: it is live coordination state,
|
||
// not data.)
|
||
for (const nativeDir of ['_blobs', '_id_mapper', '_column_index']) {
|
||
const dir = path.join(this.rootDir, nativeDir)
|
||
if (await this.directoryExists(dir)) {
|
||
await fs.promises.rm(dir, { recursive: true, force: true })
|
||
}
|
||
}
|
||
|
||
// Reset ALL shared derived state via the same path restore uses: the
|
||
// write-through cache, id→type/subtype caches, per-type/subtype count
|
||
// rollups, statistics cache, graph-index singleton, and the BlobStorage
|
||
// instance (its LRU cache holds deleted blobs), then recount from the
|
||
// now-empty data area. Without the rollup reset, stats() would keep
|
||
// reporting per-type counts for deleted entities.
|
||
await this.reloadDerivedState()
|
||
}
|
||
|
||
/**
|
||
* Enhanced clear operation with safety mechanisms and performance optimizations
|
||
* Provides progress tracking, backup options, and instance name confirmation
|
||
*/
|
||
public async clearEnhanced(options: import('../enhancedClearOperations.js').ClearOptions = {}): Promise<import('../enhancedClearOperations.js').ClearResult> {
|
||
await this.ensureInitialized()
|
||
|
||
// Check if fs module is available
|
||
if (!fs || !fs.promises) {
|
||
throw new Error('FileSystemStorage.clearEnhanced: fs module not available')
|
||
}
|
||
|
||
const { EnhancedFileSystemClear } = await import('../enhancedClearOperations.js')
|
||
const enhancedClear = new EnhancedFileSystemClear(this.rootDir, fs, path)
|
||
|
||
const result = await enhancedClear.clear(options)
|
||
|
||
if (result.success) {
|
||
// Clear the statistics cache
|
||
this.statisticsCache = null
|
||
this.statisticsModified = false
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
/**
|
||
* Get information about storage usage and capacity
|
||
*/
|
||
public async getStorageStatus(): Promise<{
|
||
type: string
|
||
used: number
|
||
quota: number | null
|
||
details?: Record<string, any>
|
||
}> {
|
||
await this.ensureInitialized()
|
||
|
||
// Check if fs module is available
|
||
if (!fs || !fs.promises) {
|
||
console.warn('FileSystemStorage.getStorageStatus: fs module not available, returning default values')
|
||
return {
|
||
type: 'filesystem',
|
||
used: 0,
|
||
quota: null,
|
||
details: {
|
||
nounsCount: 0,
|
||
verbsCount: 0,
|
||
metadataCount: 0,
|
||
directorySizes: {
|
||
nouns: 0,
|
||
verbs: 0,
|
||
metadata: 0,
|
||
index: 0
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
try {
|
||
// Calculate the total size of all files in the storage directories
|
||
let totalSize = 0
|
||
|
||
// Helper function to calculate directory size
|
||
const calculateSize = async (dirPath: string): Promise<number> => {
|
||
let size = 0
|
||
try {
|
||
const files = await fs.promises.readdir(dirPath)
|
||
for (const file of files) {
|
||
const filePath = path.join(dirPath, file)
|
||
const stats = await fs.promises.stat(filePath)
|
||
if (stats.isDirectory()) {
|
||
size += await calculateSize(filePath)
|
||
} else {
|
||
size += stats.size
|
||
}
|
||
}
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.error(
|
||
`Error calculating size for directory ${dirPath}:`,
|
||
error
|
||
)
|
||
}
|
||
}
|
||
return size
|
||
}
|
||
|
||
// Calculate size for each directory
|
||
const nounsDirSize = await calculateSize(this.nounsDir)
|
||
const verbsDirSize = await calculateSize(this.verbsDir)
|
||
const metadataDirSize = await calculateSize(this.metadataDir)
|
||
const indexDirSize = await calculateSize(this.indexDir)
|
||
|
||
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
||
|
||
// CRITICAL FIX: Use persisted counts instead of directory reads
|
||
// This is O(1) instead of O(n), and handles sharded structure correctly
|
||
const nounsCount = this.totalNounCount
|
||
const verbsCount = this.totalVerbCount
|
||
|
||
// Count metadata files (these are NOT sharded)
|
||
const metadataCount = (
|
||
await fs.promises.readdir(this.metadataDir)
|
||
).filter((file: string) => file.endsWith('.json')).length
|
||
|
||
// Use persisted entity counts by type (O(1) instead of scanning all files)
|
||
const nounTypeCounts: Record<string, number> = Object.fromEntries(this.entityCounts)
|
||
|
||
// Skip the expensive metadata file scan since we have counts
|
||
const metadataFiles: string[] = [] // Empty array to skip the loop below
|
||
for (const file of metadataFiles) {
|
||
if (file.endsWith('.json')) {
|
||
try {
|
||
const filePath = path.join(this.metadataDir, file)
|
||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||
const metadata = JSON.parse(data)
|
||
if (metadata.noun) {
|
||
nounTypeCounts[metadata.noun] =
|
||
(nounTypeCounts[metadata.noun] || 0) + 1
|
||
}
|
||
} catch (error) {
|
||
console.error(`Error reading metadata file ${file}:`, error)
|
||
}
|
||
}
|
||
}
|
||
|
||
return {
|
||
type: 'filesystem',
|
||
used: totalSize,
|
||
quota: null, // File system doesn't provide quota information
|
||
details: {
|
||
rootDirectory: this.rootDir,
|
||
nounsCount,
|
||
verbsCount,
|
||
metadataCount,
|
||
nounsDirSize,
|
||
verbsDirSize,
|
||
metadataDirSize,
|
||
indexDirSize,
|
||
nounTypes: nounTypeCounts
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.error('Failed to get storage status:', error)
|
||
return {
|
||
type: 'filesystem',
|
||
used: 0,
|
||
quota: null,
|
||
details: { error: String(error) }
|
||
}
|
||
}
|
||
}
|
||
|
||
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
||
// Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
|
||
|
||
public override supportsMultiProcessLocking(): boolean {
|
||
return true
|
||
}
|
||
|
||
/**
|
||
* Acquire the process-level writer lock for this data directory. Called by
|
||
* `Brainy.init()` in writer mode. Refuses to open if another live writer
|
||
* holds the lock — the worst possible default is to "succeed" with stale
|
||
* indexes and silently lie about query results.
|
||
*
|
||
* Lock file: `<rootDir>/locks/_writer.lock`.
|
||
*
|
||
* Stale-lock detection: a lock is considered stale when ALL of:
|
||
* 1. Same hostname as the current process (cross-host PID checks are unsafe).
|
||
* 2. The recorded PID is no longer alive (`process.kill(pid, 0)` → ESRCH), OR
|
||
* `lastHeartbeat` is older than `WRITER_STALE_THRESHOLD_MS` (60s).
|
||
* Stale locks are overwritten with a warning. `force: true` overrides even live locks.
|
||
*/
|
||
public override async acquireWriterLock(options?: { force?: boolean }): Promise<WriterLockInfo | null> {
|
||
await this.ensureInitialized()
|
||
await this.ensureDirectoryExists(this.lockDir)
|
||
|
||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||
const os = await import('node:os')
|
||
const hostname = os.hostname()
|
||
const myPid = typeof process !== 'undefined' && process.pid ? process.pid : 0
|
||
|
||
// Bounded acquire loop. The CLAIM itself is an atomic create-exclusive
|
||
// write (O_EXCL) — two processes racing an ABSENT lock can never both
|
||
// succeed, which closes the read-then-write window where the loser used
|
||
// to keep running unlocked, silently. An EEXIST loser loops, re-reads,
|
||
// and handles whatever it finds honestly (fresh foreign lock → loud
|
||
// throw; stale/forced → verified takeover).
|
||
const MAX_ATTEMPTS = 3
|
||
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
||
const now = new Date().toISOString()
|
||
const existing = await this.readWriterLock()
|
||
|
||
// TORN-LOCK RECOVERY: power loss can legally leave the lock file
|
||
// present but EMPTY/unparseable (the claim's non-atomic write died
|
||
// mid-flight). readWriterLock() reports it as null — but the O_EXCL
|
||
// claim below would EEXIST forever, a PERMANENT lockout no staleness
|
||
// check can clear (staleness needs a parsed PID). A torn lock is
|
||
// stale BY DEFINITION: no live holder has one (a holder either
|
||
// completed its write or is dead). Unlink loudly and re-loop; a
|
||
// racer that rewrites a VALID lock first simply wins the next read.
|
||
if (existing === null) {
|
||
try {
|
||
await fs.promises.access(lockFile)
|
||
console.warn(
|
||
`[brainy] Writer lock at ${lockFile} exists but is unreadable/unparseable ` +
|
||
`(torn write from a previous power loss) — treating as stale and removing.`
|
||
)
|
||
try {
|
||
await fs.promises.unlink(lockFile)
|
||
} catch (unlinkErr: any) {
|
||
if (unlinkErr.code !== 'ENOENT') throw unlinkErr
|
||
}
|
||
} catch (accessErr: any) {
|
||
if (accessErr.code !== 'ENOENT') throw accessErr
|
||
// Absent: the normal fresh-claim path below.
|
||
}
|
||
}
|
||
|
||
// THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see
|
||
// WriterCloseRecord). A lock file whose release was RECORDED is
|
||
// bookkeeping left by an orderly shutdown, not evidence of anything —
|
||
// and that is true whether the previous holder was another process or
|
||
// an earlier instance in THIS one. A production restart reported
|
||
// "Re-acquiring writer lock ... this is a bug" immediately after a clean
|
||
// close, sending an operator hunting for a leak that did not exist.
|
||
const closeRecord = existing ? await this.readWriterCloseRecord() : null
|
||
const releasedCleanly =
|
||
existing !== null &&
|
||
closeRecord !== null &&
|
||
this.closeRecordVouchesFor(closeRecord, existing)
|
||
|
||
if (existing) {
|
||
// Same-process re-open: a second Brainy instance in this Node process
|
||
// (e.g. test "simulate server restart" patterns, or a consumer that
|
||
// explicitly re-instantiates without closing first). This isn't the
|
||
// dangerous cross-process case the lock exists to prevent — the two
|
||
// instances share a memory space and can't silently diverge from each
|
||
// other beyond what their callers already see. Warn and take over —
|
||
// unless the record proves the previous instance already let go, in
|
||
// which case there is nothing to warn about.
|
||
if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
|
||
if (releasedCleanly) {
|
||
console.warn(
|
||
`[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` +
|
||
`instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` +
|
||
`${closeRecord!.closedAt} but could not remove the file. Nothing to recover.`
|
||
)
|
||
} else {
|
||
console.warn(
|
||
`[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
|
||
`If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
|
||
)
|
||
}
|
||
const info: WriterLockInfo = {
|
||
pid: myPid,
|
||
hostname,
|
||
startedAt: now,
|
||
lastHeartbeat: now,
|
||
version: getBrainyVersion(),
|
||
rootDir: this.rootDir
|
||
}
|
||
await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
|
||
await this.clearWriterCloseRecord()
|
||
this.installWriterLock(info)
|
||
return info
|
||
}
|
||
|
||
// A cleanly-released lock is stale by RECORD, not by inference. Only
|
||
// when no record vouches for this lock do we fall back to pid
|
||
// liveness, and then we say THAT honestly too: an unrecorded lock
|
||
// means the writer did not complete its close, so the store was not
|
||
// closed cleanly and this open pays recovery.
|
||
const stale =
|
||
releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing)))
|
||
if (!options?.force && !stale) {
|
||
// Consumer-facing error contract: callers detect this case via
|
||
// err.code and read the holder's details from err.lockInfo.
|
||
throw this.writerLockedError(existing)
|
||
}
|
||
|
||
console.warn(
|
||
options?.force
|
||
? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
|
||
`(was held by PID ${existing.pid} on ${existing.hostname}).`
|
||
: releasedCleanly
|
||
? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` +
|
||
`PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` +
|
||
`${closeRecord!.closedAt} but could not remove the file. ` +
|
||
`Nothing to recover.`
|
||
: `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
|
||
`(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` +
|
||
`clean-close record — that writer did not finish closing, so this ` +
|
||
`store was not closed cleanly; open will run crash recovery and ` +
|
||
`report its wall).`
|
||
)
|
||
// Takeover: verify the file still holds the lock we judged (a live
|
||
// successor may have claimed meanwhile), then remove it and fall
|
||
// through to the atomic claim below. A racing claimer who beats us to
|
||
// the create simply wins — our next loop iteration reads their fresh
|
||
// lock and throws honestly. (Advisory file locking has no
|
||
// compare-and-delete; staleness requiring a 60s-old heartbeat keeps
|
||
// the residual verify-to-unlink window practically unreachable.)
|
||
const recheck = await this.readWriterLock()
|
||
if (
|
||
recheck &&
|
||
(recheck.pid !== existing.pid ||
|
||
recheck.startedAt !== existing.startedAt ||
|
||
recheck.lastHeartbeat !== existing.lastHeartbeat)
|
||
) {
|
||
continue // the lock changed hands while we deliberated — re-evaluate
|
||
}
|
||
try {
|
||
await fs.promises.unlink(lockFile)
|
||
} catch (err: any) {
|
||
if (err.code !== 'ENOENT') throw err
|
||
}
|
||
}
|
||
|
||
const info: WriterLockInfo = {
|
||
pid: myPid,
|
||
hostname,
|
||
startedAt: existing && options?.force ? existing.startedAt : now,
|
||
lastHeartbeat: now,
|
||
version: getBrainyVersion(),
|
||
rootDir: this.rootDir
|
||
}
|
||
|
||
// The atomic claim: write the FULL contents to a temp file, then
|
||
// hard-link it into place — link(2) fails EEXIST if the target exists,
|
||
// and the lock file appears with its complete JSON in one atomic step.
|
||
// (The previous claim was writeFile with O_EXCL, whose open→write→close
|
||
// is NOT atomic: a concurrent opener could read the file in its empty
|
||
// window, judge it torn, unlink a LIVE claim, and take the lock — two
|
||
// live writers. The link claim leaves no empty window to misread.)
|
||
const claimTmp = `${lockFile}.claim-${myPid}-${Date.now()}`
|
||
try {
|
||
await fs.promises.writeFile(claimTmp, JSON.stringify(info, null, 2))
|
||
await fs.promises.link(claimTmp, lockFile)
|
||
} catch (err: any) {
|
||
if (err.code === 'EEXIST') {
|
||
continue // someone else claimed between our read and create — re-evaluate
|
||
}
|
||
throw err
|
||
} finally {
|
||
await fs.promises.unlink(claimTmp).catch(() => {})
|
||
}
|
||
|
||
// CONSUME the previous writer's clean-close record. It described the
|
||
// lock generation that just ended; leaving it in place would let it
|
||
// vouch for OUR lock if this process later dies without closing —
|
||
// turning a real crash into a "closed cleanly" verdict. One unlink.
|
||
await this.clearWriterCloseRecord()
|
||
|
||
this.installWriterLock(info)
|
||
return info
|
||
}
|
||
|
||
// Attempts exhausted: something is claiming this directory faster than we
|
||
// can evaluate it. Read whoever holds it now and fail loudly with their
|
||
// details rather than degrading into a lockless open.
|
||
const holder = await this.readWriterLock()
|
||
if (holder) throw this.writerLockedError(holder)
|
||
throw new Error(
|
||
`Failed to acquire the writer lock for ${this.rootDir} after ${MAX_ATTEMPTS} attempts — ` +
|
||
`the lock file is being contended. Retry, or inspect ${lockFile}.`
|
||
)
|
||
}
|
||
|
||
/** Record lock ownership + start the unref'd heartbeat. */
|
||
private installWriterLock(info: WriterLockInfo): void {
|
||
this.writerLockInfo = info
|
||
|
||
// Heartbeat — rewrite lastHeartbeat every WRITER_HEARTBEAT_MS so other
|
||
// processes can tell a live writer from one that crashed without releasing.
|
||
this.writerLockHeartbeat = setInterval(() => {
|
||
const tick = this.refreshWriterLockHeartbeat().catch((err) => {
|
||
// ENOENT = the lock (or its directory) vanished mid-refresh — the
|
||
// store was released or removed under us; the next acquire recreates
|
||
// it. Benign by construction; anything else stays loud.
|
||
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
|
||
console.warn('[brainy] Failed to refresh writer lock heartbeat:', err)
|
||
}
|
||
})
|
||
this.writerHeartbeatInFlight = tick.finally(() => {
|
||
if (this.writerHeartbeatInFlight === tick) {
|
||
this.writerHeartbeatInFlight = undefined
|
||
}
|
||
})
|
||
}, FileSystemStorage.WRITER_HEARTBEAT_MS)
|
||
if (typeof this.writerLockHeartbeat.unref === 'function') {
|
||
// Don't keep the event loop alive just for the heartbeat.
|
||
this.writerLockHeartbeat.unref()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* THE FENCE: verify this instance still owns the writer lock before a
|
||
* commit barrier proceeds. An evicted writer (an operator's
|
||
* `{ force: true }` takeover, or an operator deleting the lock file) must
|
||
* fail LOUDLY on its next flush instead of writing on unaware — the
|
||
* unfenced evicted writer was half of a production split-brain (each
|
||
* writer flushing its own internally-consistent id-mapper snapshot,
|
||
* alternating the store between two truths). One small file read per
|
||
* flush window, never per record. No-op when this instance holds no
|
||
* writer lock (read-only opens, in-memory stores).
|
||
*
|
||
* @throws `BRAINY_WRITER_FENCED` when the lock is gone or held by another.
|
||
*/
|
||
public override async assertWriterFenceHeld(): Promise<void> {
|
||
if (!this.writerLockInfo) return
|
||
const current = await this.readWriterLock()
|
||
// Ownership is PER-PROCESS: pid + hostname, deliberately NOT startedAt.
|
||
// The documented same-process re-open path ("warn and take over" — two
|
||
// instances in one Node process, the server-restart test pattern)
|
||
// rewrites the lock with a fresh startedAt; fencing the first instance
|
||
// on that mismatch latched its background flushes dead while its own
|
||
// process held the lock (caught by the plant's integration lane, twice).
|
||
// startedAt adds nothing against pid recycling either: a recycled pid's
|
||
// victim is a DEAD process — it runs no fence checks.
|
||
if (
|
||
current &&
|
||
current.pid === this.writerLockInfo.pid &&
|
||
current.hostname === this.writerLockInfo.hostname
|
||
) {
|
||
return
|
||
}
|
||
const err = new Error(
|
||
`Writer fence lost for ${this.rootDir}: this process (PID ${this.writerLockInfo.pid}) ` +
|
||
`no longer holds the writer lock — ` +
|
||
(current
|
||
? `it is now held by PID ${current.pid} on ${current.hostname} (since ${current.startedAt}).`
|
||
: `the lock file is gone (released or removed by an operator).`) +
|
||
`\nThis instance refuses to commit further writes: a fenced-out writer continuing to ` +
|
||
`flush is how split-brain stores are made. Close this instance; if the takeover was a ` +
|
||
`mistake, close the successor and re-open.`
|
||
) as Error & { code: string }
|
||
err.code = 'BRAINY_WRITER_FENCED'
|
||
throw err
|
||
}
|
||
|
||
/** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */
|
||
private writerLockedError(existing: WriterLockInfo): Error {
|
||
const err = new Error(
|
||
`Another writer holds this Brainy directory.\n` +
|
||
` PID: ${existing.pid} on host ${existing.hostname}\n` +
|
||
` Started: ${existing.startedAt}\n` +
|
||
` Heartbeat: ${existing.lastHeartbeat}\n` +
|
||
` Version: ${existing.version}\n` +
|
||
` Directory: ${this.rootDir}\n\n` +
|
||
`For diagnostic queries against this live store, use:\n` +
|
||
` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` +
|
||
`If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.`
|
||
) as Error & { code: string; lockInfo: WriterLockInfo }
|
||
err.code = 'BRAINY_WRITER_LOCKED'
|
||
err.lockInfo = existing
|
||
return err
|
||
}
|
||
|
||
public override async releaseWriterLock(): Promise<void> {
|
||
if (this.writerLockHeartbeat) {
|
||
clearInterval(this.writerLockHeartbeat)
|
||
this.writerLockHeartbeat = undefined
|
||
}
|
||
// Drain an in-flight heartbeat tick BEFORE unlinking: clearInterval stops
|
||
// future ticks only, and a straggler write landing after the unlink would
|
||
// re-create the lock as a phantom (blocking the next writer until the
|
||
// stale TTL). After the drain, any refresh is either fully landed (we
|
||
// unlink its output below) or not started (it sees writerLockInfo
|
||
// undefined and returns).
|
||
if (this.writerHeartbeatInFlight) {
|
||
await this.writerHeartbeatInFlight
|
||
}
|
||
if (!this.writerLockInfo) {
|
||
return
|
||
}
|
||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||
const released = this.writerLockInfo
|
||
try {
|
||
// Only delete if we still own it — avoid clobbering a successor that
|
||
// claimed the lock via force-override.
|
||
const current = await this.readWriterLock()
|
||
const ours =
|
||
current === null ||
|
||
(current.pid === released.pid && current.hostname === released.hostname)
|
||
if (current && ours) {
|
||
await fs.promises.unlink(lockFile)
|
||
}
|
||
// THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this
|
||
// instance gives up a lock nobody else has taken — the unlink above
|
||
// having succeeded OR the file already being gone. The next open reads
|
||
// it instead of guessing from pid liveness: a recorded release is an
|
||
// orderly shutdown, an absent record is a writer that never finished
|
||
// closing. Not written when a successor holds the lock: our release is
|
||
// then a no-op and a record would slander their live lock.
|
||
if (ours) {
|
||
await this.writeWriterCloseRecord(released)
|
||
}
|
||
} catch (err: any) {
|
||
if (err.code !== 'ENOENT') {
|
||
console.warn('[brainy] Failed to release writer lock file:', err)
|
||
}
|
||
} finally {
|
||
this.writerLockInfo = undefined
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @description Read the clean-close record at `locks/_writer.close`, or
|
||
* `null` when it is absent or unparseable. A torn record is treated as
|
||
* absent — the conservative direction, since an unreadable record can
|
||
* vouch for nothing.
|
||
* @returns The record, or null.
|
||
*/
|
||
public async readWriterCloseRecord(): Promise<WriterCloseRecord | null> {
|
||
await this.ensureInitialized()
|
||
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||
try {
|
||
const raw = await fs.promises.readFile(recordFile, 'utf-8')
|
||
const parsed = JSON.parse(raw) as WriterCloseRecord
|
||
if (
|
||
typeof parsed?.pid !== 'number' ||
|
||
typeof parsed?.hostname !== 'string' ||
|
||
typeof parsed?.startedAt !== 'string' ||
|
||
typeof parsed?.closedAt !== 'string'
|
||
) {
|
||
return null
|
||
}
|
||
return parsed
|
||
} catch (err: any) {
|
||
if (err.code === 'ENOENT') return null
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @description Whether a clean-close record describes the very lock
|
||
* generation `lock` represents. The match is pid + hostname + `startedAt`:
|
||
* `startedAt` is the lock generation's identity, so a record can never
|
||
* vouch for a LATER lock taken by the same pid on the same host (the
|
||
* same-process re-open path mints a fresh `startedAt`).
|
||
* @param record - The clean-close record read from disk.
|
||
* @param lock - The lock file's contents.
|
||
*/
|
||
private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean {
|
||
return (
|
||
record.pid === lock.pid &&
|
||
record.hostname === lock.hostname &&
|
||
record.startedAt === lock.startedAt
|
||
)
|
||
}
|
||
|
||
/**
|
||
* @description Write the clean-close record for a lock this instance just
|
||
* released. Atomic (temp + rename) so a concurrent opener never reads half
|
||
* a record. A failure here costs the next open nothing but the honest
|
||
* fallback (pid liveness), so it warns rather than failing the close.
|
||
* @param released - The lock info this instance held.
|
||
*/
|
||
private async writeWriterCloseRecord(released: WriterLockInfo): Promise<void> {
|
||
const record: WriterCloseRecord = {
|
||
pid: released.pid,
|
||
hostname: released.hostname,
|
||
startedAt: released.startedAt,
|
||
closedAt: new Date().toISOString(),
|
||
version: released.version
|
||
}
|
||
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||
try {
|
||
await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2))
|
||
} catch (err) {
|
||
// ENOENT = the lock directory is gone, i.e. the whole store was removed
|
||
// under us. There is no next open to inform.
|
||
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return
|
||
console.warn(
|
||
`[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` +
|
||
`the next open will fall back to pid liveness and may report this orderly ` +
|
||
`shutdown as a crash:`,
|
||
err
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @description Remove the clean-close record. Called by every successful
|
||
* lock claim so a record never outlives the lock generation it describes.
|
||
*/
|
||
private async clearWriterCloseRecord(): Promise<void> {
|
||
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||
try {
|
||
await fs.promises.unlink(recordFile)
|
||
} catch (err: any) {
|
||
if (err.code !== 'ENOENT') {
|
||
console.warn('[brainy] Failed to clear the writer clean-close record:', err)
|
||
}
|
||
}
|
||
}
|
||
|
||
public override async readWriterLock(): Promise<WriterLockInfo | null> {
|
||
await this.ensureInitialized()
|
||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||
try {
|
||
const raw = await fs.promises.readFile(lockFile, 'utf-8')
|
||
return JSON.parse(raw) as WriterLockInfo
|
||
} catch (err: any) {
|
||
if (err.code === 'ENOENT') return null
|
||
console.warn('[brainy] Failed to read writer lock file:', err)
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Atomically refresh `lastHeartbeat` on the writer lock we own. Skips if the
|
||
* lock file has been deleted out from under us (e.g. operator removed it).
|
||
*/
|
||
private async refreshWriterLockHeartbeat(): Promise<void> {
|
||
if (!this.writerLockInfo) return
|
||
const current = await this.readWriterLock()
|
||
if (!current) return
|
||
// Defensive: don't overwrite if a successor claimed the lock.
|
||
if (current.pid !== this.writerLockInfo.pid || current.hostname !== this.writerLockInfo.hostname) {
|
||
return
|
||
}
|
||
const updated: WriterLockInfo = {
|
||
...this.writerLockInfo,
|
||
lastHeartbeat: new Date().toISOString()
|
||
}
|
||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||
await this.writeFileAtomic(lockFile, JSON.stringify(updated, null, 2))
|
||
this.writerLockInfo = updated
|
||
}
|
||
|
||
/**
|
||
* Determine whether an existing writer lock is stale (safe to overwrite).
|
||
* Same hostname and DEAD PID → stale. That is the whole rule: a LIVE
|
||
* process is never auto-evicted, however old its heartbeat — a >60s
|
||
* event-loop stall (debugger pause, GC, heavy sync work) is a slow writer,
|
||
* not a dead one, and heartbeat-age eviction of live writers was the
|
||
* dominant mechanism behind a production split-brain (two live unaware
|
||
* writers alternating a store's id-mapper between two truths). A holder
|
||
* that LOOKS alive but is truly wedged is the operator's call via
|
||
* `{ force: true }` — and the fence check on every flush
|
||
* ({@link assertWriterFenceHeld}) guarantees a forced-out holder fails
|
||
* loudly instead of writing on. Different hostname → cannot prove
|
||
* anything, treat as live. The heartbeat remains for OBSERVABILITY (the
|
||
* lock error names it so an operator can judge staleness themselves).
|
||
*/
|
||
private async isWriterLockStale(lock: WriterLockInfo): Promise<boolean> {
|
||
const os = await import('node:os')
|
||
if (lock.hostname !== os.hostname()) {
|
||
return false
|
||
}
|
||
return !this.isPidAlive(lock.pid)
|
||
}
|
||
|
||
/**
|
||
* `process.kill(pid, 0)` sends signal 0 — no signal is actually delivered;
|
||
* it just checks whether the kernel still considers `pid` reachable from
|
||
* this process. ESRCH = no such process. EPERM = exists but we can't signal
|
||
* it, which still proves it's alive.
|
||
*/
|
||
private isPidAlive(pid: number): boolean {
|
||
if (!pid || pid <= 0) return false
|
||
try {
|
||
process.kill(pid, 0)
|
||
return true
|
||
} catch (err: any) {
|
||
if (err.code === 'EPERM') return true
|
||
return false
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
||
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
||
*/
|
||
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
||
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`
|
||
await fs.promises.writeFile(tmp, contents)
|
||
await fs.promises.rename(tmp, filePath)
|
||
}
|
||
|
||
/**
|
||
* Start watching for cross-process flush requests. Called by Brainy.init()
|
||
* in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers
|
||
* the supplied callback (`brain.flush()`), after which an `.ack` is written
|
||
* to `locks/_flush_responses/` with the same request ID. Stale `.req` files
|
||
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep.
|
||
*
|
||
* THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request
|
||
* directory every 500 ms, per brain, for the entire life of every writer —
|
||
* armed on every non-reader brain whether or not any inspector process
|
||
* existed. MEASURED on a production process holding 21 brains: 42 directory
|
||
* reads per second on a completely idle service, plus a stale-request GC
|
||
* pass on every one of them. The engine does no periodic work without a
|
||
* cause, and a request that has not been made is not a cause.
|
||
*
|
||
* `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is
|
||
* seen SOONER than the old poll saw it. Two honest concessions ride with it:
|
||
* - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because
|
||
* `fs.watch` can miss events on network and fuse filesystems and because
|
||
* the stale-request GC needs some tick of its own. At 30s that is 0.7
|
||
* reads/s across 21 brains where the poll cost 42.
|
||
* - a filesystem that cannot watch at all falls back to the ORIGINAL
|
||
* 500 ms poll, narrated once, because correctness outranks idle cost:
|
||
* an inspector whose request is never seen waits forever.
|
||
*/
|
||
public override startFlushRequestWatcher(onRequest: () => Promise<void>): void {
|
||
// Already watching — or already ARMING. The arm is asynchronous (the
|
||
// request directory is created before it can be watched), so neither the
|
||
// watcher nor the interval exists yet during that window; the callback is
|
||
// the flag that covers it. Without this a second call in the window would
|
||
// leave two watchers and two sweeps running for the life of the store.
|
||
if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return
|
||
this.flushWatcherOnRequest = onRequest
|
||
|
||
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
|
||
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
|
||
|
||
const sweep = (): void => {
|
||
if (this.flushWatcherInFlight) return // skip overlapping sweep
|
||
this.flushWatcherInFlight = true
|
||
this.processFlushRequests(reqDir, ackDir).finally(() => {
|
||
this.flushWatcherInFlight = false
|
||
})
|
||
}
|
||
|
||
// Ensure both dirs exist up front so the first .req drop doesn't race with
|
||
// mkdir — and so there is a directory to watch.
|
||
void this.ensureDirectoryExists(reqDir)
|
||
.then(() => this.ensureDirectoryExists(ackDir))
|
||
.then(() => {
|
||
if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile
|
||
try {
|
||
const watcher = fs.watch(reqDir, () => sweep())
|
||
this.flushWatcher = watcher
|
||
watcher.on('error', (err: Error) => {
|
||
// A watch that dies mid-life must not leave the door deaf.
|
||
console.warn(
|
||
`[brainy] Flush-request watch failed (${err.message}) — falling back to polling.`
|
||
)
|
||
this.flushWatcher?.close()
|
||
this.flushWatcher = undefined
|
||
this.startFlushRequestPolling(sweep)
|
||
})
|
||
if (typeof watcher.unref === 'function') watcher.unref()
|
||
// The safety sweep: missed events on exotic filesystems, and the
|
||
// stale-request GC.
|
||
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS)
|
||
if (typeof this.flushWatcherInterval.unref === 'function') {
|
||
this.flushWatcherInterval.unref()
|
||
}
|
||
// One sweep now: a request may have been dropped before the watch armed.
|
||
sweep()
|
||
} catch (err) {
|
||
console.warn(
|
||
`[brainy] Flush-request directory cannot be watched on this filesystem ` +
|
||
`(${(err as Error).message}) — polling every ` +
|
||
`${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.`
|
||
)
|
||
this.startFlushRequestPolling(sweep)
|
||
}
|
||
})
|
||
.catch(() => {
|
||
// The request directory could not be created; nothing to watch. A
|
||
// cross-process flush request cannot be made either, so there is
|
||
// nothing to miss.
|
||
})
|
||
}
|
||
|
||
/** The original 500 ms poll — the fallback when a directory cannot be watched. */
|
||
private startFlushRequestPolling(sweep: () => void): void {
|
||
if (this.flushWatcherInterval) return
|
||
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
|
||
if (typeof this.flushWatcherInterval.unref === 'function') {
|
||
this.flushWatcherInterval.unref()
|
||
}
|
||
}
|
||
|
||
public override stopFlushRequestWatcher(): void {
|
||
if (this.flushWatcher) {
|
||
this.flushWatcher.close()
|
||
this.flushWatcher = undefined
|
||
}
|
||
if (this.flushWatcherInterval) {
|
||
clearInterval(this.flushWatcherInterval)
|
||
this.flushWatcherInterval = undefined
|
||
}
|
||
this.flushWatcherOnRequest = undefined
|
||
}
|
||
|
||
/**
|
||
* Process any pending `.req` files: invoke the flush callback once, then
|
||
* write an `.ack` for each request. Multiple requests that arrived in the
|
||
* same tick share a single flush — they all see the same ack timestamp.
|
||
*/
|
||
private async processFlushRequests(reqDir: string, ackDir: string): Promise<void> {
|
||
let entries: string[]
|
||
try {
|
||
entries = await fs.promises.readdir(reqDir)
|
||
} catch (err: any) {
|
||
if (err.code !== 'ENOENT') {
|
||
console.warn('[brainy] Flush watcher readdir failed:', err)
|
||
}
|
||
return
|
||
}
|
||
|
||
const reqs = entries.filter((e: string) => e.endsWith('.req'))
|
||
if (reqs.length === 0) return
|
||
|
||
// One flush per tick — N pending requests share it.
|
||
const callback = this.flushWatcherOnRequest
|
||
if (!callback) return
|
||
let flushError: Error | null = null
|
||
try {
|
||
await callback()
|
||
} catch (err: any) {
|
||
flushError = err instanceof Error ? err : new Error(String(err))
|
||
console.warn('[brainy] Flush callback threw inside flush-request watcher:', err)
|
||
}
|
||
|
||
const ackTimestamp = new Date().toISOString()
|
||
for (const filename of reqs) {
|
||
const requestId = filename.replace(/\.req$/, '')
|
||
const ackPath = path.join(ackDir, `${requestId}.ack`)
|
||
const ackBody = JSON.stringify({
|
||
requestId,
|
||
completedAt: ackTimestamp,
|
||
ok: !flushError,
|
||
error: flushError ? flushError.message : undefined
|
||
})
|
||
try {
|
||
await this.writeFileAtomic(ackPath, ackBody)
|
||
await fs.promises.unlink(path.join(reqDir, filename))
|
||
} catch (err: any) {
|
||
if (err.code !== 'ENOENT') {
|
||
console.warn('[brainy] Failed to write flush ack:', err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Garbage-collect stale .req files in case a watcher missed them (e.g.
|
||
// the writer was restarted between request and processing).
|
||
const now = Date.now()
|
||
for (const filename of entries) {
|
||
if (!filename.endsWith('.req')) continue
|
||
const fp = path.join(reqDir, filename)
|
||
try {
|
||
const stat = await fs.promises.stat(fp)
|
||
if (now - stat.mtimeMs > FileSystemStorage.FLUSH_REQUEST_TTL_MS) {
|
||
await fs.promises.unlink(fp).catch(() => {})
|
||
}
|
||
} catch {
|
||
// ignore — file already removed
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Inspector side: drop a `.req` file and poll for the corresponding `.ack`.
|
||
* Returns true if the writer acknowledged in time, false on timeout.
|
||
*/
|
||
public override async requestFlushOverFilesystem(timeoutMs: number): Promise<boolean> {
|
||
await this.ensureInitialized()
|
||
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
|
||
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
|
||
await this.ensureDirectoryExists(reqDir)
|
||
await this.ensureDirectoryExists(ackDir)
|
||
|
||
const requestId = `${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2, 10)}`
|
||
const reqPath = path.join(reqDir, `${requestId}.req`)
|
||
const ackPath = path.join(ackDir, `${requestId}.ack`)
|
||
const body = JSON.stringify({
|
||
requestId,
|
||
requestedAt: new Date().toISOString(),
|
||
requesterPid: process.pid
|
||
})
|
||
|
||
await this.writeFileAtomic(reqPath, body)
|
||
|
||
const deadline = Date.now() + timeoutMs
|
||
while (Date.now() < deadline) {
|
||
try {
|
||
await fs.promises.access(ackPath, fs.constants.F_OK)
|
||
await fs.promises.unlink(ackPath).catch(() => {})
|
||
return true
|
||
} catch {
|
||
// not yet
|
||
}
|
||
await new Promise((r) => setTimeout(r, FileSystemStorage.FLUSH_POLL_INTERVAL_MS))
|
||
}
|
||
|
||
// Timeout — best effort to clean up our request so the writer doesn't act
|
||
// on stale work later. Watcher will GC anyway after FLUSH_REQUEST_TTL_MS.
|
||
await fs.promises.unlink(reqPath).catch(() => {})
|
||
return false
|
||
}
|
||
|
||
/**
|
||
* Acquire a file-based lock for coordinating operations across multiple processes
|
||
* @param lockKey The key to lock on
|
||
* @param ttl Time to live for the lock in milliseconds (default: 30 seconds)
|
||
* @returns Promise that resolves to true if lock was acquired, false otherwise
|
||
*/
|
||
private async acquireLock(
|
||
lockKey: string,
|
||
ttl: number = 30000
|
||
): Promise<boolean> {
|
||
await this.ensureInitialized()
|
||
|
||
// Ensure lock directory exists
|
||
await this.ensureDirectoryExists(this.lockDir)
|
||
|
||
const lockFile = path.join(this.lockDir, `${lockKey}.lock`)
|
||
const lockValue = `${Date.now()}_${Math.random()}_${process.pid || 'unknown'}`
|
||
const expiresAt = Date.now() + ttl
|
||
|
||
try {
|
||
// Check if lock file already exists and is still valid
|
||
try {
|
||
const lockData = await fs.promises.readFile(lockFile, 'utf-8')
|
||
const lockInfo = JSON.parse(lockData)
|
||
|
||
if (lockInfo.expiresAt > Date.now()) {
|
||
// Lock exists and is still valid
|
||
return false
|
||
}
|
||
} catch (error: any) {
|
||
// If file doesn't exist or can't be read, we can proceed to create the lock
|
||
if (error.code !== 'ENOENT') {
|
||
console.warn(`Error reading lock file ${lockFile}:`, error)
|
||
}
|
||
}
|
||
|
||
// Try to create the lock file
|
||
const lockInfo = {
|
||
lockValue,
|
||
expiresAt,
|
||
pid: process.pid || 'unknown',
|
||
timestamp: Date.now()
|
||
}
|
||
|
||
await fs.promises.writeFile(lockFile, JSON.stringify(lockInfo, null, 2))
|
||
|
||
// Add to active locks for cleanup
|
||
this.activeLocks.add(lockKey)
|
||
|
||
// Schedule automatic cleanup when lock expires
|
||
setTimeout(() => {
|
||
this.releaseLock(lockKey, lockValue).catch((error) => {
|
||
console.warn(`Failed to auto-release expired lock ${lockKey}:`, error)
|
||
})
|
||
}, ttl)
|
||
|
||
return true
|
||
} catch (error) {
|
||
console.warn(`Failed to acquire lock ${lockKey}:`, error)
|
||
return false
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Release a file-based lock
|
||
* @param lockKey The key to unlock
|
||
* @param lockValue The value used when acquiring the lock (for verification)
|
||
* @returns Promise that resolves when lock is released
|
||
*/
|
||
private async releaseLock(
|
||
lockKey: string,
|
||
lockValue?: string
|
||
): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
const lockFile = path.join(this.lockDir, `${lockKey}.lock`)
|
||
|
||
try {
|
||
// If lockValue is provided, verify it matches before releasing
|
||
if (lockValue) {
|
||
try {
|
||
const lockData = await fs.promises.readFile(lockFile, 'utf-8')
|
||
const lockInfo = JSON.parse(lockData)
|
||
|
||
if (lockInfo.lockValue !== lockValue) {
|
||
// Lock was acquired by someone else, don't release it
|
||
return
|
||
}
|
||
} catch (error: any) {
|
||
// If lock file doesn't exist, that's fine
|
||
if (error.code === 'ENOENT') {
|
||
return
|
||
}
|
||
throw error
|
||
}
|
||
}
|
||
|
||
// Delete the lock file
|
||
await fs.promises.unlink(lockFile)
|
||
|
||
// Remove from active locks
|
||
this.activeLocks.delete(lockKey)
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.warn(`Failed to release lock ${lockKey}:`, error)
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clean up expired lock files
|
||
*/
|
||
private async cleanupExpiredLocks(): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
try {
|
||
const lockFiles = await fs.promises.readdir(this.lockDir)
|
||
const now = Date.now()
|
||
|
||
for (const lockFile of lockFiles) {
|
||
if (!lockFile.endsWith('.lock')) continue
|
||
|
||
const lockPath = path.join(this.lockDir, lockFile)
|
||
try {
|
||
const lockData = await fs.promises.readFile(lockPath, 'utf-8')
|
||
const lockInfo = JSON.parse(lockData)
|
||
|
||
if (lockInfo.expiresAt <= now) {
|
||
await fs.promises.unlink(lockPath)
|
||
const lockKey = lockFile.replace('.lock', '')
|
||
this.activeLocks.delete(lockKey)
|
||
}
|
||
} catch (error) {
|
||
// If we can't read or parse the lock file, remove it
|
||
try {
|
||
await fs.promises.unlink(lockPath)
|
||
} catch (unlinkError) {
|
||
console.warn(
|
||
`Failed to cleanup invalid lock file ${lockPath}:`,
|
||
unlinkError
|
||
)
|
||
}
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.warn('Failed to cleanup expired locks:', error)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Save statistics data to storage with file-based locking
|
||
*/
|
||
protected async saveStatisticsData(
|
||
statistics: StatisticsData
|
||
): Promise<void> {
|
||
const lockKey = 'statistics'
|
||
const lockAcquired = await this.acquireLock(lockKey, 10000) // 10 second timeout
|
||
|
||
if (!lockAcquired) {
|
||
console.warn(
|
||
'Failed to acquire lock for statistics update, proceeding without lock'
|
||
)
|
||
}
|
||
|
||
try {
|
||
// Get existing statistics to merge with new data
|
||
const existingStats = await this.getStatisticsData()
|
||
|
||
if (existingStats) {
|
||
// Merge statistics data
|
||
const mergedStats: StatisticsData = {
|
||
totalNodes: Math.max(
|
||
statistics.totalNodes || 0,
|
||
existingStats.totalNodes || 0
|
||
),
|
||
totalEdges: Math.max(
|
||
statistics.totalEdges || 0,
|
||
existingStats.totalEdges || 0
|
||
),
|
||
totalMetadata: Math.max(
|
||
statistics.totalMetadata || 0,
|
||
existingStats.totalMetadata || 0
|
||
),
|
||
// Preserve any additional fields from existing stats
|
||
...existingStats,
|
||
// Override with new values where provided
|
||
...statistics,
|
||
// Always update lastUpdated to current time
|
||
lastUpdated: new Date().toISOString()
|
||
}
|
||
await this.saveStatisticsWithBackwardCompat(mergedStats)
|
||
} else {
|
||
// No existing statistics, save new ones
|
||
const newStats: StatisticsData = {
|
||
...statistics,
|
||
lastUpdated: new Date().toISOString()
|
||
}
|
||
await this.saveStatisticsWithBackwardCompat(newStats)
|
||
}
|
||
} finally {
|
||
if (lockAcquired) {
|
||
await this.releaseLock(lockKey)
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get statistics data from storage
|
||
*/
|
||
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
||
try {
|
||
const statsPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
|
||
const data = await fs.promises.readFile(statsPath, 'utf-8')
|
||
return JSON.parse(data)
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.error('Error reading statistics:', error)
|
||
}
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Save statistics to storage
|
||
*/
|
||
private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise<void> {
|
||
const statsPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
|
||
await this.ensureDirectoryExists(this.systemDir)
|
||
await fs.promises.writeFile(statsPath, JSON.stringify(statistics, null, 2))
|
||
}
|
||
|
||
// =============================================
|
||
// Count Management for O(1) Scalability
|
||
// =============================================
|
||
|
||
/**
|
||
* Initialize counts from filesystem storage
|
||
*/
|
||
protected async initializeCounts(): Promise<void> {
|
||
if (!this.countsFilePath) return
|
||
|
||
try {
|
||
if (await this.fileExists(this.countsFilePath)) {
|
||
const data = await fs.promises.readFile(this.countsFilePath, 'utf-8')
|
||
const counts = JSON.parse(data)
|
||
|
||
// Restore entity counts
|
||
this.entityCounts = new Map(Object.entries(counts.entityCounts || {}))
|
||
this.verbCounts = new Map(Object.entries(counts.verbCounts || {}))
|
||
this.totalNounCount = counts.totalNounCount || 0
|
||
this.totalVerbCount = counts.totalVerbCount || 0
|
||
|
||
// The ALL-visibility scalars (ledger denominators). A counts.json
|
||
// written before they existed carries neither key: derive both ONCE
|
||
// from the canonical id tree (an id-directory listing — O(ids), no
|
||
// record reads), persist, and never scan again. Absent keys are a
|
||
// legacy file, not a zero — a zero here would make every provider's
|
||
// coverage ledger read "over-posted" on a populated store.
|
||
let needsPersist = false
|
||
if (
|
||
typeof counts.totalNounCountAll === 'number' &&
|
||
typeof counts.totalVerbCountAll === 'number'
|
||
) {
|
||
this.totalNounCountAll = counts.totalNounCountAll
|
||
this.totalVerbCountAll = counts.totalVerbCountAll
|
||
if (counts.allCountsDerivedBy === 'identity-record') {
|
||
// Derived (or recounted) under the honest rule — one counted
|
||
// entity per metadata content leg. Trust the persisted suspect
|
||
// flag as-is; an unprovable delete since may still have set it.
|
||
this.allCountsDerivedBy = 'identity-record'
|
||
this.allCountsSuspect = counts.allCountsSuspect === true
|
||
} else {
|
||
// The ALL scalars exist but predate the identity-record stamp —
|
||
// they were derived under the legacy rule that counted one
|
||
// entity per id DIRECTORY, so orphaned ghost/scar containers (a
|
||
// pre-8.3.1 partial-delete defect — see pruneOrphanedEntities())
|
||
// were counted as entities too. O(1) field read, NEVER a walk
|
||
// here: force suspect and name it loudly. A sanctioned recount
|
||
// (repairIndex) restores exact denominators and clears this.
|
||
this.allCountsDerivedBy = undefined
|
||
this.allCountsSuspect = true
|
||
needsPersist = true
|
||
prodLog.narrate(
|
||
'[FileSystemStorage] canonical count ledger was derived under the legacy ' +
|
||
'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' +
|
||
'container inflates it. Marked suspect, and an honest recount is scheduled to ' +
|
||
'run in the background after this open; until it lands, do not subtract ' +
|
||
'against these ALL scalars.'
|
||
)
|
||
// A suspect ledger used to stay wrong for the life of the store,
|
||
// waiting for an operator to run repairIndex. A downstream index
|
||
// heal took its "remaining" figure from these inflated
|
||
// denominators and reported work that did not exist. The ledger
|
||
// now HEALS ITSELF — in the background, because a denominator is
|
||
// a derived scalar and no read is ever served from it.
|
||
this.scheduleCountLedgerDerivation('legacy container-rule ledger')
|
||
}
|
||
} else {
|
||
// No ALL scalars at all. There is nothing to serve in the meantime —
|
||
// a zero would read as an empty store — so the scalars stay unknown
|
||
// and SUSPECT until the background derivation lands. The open does
|
||
// not wait for it: an id-tree walk is O(ids) and this file has been
|
||
// the whole reason a 24k-id store opened in silence.
|
||
this.allCountsSuspect = true
|
||
this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger')
|
||
}
|
||
|
||
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
||
// counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but
|
||
// still predate THIS key). Unlike the ALL scalars, presence cannot be
|
||
// decided from the id-directory listing alone: a deferred-embed
|
||
// noun's `vectors.json` EXISTS with an empty `vector: []` until its
|
||
// embed lands, so this derivation reads every noun's `vectors.json`
|
||
// ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost.
|
||
if (typeof counts.totalVectoredNounCount === 'number') {
|
||
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
||
} else {
|
||
// O(nouns) CONTENT reads — the most expensive derivation of the
|
||
// three, and the one most likely to have been the silent minutes at
|
||
// the front of a large store's open. Background, suspect until it
|
||
// lands, same as the ALL scalars.
|
||
this.allCountsSuspect = true
|
||
this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger')
|
||
}
|
||
if (needsPersist) {
|
||
await this.persistCounts()
|
||
}
|
||
|
||
// Also populate the cache for backward compatibility
|
||
this.countCache.set('nouns_count', {
|
||
count: this.totalNounCount,
|
||
timestamp: Date.now()
|
||
})
|
||
this.countCache.set('verbs_count', {
|
||
count: this.totalVerbCount,
|
||
timestamp: Date.now()
|
||
})
|
||
} else {
|
||
// If no counts file exists, do one initial count
|
||
await this.initializeCountsFromDisk()
|
||
}
|
||
} catch (error) {
|
||
console.warn('Could not load persisted counts, will initialize from disk:', error)
|
||
await this.initializeCountsFromDisk()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Initialize counts by scanning disk (only done once)
|
||
*/
|
||
private async initializeCountsFromDisk(): Promise<void> {
|
||
const startedAt = Date.now()
|
||
// THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating:
|
||
// it derives `totalNounCount` / `totalVerbCount`, the scalars
|
||
// `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would
|
||
// make a populated store answer "0 entities" until the walk landed — a
|
||
// wrong answer, not a slow one, and the serving law grades a failure by
|
||
// whether an answer could be wrong. The ALL-visibility denominators, which
|
||
// no read is served from, DO run in the background (see
|
||
// scheduleCountLedgerDerivation). What this walk owes the operator instead
|
||
// is narration: it announces itself, and reports its wall.
|
||
prodLog.narrate(
|
||
`[FileSystemStorage] no usable counts.json — deriving the entity counters from ` +
|
||
`the canonical id tree now. This is O(ids) listings plus one vectors.json read ` +
|
||
`per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` +
|
||
`served from it. It runs once; the result is persisted.`
|
||
)
|
||
try {
|
||
// Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) —
|
||
// the tree saveNoun/getNouns actually read and write. The previous scan
|
||
// counted the vestigial `entities/*/hnsw` directories, which the 8.0
|
||
// write path never populates, so a store recovering from a lost or
|
||
// corrupted counts.json re-initialized every counter to ZERO on real
|
||
// data (wrong stats/boot logs and a mis-sized rebuild-strategy
|
||
// decision at open).
|
||
const nouns = await this.scanCanonicalEntities('nouns')
|
||
this.totalNounCount = nouns.count
|
||
const verbs = await this.scanCanonicalEntities('verbs')
|
||
this.totalVerbCount = verbs.count
|
||
// The id-tree scan counts every tier — it IS the ALL-visibility ledger.
|
||
this.totalNounCountAll = nouns.count
|
||
this.totalVerbCountAll = verbs.count
|
||
this.allCountsSuspect = false
|
||
this.allCountsDerivedBy = 'identity-record'
|
||
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
|
||
// (a deferred-embed noun's file exists but holds an empty vector until
|
||
// its embed lands), so this is a full O(nouns) content scan — see
|
||
// scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here,
|
||
// alongside the rest of this from-disk recovery.
|
||
this.totalVectoredNounCount = await this.scanVectoredNounCount()
|
||
|
||
// Sample some entities for the type distribution (don't read all).
|
||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||
// and every guarded accessor (getNounMetadata → ensureInitialized)
|
||
// re-enters init() from here, deadlocking the open. (Latent in the old
|
||
// code too: its scan of the empty hnsw dirs just never sampled.)
|
||
for (const entityDir of nouns.sampleDirs) {
|
||
const metadata = await this.readEntityMetadataRaw(entityDir)
|
||
if (metadata) {
|
||
const type = metadata.noun || 'default'
|
||
this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1)
|
||
}
|
||
}
|
||
|
||
// Extrapolate counts if we sampled
|
||
const sampleSize = nouns.sampleDirs.length
|
||
if (sampleSize < this.totalNounCount && sampleSize > 0) {
|
||
const multiplier = this.totalNounCount / sampleSize
|
||
for (const [type, count] of this.entityCounts.entries()) {
|
||
this.entityCounts.set(type, Math.round(count * multiplier))
|
||
}
|
||
}
|
||
|
||
await this.persistCounts()
|
||
prodLog.narrate(
|
||
`[FileSystemStorage] counter derivation from the canonical id tree finished in ` +
|
||
`${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` +
|
||
`${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.`
|
||
)
|
||
} catch (error) {
|
||
console.error('Error initializing counts from disk:', error)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting
|
||
* one entity per id directory that holds the metadata CONTENT leg
|
||
* (`metadata.json` or its `.json.gz` variant — see
|
||
* {@link hasMetadataContentLeg}). A bare container — a ghost (a stale
|
||
* `vectors.json` left with no metadata leg) or a scar (an empty directory),
|
||
* both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the
|
||
* identity record IS the population (ADR-008 G1), never the directory.
|
||
* This is the ONE-TIME legacy derivation walk (see callers); a prior
|
||
* version of this scan counted every id directory regardless of content,
|
||
* over-counting any store carrying orphaned containers — see
|
||
* `allCountsDerivedBy` for how a counts.json derived under that old rule is
|
||
* marked suspect on load. Returns up to 100 sampled *counted* entity
|
||
* directories (absolute paths) — nouns feed the type-distribution estimate
|
||
* above. An absent tree (fresh store) counts zero.
|
||
*/
|
||
/**
|
||
* @description Derive the ALL-visibility count ledger honestly — one entity
|
||
* per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once,
|
||
* and persist the result stamped `identity-record`.
|
||
*
|
||
* Why background: these scalars are DENOMINATORS. No read is served from
|
||
* them, so deriving them cannot be allowed to hold an open hostage — a
|
||
* store with 24,898 ids spent minutes of a production restart inside walks
|
||
* exactly like these, in silence, before serving anything. Why at all: a
|
||
* ledger derived under the old container rule stayed wrong for the life of
|
||
* the store, and a downstream index heal subtracted against it and reported
|
||
* remaining work that did not exist (measured on a real store: 14,231
|
||
* derived against 14,056 identity records — precisely the store's 25 noun
|
||
* scar directories; verbs 72,729 against 72,679, its 50 verb scars).
|
||
*
|
||
* Idempotent: a second call while one is in flight joins the first.
|
||
* @param reason - What made the ledger untrustworthy, quoted in narration.
|
||
* @returns Nothing; observe completion with {@link whenCountLedgerSettled}.
|
||
*/
|
||
private scheduleCountLedgerDerivation(reason: string): void {
|
||
if (this.countLedgerDerivation) return
|
||
this.countLedgerDerivation = (async () => {
|
||
const startedAt = Date.now()
|
||
prodLog.narrate(
|
||
`[FileSystemStorage] count-ledger derivation started in the background ` +
|
||
`(${reason}) — counting identity records, not id directories; the open does ` +
|
||
`not wait for it and no read is served from these scalars.`
|
||
)
|
||
try {
|
||
const beforeNouns = this.totalNounCountAll
|
||
const beforeVerbs = this.totalVerbCountAll
|
||
const beforeVectored = this.totalVectoredNounCount
|
||
// A walk that RACED A WRITE cannot prove its number: a row that landed
|
||
// mid-walk may or may not have been in the shard the walk had already
|
||
// passed. Rather than persist a figure that might be off by one and
|
||
// stamp it "exact", the walk is repeated once on a quiet store, and if
|
||
// the store is never quiet the ledger stays SUSPECT and says so. One
|
||
// retry, never a spin.
|
||
let attempt = 0
|
||
let derived: { nouns: number; verbs: number; vectored: number } | null = null
|
||
while (attempt < 2 && derived === null) {
|
||
attempt++
|
||
const activityBefore = this.ledgerActivityStamp()
|
||
const nouns = await this.scanCanonicalEntities('nouns')
|
||
const verbs = await this.scanCanonicalEntities('verbs')
|
||
const vectored = await this.scanVectoredNounCount()
|
||
if (this.ledgerActivityStamp() === activityBefore) {
|
||
derived = { nouns: nouns.count, verbs: verbs.count, vectored }
|
||
}
|
||
}
|
||
if (derived === null) {
|
||
this.allCountsSuspect = true
|
||
prodLog.narrate(
|
||
`[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` +
|
||
`after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` +
|
||
`every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` +
|
||
`against; brain.repairIndex() derives them under a recount barrier.`
|
||
)
|
||
return
|
||
}
|
||
this.totalNounCountAll = derived.nouns
|
||
this.totalVerbCountAll = derived.verbs
|
||
this.totalVectoredNounCount = derived.vectored
|
||
this.allCountsDerivedBy = 'identity-record'
|
||
this.allCountsSuspect = false
|
||
await this.persistCounts()
|
||
prodLog.narrate(
|
||
`[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` +
|
||
`${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` +
|
||
(beforeNouns !== derived.nouns ||
|
||
beforeVerbs !== derived.verbs ||
|
||
beforeVectored !== derived.vectored
|
||
? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` +
|
||
`difference is ghost and scar containers the old rule counted as entities)`
|
||
: ' (unchanged)') +
|
||
` — persisted, stamped identity-record, no longer suspect.`
|
||
)
|
||
} catch (error) {
|
||
// The ledger stays suspect and the next open retries. Loud: a
|
||
// denominator nobody can derive is a fact an operator must have.
|
||
this.allCountsSuspect = true
|
||
prodLog.error(
|
||
`[FileSystemStorage] count-ledger derivation FAILED after ` +
|
||
`${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` +
|
||
`and must not be subtracted against; the next open retries:`,
|
||
error
|
||
)
|
||
}
|
||
})()
|
||
}
|
||
|
||
/**
|
||
* @description A cheap witness that the ledger changed while a walk was
|
||
* running. Every landed write moves one of these live counters, so an
|
||
* unchanged stamp across a walk means no write landed during it.
|
||
* @returns A value that differs whenever the live ALL scalars have moved.
|
||
*/
|
||
private ledgerActivityStamp(): string {
|
||
return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}`
|
||
}
|
||
|
||
/**
|
||
* @description Resolve once any background count-ledger derivation has
|
||
* settled (succeeded or failed). Resolves immediately when none was needed.
|
||
* Exists so tests and operators can observe the ledger's honest value rather
|
||
* than race it; nothing in the read path waits on this.
|
||
* @returns A promise that settles with the derivation.
|
||
*/
|
||
public async whenCountLedgerSettled(): Promise<void> {
|
||
await this.countLedgerDerivation
|
||
}
|
||
|
||
private async scanCanonicalEntities(
|
||
kind: 'nouns' | 'verbs'
|
||
): Promise<{ count: number; sampleDirs: string[] }> {
|
||
const base = path.join(this.rootDir, 'entities', kind)
|
||
const SAMPLE_MAX = 100
|
||
let count = 0
|
||
const sampleDirs: string[] = []
|
||
try {
|
||
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
||
for (const shard of shards) {
|
||
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
||
const shardPath = path.join(base, shard.name)
|
||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||
for (const entry of ids) {
|
||
if (!entry.isDirectory()) continue
|
||
const idAbs = path.join(shardPath, entry.name)
|
||
let legs: string[]
|
||
try {
|
||
legs = await fs.promises.readdir(idAbs)
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') continue
|
||
throw error
|
||
}
|
||
// No metadata content leg → a ghost or scar container → not an
|
||
// entity. Same test pruneOrphanedEntities() uses, so the two agree
|
||
// by construction.
|
||
if (!this.hasMetadataContentLeg(legs)) continue
|
||
count++
|
||
if (sampleDirs.length < SAMPLE_MAX) {
|
||
sampleDirs.push(idAbs)
|
||
}
|
||
}
|
||
}
|
||
} catch (error: any) {
|
||
if (error?.code !== 'ENOENT') throw error
|
||
}
|
||
return { count, sampleDirs }
|
||
}
|
||
|
||
/**
|
||
* Read one canonical entity's `metadata.json` (or `.json.gz`) directly with
|
||
* fs — NO guarded accessors. Used only by the init-time count recovery,
|
||
* where `getNounMetadata`'s `ensureInitialized()` would re-enter `init()`.
|
||
* @param entityDir - Absolute `entities/<kind>/<shard>/<id>` directory.
|
||
* @returns The parsed metadata, or null when absent/unreadable.
|
||
*/
|
||
private async readEntityMetadataRaw(entityDir: string): Promise<any | null> {
|
||
const base = path.join(entityDir, 'metadata.json')
|
||
try {
|
||
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
||
} catch {
|
||
// fall through to the compressed variant
|
||
}
|
||
try {
|
||
const gz = await fs.promises.readFile(`${base}.gz`)
|
||
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs
|
||
* — the vector-side mirror of {@link readEntityMetadataRaw}, same
|
||
* reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`).
|
||
* @param entityDir - Absolute `entities/nouns/<shard>/<id>` directory.
|
||
* @returns The parsed vector record, or null when absent/unreadable.
|
||
*/
|
||
private async readEntityVectorRaw(entityDir: string): Promise<any | null> {
|
||
const base = path.join(entityDir, 'vectors.json')
|
||
try {
|
||
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
||
} catch {
|
||
// fall through to the compressed variant
|
||
}
|
||
try {
|
||
const gz = await fs.promises.readFile(`${base}.gz`)
|
||
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector —
|
||
* the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities},
|
||
* presence cannot be decided from the id-directory listing alone: a
|
||
* deferred-embed noun's `vectors.json` EXISTS (written at `add()` time
|
||
* with `vector: []`) until its embed LANDS, so this walk reads every
|
||
* noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing.
|
||
* ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts
|
||
* here either (Brainy's write paths normalize an explicit zero-norm
|
||
* vector to `[]` at write time, but a store created before that fix may
|
||
* still carry legacy all-zero rows on disk; this derivation must agree
|
||
* with the live ledger's definition of "vectored" regardless of when the
|
||
* row was written). Used ONLY for a one-time legacy-counts.json derivation
|
||
* or a lost/corrupted counts.json recovery; the result is persisted so
|
||
* this scan never repeats.
|
||
*/
|
||
private async scanVectoredNounCount(): Promise<number> {
|
||
const base = path.join(this.rootDir, 'entities', 'nouns')
|
||
let vectored = 0
|
||
try {
|
||
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
||
for (const shard of shards) {
|
||
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
||
const shardPath = path.join(base, shard.name)
|
||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||
for (const entry of ids) {
|
||
if (!entry.isDirectory()) continue
|
||
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
|
||
if (
|
||
record &&
|
||
Array.isArray(record.vector) &&
|
||
record.vector.length > 0 &&
|
||
!isZeroNormVector(record.vector)
|
||
) {
|
||
vectored++
|
||
}
|
||
}
|
||
}
|
||
} catch (error: any) {
|
||
if (error?.code !== 'ENOENT') throw error
|
||
}
|
||
return vectored
|
||
}
|
||
|
||
/**
|
||
* Persist counts to filesystem storage
|
||
*/
|
||
protected async persistCounts(): Promise<void> {
|
||
if (!this.countsFilePath) return
|
||
|
||
try {
|
||
const counts = {
|
||
entityCounts: Object.fromEntries(this.entityCounts),
|
||
verbCounts: Object.fromEntries(this.verbCounts),
|
||
totalNounCount: this.totalNounCount,
|
||
totalVerbCount: this.totalVerbCount,
|
||
// ALL-visibility ledger scalars (+ the suspect flag) — absent in files
|
||
// written before the ledger existed; initializeCounts() derives them once.
|
||
totalNounCountAll: this.totalNounCountAll,
|
||
totalVerbCountAll: this.totalVerbCountAll,
|
||
// Vectored-noun ledger scalar — absent in files written before it
|
||
// existed; initializeCounts() derives it once (a content scan, see
|
||
// scanVectoredNounCount()'s JSDoc).
|
||
totalVectoredNounCount: this.totalVectoredNounCount,
|
||
allCountsSuspect: this.allCountsSuspect,
|
||
// Derivation-rule stamp for the ALL scalars above — 'identity-record'
|
||
// when they were counted one-per-metadata-content-leg (the honest
|
||
// rule); omitted (JSON.stringify drops `undefined`) when the current
|
||
// in-memory scalars came from a legacy container-rule counts.json
|
||
// that hasn't been through a sanctioned recount yet, so a future load
|
||
// keeps naming them suspect rather than trusting an unproven value.
|
||
allCountsDerivedBy: this.allCountsDerivedBy,
|
||
lastUpdated: new Date().toISOString()
|
||
}
|
||
|
||
// ATOMIC (temp + rename), never a plain writeFile. A direct write
|
||
// truncates the file first, so every persist opened a window — measured
|
||
// at roughly 750ms after a flush or close on a real store — in which a
|
||
// concurrent reader saw counts.json EMPTY. An empty file is unparseable,
|
||
// and an unparseable ledger sends the next open down the full-rescan
|
||
// path: the cheapest file in the store was costing the most expensive
|
||
// recovery. The rename is atomic, so a reader sees the old ledger or the
|
||
// new one, never neither.
|
||
await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2))
|
||
} catch (error) {
|
||
console.error('Error persisting counts:', error)
|
||
}
|
||
}
|
||
|
||
|
||
|
||
// =============================================
|
||
// Intelligent Directory Sharding
|
||
// =============================================
|
||
|
||
|
||
|
||
|
||
|
||
/**
|
||
* Migrate a single file atomically
|
||
*/
|
||
private async migrateFile(
|
||
fileInfo: { oldPath: string; id: string; type: 'noun' | 'verb' },
|
||
fromDepth: number,
|
||
toDepth: number
|
||
): Promise<void> {
|
||
const baseDir = fileInfo.type === 'noun' ? this.nounsDir : this.verbsDir
|
||
|
||
// Calculate old path (already known)
|
||
const oldPath = fileInfo.oldPath
|
||
|
||
// Calculate new path using target depth
|
||
const shard = fileInfo.id.substring(0, 2).toLowerCase()
|
||
const newPath = path.join(baseDir, shard, `${fileInfo.id}.json`)
|
||
|
||
// Check if file already exists at new location
|
||
if (await this.fileExists(newPath)) {
|
||
// File already migrated or duplicate - skip
|
||
return
|
||
}
|
||
|
||
// Atomic rename/move
|
||
await fs.promises.rename(oldPath, newPath)
|
||
}
|
||
|
||
|
||
|
||
|
||
/**
|
||
* Whether this store already holds canonical 8.0 entities.
|
||
*
|
||
* 8.0 writes nouns to `entities/nouns/<shard>/<id>/vectors.json` (see
|
||
* `getNounVectorPath`). This checks that canonical shard tree — the one the
|
||
* DB actually reads and writes (the same `entities/nouns/<hex>` shards
|
||
* `getNounsWithPagination` walks) — so the new-vs-established boot log is
|
||
* truthful. (The 7.x hnsw sharding probe this replaced inspected a directory
|
||
* the 8.0 write path never populated, so it mislabeled every established
|
||
* store "New installation" on every boot; that probe and its depth-migration
|
||
* machinery are removed.)
|
||
*
|
||
* @returns true if at least one 2-hex shard directory (00–ff) exists under
|
||
* `entities/nouns/`, i.e. the store has previously persisted entities.
|
||
*/
|
||
private async hasCanonicalEntities(): Promise<boolean> {
|
||
const canonicalNounsDir = path.join(this.rootDir, 'entities', 'nouns')
|
||
try {
|
||
const entries = await fs.promises.readdir(canonicalNounsDir, {
|
||
withFileTypes: true
|
||
})
|
||
// A populated store has ≥1 two-hex shard dir (00–ff). The vestigial
|
||
// `hnsw` subdir is 4 chars and correctly excluded by the hex test.
|
||
return entries.some(
|
||
(e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name)
|
||
)
|
||
} catch {
|
||
// Directory absent (fresh store) or unreadable → no persisted entities.
|
||
return false
|
||
}
|
||
}
|
||
|
||
|
||
/**
|
||
* Check if a file exists (handles both sharded and non-sharded)
|
||
*/
|
||
private async fileExists(filePath: string): Promise<boolean> {
|
||
try {
|
||
await fs.promises.access(filePath, fs.constants.F_OK)
|
||
return true
|
||
} catch {
|
||
return false
|
||
}
|
||
}
|
||
|
||
// =============================================
|
||
// HNSW Index Persistence
|
||
// =============================================
|
||
|
||
/**
|
||
* Get vector for a noun
|
||
* Uses BaseStorage's getNoun (type-first paths)
|
||
*/
|
||
public async getNounVector(id: string): Promise<number[] | null> {
|
||
const noun = await this.getNoun(id)
|
||
return noun ? noun.vector : null
|
||
}
|
||
|
||
/**
|
||
* Save HNSW graph data for a noun
|
||
*
|
||
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
||
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
||
*/
|
||
public async saveVectorIndexData(nounId: string, hnswData: {
|
||
level: number
|
||
connections: Record<string, string[]>
|
||
}): Promise<void> {
|
||
const lockKey = `hnsw/${nounId}`
|
||
|
||
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
|
||
// Problem: Without mutex, concurrent operations can:
|
||
// 1. Thread A reads noun (connections: [1,2,3])
|
||
// 2. Thread B reads noun (connections: [1,2,3])
|
||
// 3. Thread A adds connection 4, writes [1,2,3,4]
|
||
// 4. Thread B adds connection 5, writes [1,2,3,5] ← Connection 4 LOST!
|
||
// Solution: Mutex serializes operations per entity (like Memory/OPFS adapters)
|
||
// Production scale: Prevents corruption at 1000+ concurrent operations
|
||
|
||
// Wait for any pending operations on this entity
|
||
while (this.hnswLocks.has(lockKey)) {
|
||
await this.hnswLocks.get(lockKey)
|
||
}
|
||
|
||
// Acquire lock
|
||
let releaseLock!: () => void
|
||
const lockPromise = new Promise<void>(resolve => { releaseLock = resolve })
|
||
this.hnswLocks.set(lockKey, lockPromise)
|
||
|
||
try {
|
||
// Use BaseStorage's getNoun (type-first paths)
|
||
// Read existing noun data (if exists)
|
||
const existingNoun = await this.getNoun(nounId)
|
||
|
||
if (!existingNoun) {
|
||
// Noun doesn't exist - cannot update HNSW data for non-existent noun
|
||
throw new Error(`Cannot save HNSW data: noun ${nounId} not found`)
|
||
}
|
||
|
||
// Convert connections from Record to Map format for storage
|
||
const connectionsMap = new Map<number, Set<string>>()
|
||
for (const [level, nodeIds] of Object.entries(hnswData.connections)) {
|
||
connectionsMap.set(Number(level), new Set(nodeIds))
|
||
}
|
||
|
||
// Preserve id and vector, update only HNSW graph metadata
|
||
const updatedNoun: HNSWNoun = {
|
||
...existingNoun,
|
||
level: hnswData.level,
|
||
connections: connectionsMap
|
||
}
|
||
|
||
// Use BaseStorage's saveNoun (type-first paths, atomic write)
|
||
await this.saveNoun(updatedNoun)
|
||
} finally {
|
||
// Release lock (ALWAYS runs, even if error thrown)
|
||
this.hnswLocks.delete(lockKey)
|
||
releaseLock()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get HNSW graph data for a noun
|
||
* Uses BaseStorage's getNoun (type-first paths)
|
||
*/
|
||
public async getVectorIndexData(nounId: string): Promise<{
|
||
level: number
|
||
connections: Record<string, string[]>
|
||
} | null> {
|
||
const noun = await this.getNoun(nounId)
|
||
|
||
if (!noun) {
|
||
return null
|
||
}
|
||
|
||
// Convert connections from Map to Record format
|
||
const connectionsRecord: Record<string, string[]> = {}
|
||
if (noun.connections) {
|
||
for (const [level, nodeIds] of noun.connections.entries()) {
|
||
connectionsRecord[String(level)] = Array.from(nodeIds)
|
||
}
|
||
}
|
||
|
||
return {
|
||
level: noun.level || 0,
|
||
connections: connectionsRecord
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Save HNSW system data (entry point, max level)
|
||
*
|
||
* CRITICAL FIX: Mutex lock + atomic write to prevent race conditions
|
||
*/
|
||
public async saveHNSWSystem(systemData: {
|
||
entryPointId: string | null
|
||
maxLevel: number
|
||
}): Promise<void> {
|
||
await this.ensureInitialized()
|
||
|
||
const lockKey = 'hnsw/system'
|
||
|
||
// CRITICAL FIX: Mutex lock to serialize system updates
|
||
// System data (entry point, max level) updated frequently during HNSW construction
|
||
// Without mutex, concurrent updates can lose data (same as entity-level problem)
|
||
|
||
// Wait for any pending system updates
|
||
while (this.hnswLocks.has(lockKey)) {
|
||
await this.hnswLocks.get(lockKey)
|
||
}
|
||
|
||
// Acquire lock
|
||
let releaseLock!: () => void
|
||
const lockPromise = new Promise<void>(resolve => { releaseLock = resolve })
|
||
this.hnswLocks.set(lockKey, lockPromise)
|
||
|
||
try {
|
||
const filePath = path.join(this.systemDir, 'hnsw-system.json')
|
||
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
|
||
|
||
try {
|
||
// Write to temp file
|
||
await this.ensureDirectoryExists(path.dirname(tempPath))
|
||
await fs.promises.writeFile(tempPath, JSON.stringify(systemData, null, 2))
|
||
|
||
// Atomic rename temp → final (POSIX atomicity guarantee)
|
||
await fs.promises.rename(tempPath, filePath)
|
||
} catch (error: any) {
|
||
// Clean up temp file on any error
|
||
try {
|
||
await fs.promises.unlink(tempPath)
|
||
} catch (cleanupError) {
|
||
// Ignore cleanup errors
|
||
}
|
||
throw error
|
||
}
|
||
} finally {
|
||
// Release lock
|
||
this.hnswLocks.delete(lockKey)
|
||
releaseLock()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get HNSW system data
|
||
*/
|
||
public async getHNSWSystem(): Promise<{
|
||
entryPointId: string | null
|
||
maxLevel: number
|
||
} | null> {
|
||
await this.ensureInitialized()
|
||
|
||
const filePath = path.join(this.systemDir, 'hnsw-system.json')
|
||
|
||
try {
|
||
const data = await fs.promises.readFile(filePath, 'utf-8')
|
||
return JSON.parse(data)
|
||
} catch (error: any) {
|
||
if (error.code !== 'ENOENT') {
|
||
console.error('Error reading HNSW system data:', error)
|
||
}
|
||
return null
|
||
}
|
||
}
|
||
}
|