fix: full-removal canonical deletes + family-scoped migration gate

Two production-reported spine fixes:

- Canonical delete is a FULL removal: remove()/removeMany() deleted the
  metadata leg but never the canonical vectors.json leg or the <id>/
  directory, leaving ghost rows that inflated enumerated counts forever,
  read as damage scars, and caused duplicate readdir entries for
  re-created VFS paths (the stale-unpost path). The delete operation now
  routes through storage.deleteNoun — both legs + the entity container —
  with a full two-leg before-image rollback; deleteVerb symmetric; the
  blind 'no metadata file' catch that masked real faults is gone. The
  generation log still holds the delete's before-image, so asOf() history
  is unchanged. repairIndex() gains a conservative, loud orphan-prune
  sweep (containers with no metadata content leg) + count recompute for
  stores damaged by earlier versions.

- Family-scoped migration gate: every read blocked on the whole-brain
  migration lock even when the migrating index was irrelevant. The gate
  now scopes to the index families a read actually consults — canonical
  reads never wait; find() waits only on its query shape's families;
  graph traversals wait only on the graph family. Writes keep the
  conservative whole-brain wait. A read that needs the migrating family
  still blocks (bounded) with the retryable MigrationInProgressError.
This commit is contained in:
David Snelling 2026-07-14 10:11:53 -07:00
parent 1d26988963
commit 366f9a91f5
8 changed files with 657 additions and 66 deletions

View file

@ -502,6 +502,104 @@ export class FileSystemStorage extends BaseStorage {
this.writeBarrierDeleteDirs?.add(path.dirname(pathStr))
}
/**
* @description Remove the entity container directory after both canonical legs
* were deleted (full-removal delete). `objectLegPath` is one leg's
* storage-root-relative path (e.g. `entities/nouns/<shard>/<id>/vectors.json`);
* its parent is the `<id>/` entity directory. `rmdir` removes it ONLY if empty
* both legs are gone by this point, so it should be. A non-empty dir means an
* unexpected leftover (a leg whose delete faulted already surfaced upstream
* or foreign data): we do NOT recursively nuke it (loud errors, never quiet
* losses); we log and leave it for the orphan repair sweep (`repairIndex`).
*/
protected override async removeCanonicalContainer(objectLegPath: string): Promise<void> {
const relDir = path.dirname(objectLegPath)
const absDir = path.join(this.rootDir, relDir)
try {
await fs.promises.rmdir(absDir)
// The dir removal is durable once its PARENT (the shard dir) is fsync'd.
this.writeBarrierDeleteDirs?.add(path.dirname(relDir))
} catch (error: any) {
if (error?.code === 'ENOENT') return // container already gone — fine
if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') {
console.warn(
`[FileSystemStorage] entity container ${relDir} not empty after delete — ` +
`leaving it for the orphan repair sweep (brain.repairIndex()).`
)
return
}
throw error
}
}
/**
* @description Prune orphaned entity directories left by the pre-8.3.1
* partial-delete defect. A delete used to remove the metadata (content) leg
* but leave the `vectors.json` leg + the `<id>/` directory (a "ghost"), or
* on the direct storage path remove both legs but leave an empty directory
* (a "scar"). Neither is a live entity (`getNoun` needs the metadata content
* leg) yet each lingers on disk, inflating enumerated counts and confusing
* locator resolution.
*
* CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg a
* directory that still holds its content is left untouched. LOUD: logs every
* removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs
* automatically. Returns the pruned container ids so the caller can recompute
* counts.
*/
public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
await this.ensureInitialized()
const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
for (const [kind, root] of [
['nouns', 'entities/nouns'],
['verbs', 'entities/verbs']
] as const) {
const rootAbs = path.join(this.rootDir, root)
let shards: string[]
try {
shards = await fs.promises.readdir(rootAbs)
} catch (error: any) {
if (error?.code === 'ENOENT') continue // no entities of this kind yet
throw error
}
for (const shard of shards) {
const shardAbs = path.join(rootAbs, shard)
let entries: import('fs').Dirent[]
try {
entries = await fs.promises.readdir(shardAbs, { withFileTypes: true })
} catch (error: any) {
if (error?.code === 'ENOENT') continue
throw error
}
for (const entry of entries) {
if (!entry.isDirectory()) continue
const idAbs = path.join(shardAbs, entry.name)
let legs: string[]
try {
legs = await fs.promises.readdir(idAbs)
} catch (error: any) {
if (error?.code === 'ENOENT') continue
throw error
}
// A live entity has its metadata content leg. No content leg → a
// vector-only ghost or an empty scar → prune the whole container.
if (legs.some((f) => f.startsWith('metadata.json'))) continue
await fs.promises.rm(idAbs, { recursive: true, force: true })
pruned[kind].push(entry.name)
console.warn(
`[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` +
`container ${root}/${shard}/${entry.name} (no metadata content leg)`
)
}
}
}
return pruned
}
/**
* Primitive operation: List objects under path prefix
* All metadata operations use this internally via base class routing
@ -631,7 +729,7 @@ export class FileSystemStorage extends BaseStorage {
public override async removeRawPrefix(prefix: string): Promise<void> {
await this.ensureInitialized()
// Registered-blob contract: a prefix-nuke must not take out a protected
// family member (ADR-004 §7). Throws if the prefix intersects one.
// family member. Throws if the prefix intersects one.
await this.assertPrefixNotProtected(prefix)
await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true })
}
@ -1240,7 +1338,7 @@ export class FileSystemStorage extends BaseStorage {
// errors, never quiet losses. Fault-propagation restored lockstep with
// cortex 3.0.13, whose two column-store call sites now handle the throw
// (they mark the field unavailable + throw a named error) instead of
// relying on null-on-error (CORTEX-BILLION-SCALE-SPINE).
// relying on null-on-error.
if (isAbsentError(err)) return null
throw err
}
@ -1253,7 +1351,7 @@ export class FileSystemStorage extends BaseStorage {
*/
public async deleteBinaryBlob(key: string): Promise<void> {
await this.ensureInitialized()
// Registered-blob contract (ADR-004 §7): refuse to delete a declared
// Registered-blob contract: refuse to delete a declared
// derived-index family member — an in-process GC/sweeper cannot remove a
// load-bearing index file. Throws ProtectedArtifactError; no-op when no
// families are registered.