fix: full-removal canonical deletes + family-scoped migration gate
Two production-reported spine fixes: - Canonical delete is a FULL removal: remove()/removeMany() deleted the metadata leg but never the canonical vectors.json leg or the <id>/ directory, leaving ghost rows that inflated enumerated counts forever, read as damage scars, and caused duplicate readdir entries for re-created VFS paths (the stale-unpost path). The delete operation now routes through storage.deleteNoun — both legs + the entity container — with a full two-leg before-image rollback; deleteVerb symmetric; the blind 'no metadata file' catch that masked real faults is gone. The generation log still holds the delete's before-image, so asOf() history is unchanged. repairIndex() gains a conservative, loud orphan-prune sweep (containers with no metadata content leg) + count recompute for stores damaged by earlier versions. - Family-scoped migration gate: every read blocked on the whole-brain migration lock even when the migrating index was irrelevant. The gate now scopes to the index families a read actually consults — canonical reads never wait; find() waits only on its query shape's families; graph traversals wait only on the graph family. Writes keep the conservative whole-brain wait. A read that needs the migrating family still blocks (bounded) with the retryable MigrationInProgressError.
This commit is contained in:
parent
1d26988963
commit
366f9a91f5
8 changed files with 657 additions and 66 deletions
|
|
@ -502,6 +502,104 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.writeBarrierDeleteDirs?.add(path.dirname(pathStr))
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Remove the entity container directory after both canonical legs
|
||||
* were deleted (full-removal delete). `objectLegPath` is one leg's
|
||||
* storage-root-relative path (e.g. `entities/nouns/<shard>/<id>/vectors.json`);
|
||||
* its parent is the `<id>/` entity directory. `rmdir` removes it ONLY if empty
|
||||
* — both legs are gone by this point, so it should be. A non-empty dir means an
|
||||
* unexpected leftover (a leg whose delete faulted — already surfaced upstream —
|
||||
* or foreign data): we do NOT recursively nuke it (loud errors, never quiet
|
||||
* losses); we log and leave it for the orphan repair sweep (`repairIndex`).
|
||||
*/
|
||||
protected override async removeCanonicalContainer(objectLegPath: string): Promise<void> {
|
||||
const relDir = path.dirname(objectLegPath)
|
||||
const absDir = path.join(this.rootDir, relDir)
|
||||
try {
|
||||
await fs.promises.rmdir(absDir)
|
||||
// The dir removal is durable once its PARENT (the shard dir) is fsync'd.
|
||||
this.writeBarrierDeleteDirs?.add(path.dirname(relDir))
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') return // container already gone — fine
|
||||
if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') {
|
||||
console.warn(
|
||||
`[FileSystemStorage] entity container ${relDir} not empty after delete — ` +
|
||||
`leaving it for the orphan repair sweep (brain.repairIndex()).`
|
||||
)
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Prune orphaned entity directories left by the pre-8.3.1
|
||||
* partial-delete defect. A delete used to remove the metadata (content) leg
|
||||
* but leave the `vectors.json` leg + the `<id>/` directory (a "ghost"), or —
|
||||
* on the direct storage path — remove both legs but leave an empty directory
|
||||
* (a "scar"). Neither is a live entity (`getNoun` needs the metadata content
|
||||
* leg) yet each lingers on disk, inflating enumerated counts and confusing
|
||||
* locator resolution.
|
||||
*
|
||||
* CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg — a
|
||||
* directory that still holds its content is left untouched. LOUD: logs every
|
||||
* removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs
|
||||
* automatically. Returns the pruned container ids so the caller can recompute
|
||||
* counts.
|
||||
*/
|
||||
public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
|
||||
await this.ensureInitialized()
|
||||
const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
|
||||
|
||||
for (const [kind, root] of [
|
||||
['nouns', 'entities/nouns'],
|
||||
['verbs', 'entities/verbs']
|
||||
] as const) {
|
||||
const rootAbs = path.join(this.rootDir, root)
|
||||
let shards: string[]
|
||||
try {
|
||||
shards = await fs.promises.readdir(rootAbs)
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') continue // no entities of this kind yet
|
||||
throw error
|
||||
}
|
||||
|
||||
for (const shard of shards) {
|
||||
const shardAbs = path.join(rootAbs, shard)
|
||||
let entries: import('fs').Dirent[]
|
||||
try {
|
||||
entries = await fs.promises.readdir(shardAbs, { withFileTypes: true })
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') continue
|
||||
throw error
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const idAbs = path.join(shardAbs, entry.name)
|
||||
let legs: string[]
|
||||
try {
|
||||
legs = await fs.promises.readdir(idAbs)
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') continue
|
||||
throw error
|
||||
}
|
||||
// A live entity has its metadata content leg. No content leg → a
|
||||
// vector-only ghost or an empty scar → prune the whole container.
|
||||
if (legs.some((f) => f.startsWith('metadata.json'))) continue
|
||||
await fs.promises.rm(idAbs, { recursive: true, force: true })
|
||||
pruned[kind].push(entry.name)
|
||||
console.warn(
|
||||
`[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` +
|
||||
`container ${root}/${shard}/${entry.name} (no metadata content leg)`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pruned
|
||||
}
|
||||
|
||||
/**
|
||||
* Primitive operation: List objects under path prefix
|
||||
* All metadata operations use this internally via base class routing
|
||||
|
|
@ -631,7 +729,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
public override async removeRawPrefix(prefix: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// Registered-blob contract: a prefix-nuke must not take out a protected
|
||||
// family member (ADR-004 §7). Throws if the prefix intersects one.
|
||||
// family member. Throws if the prefix intersects one.
|
||||
await this.assertPrefixNotProtected(prefix)
|
||||
await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true })
|
||||
}
|
||||
|
|
@ -1240,7 +1338,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// errors, never quiet losses. Fault-propagation restored lockstep with
|
||||
// cortex 3.0.13, whose two column-store call sites now handle the throw
|
||||
// (they mark the field unavailable + throw a named error) instead of
|
||||
// relying on null-on-error (CORTEX-BILLION-SCALE-SPINE).
|
||||
// relying on null-on-error.
|
||||
if (isAbsentError(err)) return null
|
||||
throw err
|
||||
}
|
||||
|
|
@ -1253,7 +1351,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
*/
|
||||
public async deleteBinaryBlob(key: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
// Registered-blob contract (ADR-004 §7): refuse to delete a declared
|
||||
// Registered-blob contract: refuse to delete a declared
|
||||
// derived-index family member — an in-process GC/sweeper cannot remove a
|
||||
// load-bearing index file. Throws ProtectedArtifactError; no-op when no
|
||||
// families are registered.
|
||||
|
|
|
|||
|
|
@ -253,7 +253,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
/** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
|
||||
private _graphFastPathProbed = false
|
||||
/**
|
||||
* Registered-blob contract (ADR-004 §7): declared derived-index families, keyed
|
||||
* Registered-blob contract: declared derived-index families, keyed
|
||||
* by name. Members are undeletable through the blob delete seams. Loaded lazily
|
||||
* from `_system/derived-artifacts.json` and re-persisted on every change.
|
||||
*/
|
||||
|
|
@ -842,6 +842,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
return this.deleteObjectFromPath(path)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Remove the container that held a canonical entity's leg files,
|
||||
* called after both legs are deleted so a delete leaves NOTHING behind — no
|
||||
* orphan "scar" directory. `objectLegPath` is any one of the entity's leg
|
||||
* paths (e.g. its `vectors.json`); the container is that path's parent.
|
||||
*
|
||||
* Default: a no-op. Key/prefix-addressed stores (in-memory, cloud object
|
||||
* stores) have no empty-container concept — deleting the leg keys already
|
||||
* removes the entity entirely. The filesystem adapter overrides this to
|
||||
* `rmdir` the emptied entity directory.
|
||||
*
|
||||
* @param objectLegPath - Storage-root-relative path of one of the entity's legs.
|
||||
* @protected
|
||||
*/
|
||||
protected async removeCanonicalContainer(objectLegPath: string): Promise<void> {
|
||||
void objectLegPath
|
||||
}
|
||||
|
||||
/**
|
||||
* @description List canonical objects under a storage-root-relative prefix.
|
||||
*
|
||||
|
|
@ -1062,7 +1080,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Registered-blob contract (ADR-004 §7) — declared derived-index families are
|
||||
// Registered-blob contract — declared derived-index families are
|
||||
// undeletable through the blob delete seams. Shared here so every adapter that
|
||||
// extends BaseStorage inherits the same enforcement; the concrete
|
||||
// deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable /
|
||||
|
|
@ -1116,7 +1134,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
/**
|
||||
* @description A blob key that is a transient write-scratch file (a `*.tmp.*`
|
||||
* temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as
|
||||
* part of an atomic write; it is never a protected family member (ADR-004 §7).
|
||||
* part of an atomic write; it is never a protected family member.
|
||||
*/
|
||||
private isTransientBlobKey(key: string): boolean {
|
||||
return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key)
|
||||
|
|
@ -1185,7 +1203,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
/**
|
||||
* @description Verify every declared family has all its members present on
|
||||
* disk (ADR-004 §7 missing-on-open catch for an EXTERNAL deleter that bypasses
|
||||
* disk (missing-on-open catch for an EXTERNAL deleter that bypasses
|
||||
* the in-process refusal). Returns the incomplete families (name + missing
|
||||
* members) — the caller decides how to heal (rebuild from canonical). Loud by
|
||||
* construction: a missing load-bearing blob is named, never silently tolerated.
|
||||
|
|
@ -1601,16 +1619,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async deleteNoun(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Delete both the vector file and metadata file (2-file system)
|
||||
await this.deleteNoun_internal(id)
|
||||
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
||||
// entity's container, so a delete leaves nothing behind — no orphan/scar
|
||||
// directory to inflate the enumerated count or ghost resolveLocator. The
|
||||
// generation store holds the immutable before-image, so asOf() still
|
||||
// reconstructs the deleted entity until retention expires.
|
||||
await this.deleteNoun_internal(id) // vectors.json leg
|
||||
|
||||
// Delete metadata file (if it exists)
|
||||
try {
|
||||
await this.deleteNounMetadata(id)
|
||||
} catch (error) {
|
||||
// Ignore if metadata file doesn't exist
|
||||
prodLog.debug(`No metadata file to delete for noun ${id}`)
|
||||
}
|
||||
// Metadata leg + count decrement. A genuine "already absent" is a no-op
|
||||
// downstream (readCanonicalObject / deleteObjectFromPath both treat ENOENT as
|
||||
// success); a REAL fault must surface loudly (loud errors, never quiet
|
||||
// losses) and must never silently skip the count decrement — so this is NO
|
||||
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
||||
await this.deleteNounMetadata(id)
|
||||
|
||||
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
||||
await this.removeCanonicalContainer(getNounVectorPath(id))
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -2915,16 +2939,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async deleteVerb(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Delete both the vector file and metadata file (2-file system)
|
||||
await this.deleteVerb_internal(id)
|
||||
|
||||
// Delete metadata file (if it exists)
|
||||
try {
|
||||
await this.deleteVerbMetadata(id)
|
||||
} catch (error) {
|
||||
// Ignore if metadata file doesn't exist
|
||||
prodLog.debug(`No metadata file to delete for verb ${id}`)
|
||||
}
|
||||
// FULL removal (see deleteNoun): both canonical legs + the verb container.
|
||||
// The blind catch that masked real faults as "no metadata file" is gone — a
|
||||
// genuine absence is already a no-op downstream; a real fault surfaces loudly.
|
||||
await this.deleteVerb_internal(id) // vectors.json leg
|
||||
await this.deleteVerbMetadata(id) // metadata leg + count decrement
|
||||
await this.removeCanonicalContainer(getVerbVectorPath(id))
|
||||
}
|
||||
/**
|
||||
* Get graph index (lazy initialization with concurrent access protection)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue