feat: registered-blob family contract — declared index blobs are undeletable (ADR-004 Pass 2)

The class-killer for the lost-main.dkann incident (ADR-004 §7). A provider can
declare a derived-index blob FAMILY (a set of members that are load-bearing
together, e.g. vector-base = main.dkann + main.slotmap + main.slotrev). Once
declared:

- deleteBinaryBlob / removeRawPrefix REFUSE to remove a declared member, throwing
  the new ProtectedArtifactError — an in-process GC / sweeper is now INCAPABLE of
  deleting a load-bearing index file (COLD != DEAD). Intentional retirement is an
  explicit unregisterDerivedFamily(name) first.
- The declaration persists to _system/derived-artifacts.json, so protection
  survives a reopen; clear() resets it with the rest of the derived footprint.
- checkDerivedFamiliesPresent() names any member missing on open (the catch for an
  EXTERNAL deleter that bypasses the in-process refusal) → rebuild from canonical.
- Transients (*.tmp.*, *.rebuild-tmp, *.rotate-tmp) are never protected; a
  namespace family protects a growing prefix (seg-*).

New StorageAdapter surface (optional): registerDerivedFamily / unregisterDerivedFamily
/ listDerivedFamilies + DerivedFamilyDeclaration; new exported errors
ProtectedArtifactError / DerivedArtifactMissingError. Enforcement is inert until a
provider declares a family (no regression). Cor declares its 6 families + does the
atomic set-swap in 3.0.15 (M2); brainy builds the contract now. 8 tests.
This commit is contained in:
David Snelling 2026-07-13 14:52:06 -07:00
parent 6bcb54f0d9
commit bfa1762107
7 changed files with 429 additions and 5 deletions

View file

@ -754,6 +754,39 @@ export interface Change {
data?: HNSWNounWithMetadata | HNSWVerbWithMetadata
}
/**
* @description A declared derived-index blob FAMILY (ADR-004 §7 the
* registered-blob contract). A family names the set of on-disk blobs that a
* derived index needs AS A SET (e.g. the vector base = `main.dkann` +
* `main.slotmap` + `main.slotrev`): losing ANY member corrupts the index. Once a
* family is declared:
* - its members are UNDELETABLE through the storage layer `deleteBinaryBlob` /
* `removeRawPrefix` refuse with a `ProtectedArtifactError`, so an in-process
* GC/sweeper cannot remove a load-bearing file (intentional retirement =
* `unregisterDerivedFamily` first);
* - a member missing on open is a loud `DerivedArtifactMissingError` rebuild.
* `COLD ≠ DEAD`: a write-once segment or a recovery-critical archive is
* load-bearing even when it has not been touched in a long time.
*/
export interface DerivedFamilyDeclaration {
/** Stable family id, e.g. `'vector-base'` / `'metadata-sstables'`. */
name: string
/**
* The logical blob keys that make up the family. When {@link namespace} is
* set, each entry is a PREFIX protecting every key beneath it (for growing
* sets like `seg-*`); otherwise each entry is an exact member key.
*/
members: string[]
/** When true, {@link members} are prefixes (protect all keys beneath each). */
namespace?: boolean
/**
* Whether a missing family can be rebuilt from the canonical records (default
* `true`). `false` marks an irreplaceable family (a missing member is data
* loss, not a rebuild).
*/
rebuildable?: boolean
}
export interface StorageAdapter {
init(): Promise<void>
@ -1049,6 +1082,33 @@ export interface StorageAdapter {
*/
getBinaryBlobPath(key: string): string | null
/**
* @description OPTIONAL (ADR-004 §7 registered-blob contract). Declare a
* derived-index blob {@link DerivedFamilyDeclaration | family} whose members
* become UNDELETABLE through this adapter a subsequent `deleteBinaryBlob` /
* `removeRawPrefix` that would remove a declared member throws a
* `ProtectedArtifactError`. Providers declare their families on create; the
* declaration is persisted so protection survives a reopen. Idempotent per
* `name` (re-declaring replaces).
* @param family - The family to protect.
*/
registerDerivedFamily?(family: DerivedFamilyDeclaration): Promise<void>
/**
* @description OPTIONAL. Remove a family's protection so its members can be
* deleted again the explicit, auditable step for intentional retirement of a
* derived index (the ONLY way a declared member becomes deletable).
* @param name - The {@link DerivedFamilyDeclaration.name} to unregister.
*/
unregisterDerivedFamily?(name: string): Promise<void>
/**
* @description OPTIONAL. List the currently-declared derived-index families
* the source of truth for what `clear()` must wipe and what a
* missing-on-open check verifies.
*/
listDerivedFamilies?(): Promise<DerivedFamilyDeclaration[]>
/**
* Save statistics data
* @param statistics The statistics data to save

View file

@ -15,6 +15,8 @@ export type BrainyErrorType =
| 'GRAPH_INDEX_NOT_READY'
| 'METADATA_INDEX_NOT_READY'
| 'VECTOR_INDEX_NOT_READY'
| 'PROTECTED_ARTIFACT'
| 'DERIVED_ARTIFACT_MISSING'
| 'MIGRATION_IN_PROGRESS'
/**
@ -307,6 +309,60 @@ export class VectorIndexNotReadyError extends BrainyError {
}
}
/**
* Thrown when a delete (`deleteBinaryBlob` / `removeRawPrefix`) would remove a
* blob that is a declared member of a protected derived-index FAMILY (ADR-004 §7
* registered-blob contract). Declared derived artifacts are undeletable through
* the storage layer this makes an in-process GC / sweeper INCAPABLE of removing
* a load-bearing index file (the lost-`main.dkann` class). Intentional retirement
* is the explicit `unregisterDerivedFamily(name)` step, then the delete.
*/
export class ProtectedArtifactError extends BrainyError {
/** The blob key the delete targeted. */
public readonly key: string
/** The protected family the key belongs to. */
public readonly family: string
constructor(key: string, family: string) {
super(
`Refused to delete '${key}': it is a declared member of the protected ` +
`derived-index family '${family}'. Declared derived artifacts are undeletable ` +
`through the storage layer (COLD ≠ DEAD) — unregisterDerivedFamily('${family}') ` +
`first if retirement is intentional.`,
'PROTECTED_ARTIFACT',
false
)
this.name = 'ProtectedArtifactError'
this.key = key
this.family = family
}
}
/**
* Raised (loudly) when a declared derived-index family is missing one or more of
* its members on open i.e. a load-bearing blob was deleted OUTSIDE the write
* path (an external sweeper the in-process refusal cannot stop). The index must
* be rebuilt from canonical; "healthy-while-broken" is impossible because the
* missing member is named, not silently tolerated.
*/
export class DerivedArtifactMissingError extends BrainyError {
/** The family with missing members. */
public readonly family: string
/** The member keys that are absent. */
public readonly missing: string[]
constructor(family: string, missing: string[]) {
super(
`Derived-index family '${family}' is missing ${missing.length} declared ` +
`member(s) on open (${missing.join(', ')}) — deleted outside the write path. ` +
`The index must be rebuilt from canonical.`,
'DERIVED_ARTIFACT_MISSING',
false
)
this.name = 'DerivedArtifactMissingError'
this.family = family
this.missing = missing
}
}
/**
* Thrown when a data-plane read or write is issued against a brain that is
* running its one-time, automatic 7.x 8.0 on-disk upgrade the coordinated

View file

@ -150,7 +150,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
// Base error + typed migration-lock error — thrown by any data-plane call while a
// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError } from './errors/brainyError.js'
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
export type { BrainyErrorType } from './errors/brainyError.js'
// ============= 8.0 Db API — generational MVCC =============
@ -301,7 +301,8 @@ import type {
HNSWNoun,
HNSWVerb,
HNSWConfig,
StorageAdapter
StorageAdapter,
DerivedFamilyDeclaration
} from './coreTypes.js'
// Export vector index implementation (the JS HNSW path)
@ -319,7 +320,8 @@ export type {
HNSWNoun,
HNSWVerb,
HNSWConfig,
StorageAdapter
StorageAdapter,
DerivedFamilyDeclaration
}
// Export graph types

View file

@ -630,6 +630,9 @@ export class FileSystemStorage extends BaseStorage {
*/
public override async removeRawPrefix(prefix: string): Promise<void> {
await this.ensureInitialized()
// Registered-blob contract: a prefix-nuke must not take out a protected
// family member (ADR-004 §7). Throws if the prefix intersects one.
await this.assertPrefixNotProtected(prefix)
await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true })
}
@ -1250,6 +1253,11 @@ export class FileSystemStorage extends BaseStorage {
*/
public async deleteBinaryBlob(key: string): Promise<void> {
await this.ensureInitialized()
// Registered-blob contract (ADR-004 §7): refuse to delete a declared
// derived-index family member — an in-process GC/sweeper cannot remove a
// load-bearing index file. Throws ProtectedArtifactError; no-op when no
// families are registered.
await this.assertBlobKeyDeletable(key)
try {
await fs.promises.unlink(this.blobPath(key))
} catch {

View file

@ -183,6 +183,9 @@ export class MemoryStorage extends BaseStorage {
* @param key - The blob key.
*/
public async deleteBinaryBlob(key: string): Promise<void> {
// Registered-blob contract (ADR-004 §7) — parity with the filesystem adapter:
// a declared family member is undeletable (throws ProtectedArtifactError).
await this.assertBlobKeyDeletable(key)
this.blobStore.delete(key)
}

View file

@ -15,7 +15,8 @@ import {
VerbMetadata,
HNSWNounWithMetadata,
HNSWVerbWithMetadata,
StatisticsData
StatisticsData,
DerivedFamilyDeclaration
} from '../coreTypes.js'
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
import { validateNounType, validateVerbType } from '../utils/typeValidation.js'
@ -31,7 +32,7 @@ import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js'
import { unwrapBinaryData } from './binaryDataCodec.js'
import { prodLog } from '../utils/logger.js'
import { isAbsentError } from '../utils/errorClassification.js'
import { BrainyError } from '../errors/brainyError.js'
import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js'
import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js'
import {
splitNounMetadataRecord,
@ -251,6 +252,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
protected isInitialized = false
/** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
private _graphFastPathProbed = false
/**
* Registered-blob contract (ADR-004 §7): declared derived-index families, keyed
* by name. Members are undeletable through the blob delete seams. Loaded lazily
* from `_system/derived-artifacts.json` and re-persisted on every change.
*/
private _derivedFamilies = new Map<string, DerivedFamilyDeclaration>()
/** One-shot guard for loading the persisted family registry. */
private _derivedFamiliesLoaded = false
/** Storage-root-relative path of the persisted family registry. */
private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json'
protected graphIndex?: GraphAdjacencyIndex
protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
/**
@ -1041,6 +1052,154 @@ export abstract class BaseStorage extends BaseStorageAdapter {
await this.writeObjectToPath(path, data)
}
// ==========================================================================
// Registered-blob contract (ADR-004 §7) — declared derived-index families are
// undeletable through the blob delete seams. Shared here so every adapter that
// extends BaseStorage inherits the same enforcement; the concrete
// deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable /
// assertPrefixNotProtected before removing anything.
// ==========================================================================
/** Load the persisted family registry once (lazy). */
private async ensureDerivedFamiliesLoaded(): Promise<void> {
if (this._derivedFamiliesLoaded) return
const stored = await this.readRawObject(BaseStorage.DERIVED_FAMILIES_KEY).catch(() => null)
const families = (stored as { families?: DerivedFamilyDeclaration[] } | null)?.families
if (Array.isArray(families)) {
for (const f of families) {
if (f && typeof f.name === 'string' && Array.isArray(f.members)) this._derivedFamilies.set(f.name, f)
}
}
this._derivedFamiliesLoaded = true
}
/** Persist the current family registry (fsync'd via the raw-object write). */
private async persistDerivedFamilies(): Promise<void> {
await this.writeRawObject(BaseStorage.DERIVED_FAMILIES_KEY, {
families: [...this._derivedFamilies.values()]
})
}
public async registerDerivedFamily(family: DerivedFamilyDeclaration): Promise<void> {
await this.ensureInitialized()
await this.ensureDerivedFamiliesLoaded()
this._derivedFamilies.set(family.name, {
name: family.name,
members: [...family.members],
...(family.namespace !== undefined ? { namespace: family.namespace } : {}),
...(family.rebuildable !== undefined ? { rebuildable: family.rebuildable } : {})
})
await this.persistDerivedFamilies()
}
public async unregisterDerivedFamily(name: string): Promise<void> {
await this.ensureInitialized()
await this.ensureDerivedFamiliesLoaded()
if (this._derivedFamilies.delete(name)) await this.persistDerivedFamilies()
}
public async listDerivedFamilies(): Promise<DerivedFamilyDeclaration[]> {
await this.ensureInitialized()
await this.ensureDerivedFamiliesLoaded()
return [...this._derivedFamilies.values()]
}
/**
* @description A blob key that is a transient write-scratch file (a `*.tmp.*`
* temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as
* part of an atomic write; it is never a protected family member (ADR-004 §7).
*/
private isTransientBlobKey(key: string): boolean {
return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key)
}
/**
* @description The name of the protected family a blob key belongs to, or
* `null`. A `namespace` member protects every key beneath it; a plain member
* protects that exact key.
*/
private protectingFamilyOf(key: string): string | null {
for (const family of this._derivedFamilies.values()) {
for (const member of family.members) {
if (family.namespace ? key === member || key.startsWith(member) : key === member) {
return family.name
}
}
}
return null
}
/**
* @description Enforcement point for `deleteBinaryBlob`: refuse (throw
* {@link ProtectedArtifactError}) when the key is a declared family member.
* Transients pass through. An undeclared blob delete under an ACTIVE contract
* (families are registered) is logged loudly nothing under `_blobs/` should
* vanish unremarked once the contract is in force.
*/
protected async assertBlobKeyDeletable(key: string): Promise<void> {
await this.ensureDerivedFamiliesLoaded()
if (this._derivedFamilies.size === 0) return // contract inactive — no enforcement
if (this.isTransientBlobKey(key)) return
const family = this.protectingFamilyOf(key)
if (family) throw new ProtectedArtifactError(key, family)
prodLog.warn(
`[BaseStorage] deleteBinaryBlob('${key}') removes an UNDECLARED blob while the ` +
`registered-blob contract is active — permitted, but surfaced so no _blobs/ file ` +
`disappears silently.`
)
}
/**
* @description Enforcement point for `removeRawPrefix`: refuse when the prefix
* would take out a protected family member (a prefix-nuke must not remove a
* declared blob). Transients are ignored.
*/
protected async assertPrefixNotProtected(prefix: string): Promise<void> {
await this.ensureDerivedFamiliesLoaded()
if (this._derivedFamilies.size === 0) return
for (const family of this._derivedFamilies.values()) {
for (const member of family.members) {
// Under the shared `_blobs/` root, member keys resolve beneath it; a
// prefix intersects a member when either contains the other.
const memberBlobPath = `_blobs/${member}`
if (
memberBlobPath.startsWith(prefix) ||
prefix.startsWith(memberBlobPath) ||
member.startsWith(prefix) ||
prefix.startsWith(member)
) {
if (!this.isTransientBlobKey(member)) throw new ProtectedArtifactError(member, family.name)
}
}
}
}
/**
* @description Verify every declared family has all its members present on
* disk (ADR-004 §7 missing-on-open catch for an EXTERNAL deleter that bypasses
* the in-process refusal). Returns the incomplete families (name + missing
* members) the caller decides how to heal (rebuild from canonical). Loud by
* construction: a missing load-bearing blob is named, never silently tolerated.
*/
public async checkDerivedFamiliesPresent(): Promise<Array<{ name: string; missing: string[]; rebuildable: boolean }>> {
await this.ensureInitialized()
await this.ensureDerivedFamiliesLoaded()
const incomplete: Array<{ name: string; missing: string[]; rebuildable: boolean }> = []
for (const family of this._derivedFamilies.values()) {
if (family.namespace) continue // a growing prefix has no fixed member set to verify
const missing: string[] = []
for (const member of family.members) {
const blob = await this.loadBinaryBlob(member).catch(() => null)
if (!blob) missing.push(member)
}
if (missing.length > 0) {
prodLog.warn(new DerivedArtifactMissingError(family.name, missing).message)
incomplete.push({ name: family.name, missing, rebuildable: family.rebuildable !== false })
}
}
return incomplete
}
/**
* Delete a raw object at a storage-root-relative path (no-op if absent).
*
@ -1224,6 +1383,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/
protected async reloadDerivedState(): Promise<void> {
this.clearWriteCache()
// Registered-blob registry: clear() wipes the persisted `_system/` copy and
// the family members under `_blobs/`, so drop the in-memory cache too — a
// reset brain must not keep stale family protection or report their members
// "missing" on the next check.
this._derivedFamilies.clear()
this._derivedFamiliesLoaded = false
this.nounCountsByType.fill(0)
this.verbCountsByType.fill(0)
this.subtypeCountsByType.clear()

View file

@ -0,0 +1,130 @@
/**
* @module tests/unit/storage/registered-blob-contract
* @description Pass 2 (ADR-004 §7): declared derived-index blob FAMILIES are
* undeletable through the storage layer an in-process GC/sweeper cannot remove
* a load-bearing index file (the lost-main.dkann class). Covers declare
* protected-delete-throws; unregister delete-ok; transient passthrough;
* namespace prefix protects children; removeRawPrefix refuses a protected
* intersection; persistence across reopen; missing-on-open detection.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
import { ProtectedArtifactError } from '../../../src/index.js'
describe('registered-blob family contract (Pass 2, ADR-004 §7)', () => {
let storage: any
beforeEach(async () => {
storage = new MemoryStorage()
await storage.init()
})
const seedVectorFamily = async () => {
await storage.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1]))
await storage.saveBinaryBlob('_system/vector-index/main.slotmap', Buffer.from([2]))
await storage.saveBinaryBlob('_system/vector-index/main.slotrev', Buffer.from([3]))
await storage.registerDerivedFamily({
name: 'vector-base',
members: [
'_system/vector-index/main.dkann',
'_system/vector-index/main.slotmap',
'_system/vector-index/main.slotrev'
]
})
}
it('a declared member is undeletable (throws ProtectedArtifactError)', async () => {
await seedVectorFamily()
await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf(
ProtectedArtifactError
)
// The blob is still there.
expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).not.toBeNull()
})
it('unregistering the family makes its members deletable again', async () => {
await seedVectorFamily()
await storage.unregisterDerivedFamily('vector-base')
await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).resolves.toBeUndefined()
expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).toBeNull()
})
it('an undeclared blob is still deletable (contract does not lock everything)', async () => {
await seedVectorFamily()
await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([9]))
await expect(storage.deleteBinaryBlob('graph-lsm/source/sstable-1')).resolves.toBeUndefined()
})
it('a transient (*.tmp.*) is deletable even if it sits under a protected namespace', async () => {
await storage.registerDerivedFamily({
name: 'vector-ns',
members: ['_system/vector-index/'],
namespace: true
})
await storage.saveBinaryBlob('_system/vector-index/main.dkann.tmp.123', Buffer.from([1]))
await expect(
storage.deleteBinaryBlob('_system/vector-index/main.dkann.tmp.123')
).resolves.toBeUndefined()
})
it('a namespace family protects a growing child key', async () => {
await storage.registerDerivedFamily({
name: 'vector-segments',
members: ['_system/vector-index/'],
namespace: true
})
await storage.saveBinaryBlob('_system/vector-index/seg-42', Buffer.from([7]))
await expect(storage.deleteBinaryBlob('_system/vector-index/seg-42')).rejects.toBeInstanceOf(
ProtectedArtifactError
)
})
it('checkDerivedFamiliesPresent() names a member deleted outside the write path', async () => {
await seedVectorFamily()
// Simulate an EXTERNAL deleter (bypasses the in-process refusal): drop a
// member straight from the underlying store.
;(storage as any).blobStore.delete('_system/vector-index/main.slotmap')
const incomplete = await storage.checkDerivedFamiliesPresent()
expect(incomplete).toHaveLength(1)
expect(incomplete[0].name).toBe('vector-base')
expect(incomplete[0].missing).toContain('_system/vector-index/main.slotmap')
expect(incomplete[0].rebuildable).toBe(true)
})
it('no families registered → deleteBinaryBlob behaves exactly as before (no enforcement)', async () => {
await storage.saveBinaryBlob('some/blob', Buffer.from([1]))
await expect(storage.deleteBinaryBlob('some/blob')).resolves.toBeUndefined()
})
})
describe('registered-blob family protection survives a reopen (FileSystemStorage)', () => {
let dir: string
beforeEach(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-regblob-'))
})
afterEach(() => fs.rmSync(dir, { recursive: true, force: true }))
it('a family declared in one session protects members after reopen', async () => {
const s1: any = new FileSystemStorage(dir)
await s1.init()
await s1.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1]))
await s1.registerDerivedFamily({ name: 'vector-base', members: ['_system/vector-index/main.dkann'] })
// Reopen — the registry is loaded from _system/derived-artifacts.json.
const s2: any = new FileSystemStorage(dir)
await s2.init()
expect(await s2.listDerivedFamilies()).toHaveLength(1)
await expect(s2.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf(
ProtectedArtifactError
)
// removeRawPrefix nuking the vector dir is also refused.
await expect(s2.removeRawPrefix('_blobs/_system/vector-index')).rejects.toBeInstanceOf(
ProtectedArtifactError
)
})
})