feat: registered-blob family contract — declared index blobs are undeletable (ADR-004 Pass 2)
The class-killer for the lost-main.dkann incident (ADR-004 §7). A provider can declare a derived-index blob FAMILY (a set of members that are load-bearing together, e.g. vector-base = main.dkann + main.slotmap + main.slotrev). Once declared: - deleteBinaryBlob / removeRawPrefix REFUSE to remove a declared member, throwing the new ProtectedArtifactError — an in-process GC / sweeper is now INCAPABLE of deleting a load-bearing index file (COLD != DEAD). Intentional retirement is an explicit unregisterDerivedFamily(name) first. - The declaration persists to _system/derived-artifacts.json, so protection survives a reopen; clear() resets it with the rest of the derived footprint. - checkDerivedFamiliesPresent() names any member missing on open (the catch for an EXTERNAL deleter that bypasses the in-process refusal) → rebuild from canonical. - Transients (*.tmp.*, *.rebuild-tmp, *.rotate-tmp) are never protected; a namespace family protects a growing prefix (seg-*). New StorageAdapter surface (optional): registerDerivedFamily / unregisterDerivedFamily / listDerivedFamilies + DerivedFamilyDeclaration; new exported errors ProtectedArtifactError / DerivedArtifactMissingError. Enforcement is inert until a provider declares a family (no regression). Cor declares its 6 families + does the atomic set-swap in 3.0.15 (M2); brainy builds the contract now. 8 tests.
This commit is contained in:
parent
6bcb54f0d9
commit
bfa1762107
7 changed files with 429 additions and 5 deletions
|
|
@ -15,7 +15,8 @@ import {
|
|||
VerbMetadata,
|
||||
HNSWNounWithMetadata,
|
||||
HNSWVerbWithMetadata,
|
||||
StatisticsData
|
||||
StatisticsData,
|
||||
DerivedFamilyDeclaration
|
||||
} from '../coreTypes.js'
|
||||
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
|
||||
import { validateNounType, validateVerbType } from '../utils/typeValidation.js'
|
||||
|
|
@ -31,7 +32,7 @@ import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js'
|
|||
import { unwrapBinaryData } from './binaryDataCodec.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
import { isAbsentError } from '../utils/errorClassification.js'
|
||||
import { BrainyError } from '../errors/brainyError.js'
|
||||
import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js'
|
||||
import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js'
|
||||
import {
|
||||
splitNounMetadataRecord,
|
||||
|
|
@ -251,6 +252,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
protected isInitialized = false
|
||||
/** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
|
||||
private _graphFastPathProbed = false
|
||||
/**
|
||||
* Registered-blob contract (ADR-004 §7): declared derived-index families, keyed
|
||||
* by name. Members are undeletable through the blob delete seams. Loaded lazily
|
||||
* from `_system/derived-artifacts.json` and re-persisted on every change.
|
||||
*/
|
||||
private _derivedFamilies = new Map<string, DerivedFamilyDeclaration>()
|
||||
/** One-shot guard for loading the persisted family registry. */
|
||||
private _derivedFamiliesLoaded = false
|
||||
/** Storage-root-relative path of the persisted family registry. */
|
||||
private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json'
|
||||
protected graphIndex?: GraphAdjacencyIndex
|
||||
protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
|
||||
/**
|
||||
|
|
@ -1041,6 +1052,154 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
await this.writeObjectToPath(path, data)
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Registered-blob contract (ADR-004 §7) — declared derived-index families are
|
||||
// undeletable through the blob delete seams. Shared here so every adapter that
|
||||
// extends BaseStorage inherits the same enforcement; the concrete
|
||||
// deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable /
|
||||
// assertPrefixNotProtected before removing anything.
|
||||
// ==========================================================================
|
||||
|
||||
/** Load the persisted family registry once (lazy). */
|
||||
private async ensureDerivedFamiliesLoaded(): Promise<void> {
|
||||
if (this._derivedFamiliesLoaded) return
|
||||
const stored = await this.readRawObject(BaseStorage.DERIVED_FAMILIES_KEY).catch(() => null)
|
||||
const families = (stored as { families?: DerivedFamilyDeclaration[] } | null)?.families
|
||||
if (Array.isArray(families)) {
|
||||
for (const f of families) {
|
||||
if (f && typeof f.name === 'string' && Array.isArray(f.members)) this._derivedFamilies.set(f.name, f)
|
||||
}
|
||||
}
|
||||
this._derivedFamiliesLoaded = true
|
||||
}
|
||||
|
||||
/** Persist the current family registry (fsync'd via the raw-object write). */
|
||||
private async persistDerivedFamilies(): Promise<void> {
|
||||
await this.writeRawObject(BaseStorage.DERIVED_FAMILIES_KEY, {
|
||||
families: [...this._derivedFamilies.values()]
|
||||
})
|
||||
}
|
||||
|
||||
public async registerDerivedFamily(family: DerivedFamilyDeclaration): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
await this.ensureDerivedFamiliesLoaded()
|
||||
this._derivedFamilies.set(family.name, {
|
||||
name: family.name,
|
||||
members: [...family.members],
|
||||
...(family.namespace !== undefined ? { namespace: family.namespace } : {}),
|
||||
...(family.rebuildable !== undefined ? { rebuildable: family.rebuildable } : {})
|
||||
})
|
||||
await this.persistDerivedFamilies()
|
||||
}
|
||||
|
||||
public async unregisterDerivedFamily(name: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
await this.ensureDerivedFamiliesLoaded()
|
||||
if (this._derivedFamilies.delete(name)) await this.persistDerivedFamilies()
|
||||
}
|
||||
|
||||
public async listDerivedFamilies(): Promise<DerivedFamilyDeclaration[]> {
|
||||
await this.ensureInitialized()
|
||||
await this.ensureDerivedFamiliesLoaded()
|
||||
return [...this._derivedFamilies.values()]
|
||||
}
|
||||
|
||||
/**
|
||||
* @description A blob key that is a transient write-scratch file (a `*.tmp.*`
|
||||
* temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as
|
||||
* part of an atomic write; it is never a protected family member (ADR-004 §7).
|
||||
*/
|
||||
private isTransientBlobKey(key: string): boolean {
|
||||
return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The name of the protected family a blob key belongs to, or
|
||||
* `null`. A `namespace` member protects every key beneath it; a plain member
|
||||
* protects that exact key.
|
||||
*/
|
||||
private protectingFamilyOf(key: string): string | null {
|
||||
for (const family of this._derivedFamilies.values()) {
|
||||
for (const member of family.members) {
|
||||
if (family.namespace ? key === member || key.startsWith(member) : key === member) {
|
||||
return family.name
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Enforcement point for `deleteBinaryBlob`: refuse (throw
|
||||
* {@link ProtectedArtifactError}) when the key is a declared family member.
|
||||
* Transients pass through. An undeclared blob delete under an ACTIVE contract
|
||||
* (families are registered) is logged loudly — nothing under `_blobs/` should
|
||||
* vanish unremarked once the contract is in force.
|
||||
*/
|
||||
protected async assertBlobKeyDeletable(key: string): Promise<void> {
|
||||
await this.ensureDerivedFamiliesLoaded()
|
||||
if (this._derivedFamilies.size === 0) return // contract inactive — no enforcement
|
||||
if (this.isTransientBlobKey(key)) return
|
||||
const family = this.protectingFamilyOf(key)
|
||||
if (family) throw new ProtectedArtifactError(key, family)
|
||||
prodLog.warn(
|
||||
`[BaseStorage] deleteBinaryBlob('${key}') removes an UNDECLARED blob while the ` +
|
||||
`registered-blob contract is active — permitted, but surfaced so no _blobs/ file ` +
|
||||
`disappears silently.`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Enforcement point for `removeRawPrefix`: refuse when the prefix
|
||||
* would take out a protected family member (a prefix-nuke must not remove a
|
||||
* declared blob). Transients are ignored.
|
||||
*/
|
||||
protected async assertPrefixNotProtected(prefix: string): Promise<void> {
|
||||
await this.ensureDerivedFamiliesLoaded()
|
||||
if (this._derivedFamilies.size === 0) return
|
||||
for (const family of this._derivedFamilies.values()) {
|
||||
for (const member of family.members) {
|
||||
// Under the shared `_blobs/` root, member keys resolve beneath it; a
|
||||
// prefix intersects a member when either contains the other.
|
||||
const memberBlobPath = `_blobs/${member}`
|
||||
if (
|
||||
memberBlobPath.startsWith(prefix) ||
|
||||
prefix.startsWith(memberBlobPath) ||
|
||||
member.startsWith(prefix) ||
|
||||
prefix.startsWith(member)
|
||||
) {
|
||||
if (!this.isTransientBlobKey(member)) throw new ProtectedArtifactError(member, family.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Verify every declared family has all its members present on
|
||||
* disk (ADR-004 §7 missing-on-open catch for an EXTERNAL deleter that bypasses
|
||||
* the in-process refusal). Returns the incomplete families (name + missing
|
||||
* members) — the caller decides how to heal (rebuild from canonical). Loud by
|
||||
* construction: a missing load-bearing blob is named, never silently tolerated.
|
||||
*/
|
||||
public async checkDerivedFamiliesPresent(): Promise<Array<{ name: string; missing: string[]; rebuildable: boolean }>> {
|
||||
await this.ensureInitialized()
|
||||
await this.ensureDerivedFamiliesLoaded()
|
||||
const incomplete: Array<{ name: string; missing: string[]; rebuildable: boolean }> = []
|
||||
for (const family of this._derivedFamilies.values()) {
|
||||
if (family.namespace) continue // a growing prefix has no fixed member set to verify
|
||||
const missing: string[] = []
|
||||
for (const member of family.members) {
|
||||
const blob = await this.loadBinaryBlob(member).catch(() => null)
|
||||
if (!blob) missing.push(member)
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
prodLog.warn(new DerivedArtifactMissingError(family.name, missing).message)
|
||||
incomplete.push({ name: family.name, missing, rebuildable: family.rebuildable !== false })
|
||||
}
|
||||
}
|
||||
return incomplete
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a raw object at a storage-root-relative path (no-op if absent).
|
||||
*
|
||||
|
|
@ -1224,6 +1383,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
*/
|
||||
protected async reloadDerivedState(): Promise<void> {
|
||||
this.clearWriteCache()
|
||||
// Registered-blob registry: clear() wipes the persisted `_system/` copy and
|
||||
// the family members under `_blobs/`, so drop the in-memory cache too — a
|
||||
// reset brain must not keep stale family protection or report their members
|
||||
// "missing" on the next check.
|
||||
this._derivedFamilies.clear()
|
||||
this._derivedFamiliesLoaded = false
|
||||
this.nounCountsByType.fill(0)
|
||||
this.verbCountsByType.fill(0)
|
||||
this.subtypeCountsByType.clear()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue