soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): brain.fillSubtypes migration helper + pre-RC1 gap closure

Browse source 
- brain.fillSubtypes(rules): idempotent subtype back-fill for pre-8.0 data.
  One rule per NounType/VerbType (literal default or per-entry function);
  fills only entries still missing a subtype through the public update()/
  updateRelation() paths; returns { scanned, filled, skipped, errors, byType }.
  Full unit suite in tests/unit/brainy/fill-subtypes.test.ts.
- Fix getNouns/getVerbs pagination hasMore (peek one past the window) —
  was permanently false, silently truncating every multi-page walk.
- find({ near }) without near.id now throws a teaching error instead of an
  opaque storage sharding failure; CLI --threshold without --near applies a
  plain score floor.
- CLI init/close audit: every one-shot command init()s, close()s, and exits
  explicitly; delete the unmaintained interactive REPL; replace the cloud-era
  storage subcommands with status/batch-delete; new types/validate commands.
- requireSubtype JSDoc now documents the 8.0 default-on contract; audit()
  recommendation points at fillSubtypes.
- Docs: data-storage-architecture rewritten to the real 8.0 on-disk layout;
  README storage section reflects filesystem+memory and snapshots; eli5 and
  SEMANTIC_VFS /as-of/ semantics corrected; internal tracker IDs and
  .strategy references scrubbed from published files.
This commit is contained in:
David Snelling 2026-06-11 10:42:34 -07:00
parent 9b0f4acd5b
commit c44678390e
30 changed files with 1517 additions and 3226 deletions
Download patch file Download diff file Expand all files Collapse all files

305
src/brainy.ts
View file

@ -94,7 +94,10 @@ import {
BatchResult,
BrainyConfig,
BrainyStats,
ScoreExplanation
ScoreExplanation,
FillSubtypeRule,
FillSubtypeRules,
FillSubtypesResult
} from './types/brainy.types.js'
import { NounType, VerbType, TypeUtils } from './types/graphTypes.js'
import { BrainyInterface } from './types/brainyInterface.js'
@ -2885,7 +2888,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* })
*
* @example Lock down management relationships
* brain.requireSubtype(VerbType.Manages, {
* brain.requireSubtype(VerbType.ReportsTo, {
* values: ['direct', 'dotted-line'],
* required: true
* })
@ -7330,10 +7333,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* @param subtype - Optional specific subtype string for O(1) point count
*
* @example
* brain.counts.byRelationshipSubtype(VerbType.Manages)
* brain.counts.byRelationshipSubtype(VerbType.ReportsTo)
* // → { direct: 12, 'dotted-line': 3 }
*
* brain.counts.byRelationshipSubtype(VerbType.Manages, 'direct')
* brain.counts.byRelationshipSubtype(VerbType.ReportsTo, 'direct')
* // → 12
*/
byRelationshipSubtype: (verb: VerbType, subtype?: string): number | Record<string, number> => {
@ -7361,7 +7364,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* `topSubtypes` for verbs.
*
* @example
* brain.counts.topRelationshipSubtypes(VerbType.Manages, 5)
* brain.counts.topRelationshipSubtypes(VerbType.ReportsTo, 5)
* // → [['direct', 12], ['dotted-line', 3]]
*/
topRelationshipSubtypes: (verb: VerbType, n: number = 10): Array<[string, number]> => {
@ -7533,7 +7536,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* @returns Sorted list of distinct subtype strings (empty if none)
*
* @example
* const variants = brain.relationshipSubtypesOf(VerbType.Manages)
* const variants = brain.relationshipSubtypesOf(VerbType.ReportsTo)
* // → ['direct', 'dotted-line']
*/
relationshipSubtypesOf(verb: VerbType): string[] {
@ -7550,17 +7553,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
/**
* Find entities and relationships missing a `subtype` value, grouped by type.
*
* The diagnostic pair to `migrateField()` / `fillSubtypes()` — answers the
* question "what would break if I enabled strict subtype enforcement?". Run
* this before adopting an SDK that registers `requireSubtype()` rules on
* common NounTypes, or before upgrading to Brainy 8.0 (which makes
* `requireSubtype: true` the default).
* The diagnostic pair to `fillSubtypes()` / `migrateField()` — answers the
* question "what would strict subtype enforcement reject?". 8.0 makes
* `requireSubtype: true` the default, so run this when opening a pre-8.0
* brain (with `requireSubtype: false` as the temporary escape hatch), then
* back-fill the reported gaps with `fillSubtypes(rules)`.
*
* Streams the brain via the same paginated `storage.getNouns()` /
* `storage.getVerbs()` pattern `migrateField()` uses — safe for large brains
* but linear in `O(N)`. Cortex 3.0+ may proxy this through the native
* column-store null-subtype bitmap for sub-linear performance on billion-scale
* brains (tracked in `CTX-SUBTYPE-8.0-CONTRACT`).
* `storage.getVerbs()` pattern `fillSubtypes()` uses — safe for large brains
* but linear in `O(N)`. A native index provider may serve this from a
* column-store null-subtype bitmap in the future for sub-linear performance
* on billion-scale brains.
*
* @param options.includeVFS - When `false` (default), entities marked with
* `metadata.isVFSEntity` or `metadata.isVFS` are excluded from the report —
@ -7570,7 +7573,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* @param options.onProgress - Optional progress callback invoked after each batch.
* @returns Report with per-type counts of entities/relationships without a
* subtype, plus the overall total and a one-line recommendation pointing at
* `migrateField()` (7.x) or `fillSubtypes()` (8.0).
* `fillSubtypes()`.
*
* @example Find pre-existing gaps before turning on strict mode
* const report = await brain.audit()
@ -7655,7 +7658,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const recommendation = missingSubtype === 0
? 'No subtype gaps detected — this brain is strict-mode-ready.'
: 'Found ' + missingSubtype + ' entries without subtype. Migrate via `brain.migrateField()` (7.x) — or wait for `brain.fillSubtypes()` (8.0) which closes the same gap with caller-supplied rules.'
: 'Found ' + missingSubtype + ' entries without subtype. Back-fill with `brain.fillSubtypes(rules)` — one rule per NounType/VerbType, literal default or per-entry function. To lift an existing field into `subtype` instead, use `brain.migrateField({ from, to: \'subtype\' })`.'
return {
entitiesWithoutSubtype,
@ -7666,6 +7669,262 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* Back-fill missing `subtype` values across the whole brain — the 8.0
* migration helper for data written before subtype became required.
*
* 8.0 enforces a non-empty `subtype` on every write by default
* (`requireSubtype: true`). A brain created on 7.x typically carries entities
* and relationships without one; this method clears that debt in a single
* idempotent pass so the opt-out (`requireSubtype: false`) can be removed.
* The intended upgrade flow:
*
* 1. Open the brain with `requireSubtype: false` (temporary escape hatch).
* 2. `await brain.audit()` — see what's missing, grouped by type.
* 3. `await brain.fillSubtypes(rules)` — back-fill with one rule per type.
* 4. Re-run `audit()` until `total === 0`, then drop the opt-out.
*
* **Rules.** One rule per NounType (entities) and/or VerbType
* (relationships) — the two vocabularies don't overlap, so a single map
* covers both sides. A rule is either a literal subtype string (blanket
* default) or a function deriving the subtype from the entry; functions
* subsume `where`-style filtering by returning `undefined` for entries they
* decline (those stay untouched and count as `skipped`, so a later run with
* a stricter rule can pick them up).
*
* **What is never touched:** entries that already carry a non-empty
* `subtype` (re-running is a no-op on them), and — unless
* `includeVFS: true` — Brainy's own VFS infrastructure entries
* (`metadata.isVFSEntity` / `metadata.isVFS`), which bypass enforcement
* anyway and are not migration debt.
*
* **Write strategy (deliberate):** each fill goes through the public
* `update()` / `updateRelation()` paths, so every write is individually
* atomic (storage record + indexes + subtype rollups commit together) and
* bumps the entry's `_rev` like any other update. The pass is *not* one
* whole-brain transaction: a migration over millions of entries inside a
* single transaction would hold an unbounded working set and turn one bad
* entry into an all-or-nothing failure. Idempotence is the recovery model —
* a crashed or partially-failed run is resumed safely by re-running, because
* only entries still missing a subtype are written.
*
* Streams via the same paginated `storage.getNouns()` / `storage.getVerbs()`
* walk `audit()` uses — `O(N)` but constant memory. The noun pass is skipped
* entirely when the map has no NounType rules, and vice versa.
*
* @param rules - Map of NounType/VerbType → literal subtype or rule function.
* Must contain at least one valid type key; invalid keys, empty-string
* literals, and non-string/non-function values throw before any data is
* touched.
* @param options.includeVFS - Also fill VFS infrastructure entries (default
* `false` — they bypass enforcement and don't need a subtype).
* @param options.batchSize - Pagination batch size (default `200`).
* @param options.onProgress - Optional callback invoked after each batch.
* @returns `{ scanned, filled, skipped, errors, byType }` — see
* {@link FillSubtypesResult}. After a clean run, `skipped` equals the
* remaining `audit().total`.
* @throws If the brain is read-only, the rule map is empty/malformed, or a
* key is not a valid NounType/VerbType. Per-entry write failures do NOT
* throw — they are collected in `errors` and the pass continues.
*
* @example Back-fill entities and relationships in one pass
* const report = await brain.fillSubtypes({
* [NounType.Person]: (e) => e.metadata?.kind ?? 'unspecified',
* [NounType.Document]: 'general',
* [VerbType.RelatedTo]: 'unspecified'
* })
* // → { scanned: 5200, filled: 1429, skipped: 0, errors: [], byType: { person: 800, document: 600, relatedTo: 29 } }
*
* @example Selective fill — decline entries a rule can't classify
* await brain.fillSubtypes({
* [NounType.Person]: (e) => e.metadata?.department ? 'employee' : undefined
* })
* // Persons without a department stay untouched (counted as skipped).
*
* @since 8.0.0
*/
async fillSubtypes(
rules: FillSubtypeRules<T>,
options: {
includeVFS?: boolean
batchSize?: number
onProgress?: (progress: { scanned: number; filled: number; skipped: number }) => void
} = {}
): Promise<FillSubtypesResult> {
this.assertWritable('fillSubtypes')
await this.ensureInitialized()
// Validate the rule map up front — fail fast on shape errors before any
// data is touched.
if (!rules || typeof rules !== 'object' || Array.isArray(rules)) {
throw new Error(
'fillSubtypes: rules must be a map of NounType/VerbType → subtype string or rule function'
)
}
const nounTypeValues = new Set<string>(Object.values(NounType))
const verbTypeValues = new Set<string>(Object.values(VerbType))
const nounRules = new Map<string, FillSubtypeRule<Entity<T>>>()
const verbRules = new Map<string, FillSubtypeRule<Relation<T>>>()
for (const [key, rule] of Object.entries(rules)) {
if (rule === undefined) continue
if (typeof rule !== 'string' && typeof rule !== 'function') {
throw new Error(
`fillSubtypes: rule for '${key}' must be a subtype string or a function (got ${typeof rule})`
)
}
if (typeof rule === 'string' && rule.length === 0) {
throw new Error(
`fillSubtypes: rule for '${key}' is an empty string — a subtype must be non-empty`
)
}
if (nounTypeValues.has(key)) {
nounRules.set(key, rule as FillSubtypeRule<Entity<T>>)
} else if (verbTypeValues.has(key)) {
verbRules.set(key, rule as FillSubtypeRule<Relation<T>>)
} else {
throw new Error(`fillSubtypes: '${key}' is not a valid NounType or VerbType`)
}
}
if (nounRules.size === 0 && verbRules.size === 0) {
throw new Error(
'fillSubtypes: rules map is empty — provide at least one NounType or VerbType rule'
)
}
const includeVFS = options.includeVFS === true
const batchSize = Math.max(1, options.batchSize ?? 200)
let scanned = 0
let filled = 0
let skipped = 0
const errors: Array<{ id: string; error: string }> = []
const byType: Record<string, number> = {}
const reportProgress = (): void => {
if (options.onProgress) options.onProgress({ scanned, filled, skipped })
}
// Normalize a rule's output: only a non-empty string is a fill; anything
// else (undefined, empty string) is a decline. An empty subtype would fail
// the very strict-mode check this helper exists to satisfy.
const normalize = (value: string | undefined): string | undefined =>
typeof value === 'string' && value.length > 0 ? value : undefined
// Pass 1: entities (skipped entirely when the map has no NounType rules).
if (nounRules.size > 0) {
let offset = 0
while (true) {
const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } })
if (page.items.length === 0) break
for (const noun of page.items) {
scanned++
// VFS infrastructure entries bypass enforcement via their marker, so
// they're not migration debt — leave them alone unless asked.
if (
!includeVFS &&
(noun.metadata?.isVFSEntity === true || noun.metadata?.isVFS === true)
) {
continue
}
if (typeof noun.subtype === 'string' && noun.subtype.length > 0) continue // already filled — never overwrite
const rule = nounRules.get(noun.type)
if (rule === undefined) {
skipped++
continue
}
try {
let subtype: string | undefined
if (typeof rule === 'function') {
const entity = await this.convertNounToEntity(noun)
subtype = normalize(rule(entity))
} else {
subtype = rule
}
if (subtype === undefined) {
skipped++
continue
}
await this.update({ id: noun.id, subtype })
filled++
byType[noun.type] = (byType[noun.type] || 0) + 1
} catch (err) {
errors.push({
id: noun.id,
error: err instanceof Error ? err.message : String(err)
})
}
}
reportProgress()
if (!page.hasMore) break
offset += page.items.length
}
}
// Pass 2: relationships (skipped entirely when the map has no VerbType rules).
if (verbRules.size > 0) {
let offset = 0
while (true) {
const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } })
if (page.items.length === 0) break
for (const verb of page.items) {
scanned++
if (
!includeVFS &&
(verb.metadata?.isVFSEntity === true || verb.metadata?.isVFS === true)
) {
continue
}
if (typeof verb.subtype === 'string' && verb.subtype.length > 0) continue // already filled — never overwrite
const rule = verbRules.get(verb.verb)
if (rule === undefined) {
skipped++
continue
}
try {
let subtype: string | undefined
if (typeof rule === 'function') {
// Project the stored verb onto the public Relation<T> shape the
// rule function is typed against.
const relation: Relation<T> = {
id: verb.id,
from: verb.sourceId,
to: verb.targetId,
type: verb.verb,
weight: verb.weight,
data: verb.data,
metadata: verb.metadata as T,
service: verb.service,
createdAt: verb.createdAt,
updatedAt: verb.updatedAt,
confidence: verb.confidence
}
subtype = normalize(rule(relation))
} else {
subtype = rule
}
if (subtype === undefined) {
skipped++
continue
}
await this.updateRelation({ id: verb.id, subtype })
filled++
byType[verb.verb] = (byType[verb.verb] || 0) + 1
} catch (err) {
errors.push({
id: verb.id,
error: err instanceof Error ? err.message : String(err)
})
}
}
reportProgress()
if (!page.hasMore) break
offset += page.items.length
}
}
return { scanned, filled, skipped, errors, byType }
}
/**
* Stream-and-rewrite a field across every entity in the brain.
*
@ -8845,6 +9104,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
private async executeProximitySearch(params: FindParams<T>): Promise<Result<T>[]> {
if (!params.near) return []
// Teaching error: without an anchor id the constraint is meaningless, and
// letting it fall through produces an opaque storage-layer sharding error.
if (!params.near.id) {
throw new Error(
"find({ near }): 'near.id' is required — pass the entity to search around, " +
'e.g. near: { id, threshold }. To impose a minimum score on plain semantic ' +
'results, filter on result.score instead.'
)
}
const nearEntity = await this.get(params.near.id)
if (!nearEntity) return []
@ -9680,7 +9949,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
*/
private normalizeConfig(config?: BrainyConfig): Required<BrainyConfig> {
// Validate storage configuration. Brainy 8.0 ships two adapters only —
// FileSystemStorage and MemoryStorage — per BR-BRAINY-80-STORAGE-SIMPLIFY.
// FileSystemStorage and MemoryStorage (cloud + OPFS adapters were removed).
// Cloud backup remains supported via operator tooling (db.persist() +
// gsutil / aws s3 cp / rclone / azcopy). Pre-constructed adapter
// instances bypass the type check (they ARE the storage).