feat(8.0): brain.fillSubtypes migration helper + pre-RC1 gap closure

- brain.fillSubtypes(rules): idempotent subtype back-fill for pre-8.0 data.
  One rule per NounType/VerbType (literal default or per-entry function);
  fills only entries still missing a subtype through the public update()/
  updateRelation() paths; returns { scanned, filled, skipped, errors, byType }.
  Full unit suite in tests/unit/brainy/fill-subtypes.test.ts.
- Fix getNouns/getVerbs pagination hasMore (peek one past the window) —
  was permanently false, silently truncating every multi-page walk.
- find({ near }) without near.id now throws a teaching error instead of an
  opaque storage sharding failure; CLI --threshold without --near applies a
  plain score floor.
- CLI init/close audit: every one-shot command init()s, close()s, and exits
  explicitly; delete the unmaintained interactive REPL; replace the cloud-era
  storage subcommands with status/batch-delete; new types/validate commands.
- requireSubtype JSDoc now documents the 8.0 default-on contract; audit()
  recommendation points at fillSubtypes.
- Docs: data-storage-architecture rewritten to the real 8.0 on-disk layout;
  README storage section reflects filesystem+memory and snapshots; eli5 and
  SEMANTIC_VFS /as-of/ semantics corrected; internal tracker IDs and
  .strategy references scrubbed from published files.
This commit is contained in:
David Snelling 2026-06-11 10:42:34 -07:00
parent 9b0f4acd5b
commit c44678390e
30 changed files with 1517 additions and 3226 deletions

View file

@ -1528,10 +1528,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const { limit, offset = 0, filter } = options
const collectedNouns: HNSWNounWithMetadata[] = []
// Collect ONE item past the requested window so `hasMore` is decidable:
// stopping exactly at offset+limit cannot distinguish "page full, nothing
// after it" from "page full, more behind it" — which made hasMore
// permanently false and silently truncated every multi-page walk
// (audit/migrateField/fillSubtypes, graph verb-id recovery, count rebuilds).
const targetCount = offset + limit
const peekCount = targetCount + 1
// Iterate by shards (0x00-0xFF) instead of types
for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) {
for (let shard = 0; shard < 256 && collectedNouns.length < peekCount; shard++) {
const shardHex = shard.toString(16).padStart(2, '0')
const shardDir = `entities/nouns/${shardHex}`
@ -1539,7 +1545,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const nounFiles = await this.listCanonicalObjects(shardDir)
for (const nounPath of nounFiles) {
if (collectedNouns.length >= targetCount) break
if (collectedNouns.length >= peekCount) break
if (!nounPath.includes('/vectors.json')) continue
try {
@ -1597,7 +1603,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
}
}
// Apply pagination
// Apply pagination. The peeked extra item (if any) is dropped by the slice;
// its existence is exactly what makes hasMore true.
const paginatedNouns = collectedNouns.slice(offset, offset + limit)
const hasMore = collectedNouns.length > targetCount
@ -1647,7 +1654,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const { limit, offset = 0, filter } = options // cursor intentionally not extracted (not yet implemented)
const collectedVerbs: HNSWVerbWithMetadata[] = []
const targetCount = offset + limit // Early termination target
// Same peek-one-past-the-window strategy as getNounsWithPagination — see
// the comment there. Without the extra item, hasMore is undecidable and
// was permanently false (silent truncation of every multi-page walk).
const targetCount = offset + limit // Requested window end
const peekCount = targetCount + 1 // Early termination target (window + 1 peek)
// Prepare filter sets for efficient lookup
const filterVerbTypes = filter?.verbType
@ -1668,7 +1679,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
: null
// Iterate by shards (0x00-0xFF) instead of types - single pass!
for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) {
const shardHex = shard.toString(16).padStart(2, '0')
const shardDir = `entities/verbs/${shardHex}`
@ -1676,7 +1687,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const verbFiles = await this.listCanonicalObjects(shardDir)
for (const verbPath of verbFiles) {
if (collectedVerbs.length >= targetCount) break
if (collectedVerbs.length >= peekCount) break
if (!verbPath.includes('/vectors.json')) continue
try {
@ -1737,9 +1748,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
}
}
// Apply pagination (Efficient slicing after early termination)
// Apply pagination. The peeked extra item (if any) is dropped by the slice;
// its existence is exactly what makes hasMore true. `>` (not `>=`) keeps the
// exact-boundary case (total == offset+limit) from looping forever.
const paginatedVerbs = collectedVerbs.slice(offset, offset + limit)
const hasMore = collectedVerbs.length > targetCount // Fixed >= to > (was causing infinite loop)
const hasMore = collectedVerbs.length > targetCount
return {
items: paginatedVerbs,
@ -3235,8 +3248,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* Before this override the Uint32Array counters were only persisted
* on a heuristic schedule inside `saveNoun_internal` (first-of-type or
* every-100th), which left readers seeing stale counts after a clean
* writer flush the same failure mode that BR-FIND-WHERE-ZERO surfaced
* via `brain.stats()`.
* writer flush the same silent-stale failure mode that once produced
* zero counts from `brain.stats()`.
*/
public async flushCounts(): Promise<void> {
await super.flushCounts()