feat(8.0): brain.fillSubtypes migration helper + pre-RC1 gap closure
- brain.fillSubtypes(rules): idempotent subtype back-fill for pre-8.0 data.
One rule per NounType/VerbType (literal default or per-entry function);
fills only entries still missing a subtype through the public update()/
updateRelation() paths; returns { scanned, filled, skipped, errors, byType }.
Full unit suite in tests/unit/brainy/fill-subtypes.test.ts.
- Fix getNouns/getVerbs pagination hasMore (peek one past the window) —
was permanently false, silently truncating every multi-page walk.
- find({ near }) without near.id now throws a teaching error instead of an
opaque storage sharding failure; CLI --threshold without --near applies a
plain score floor.
- CLI init/close audit: every one-shot command init()s, close()s, and exits
explicitly; delete the unmaintained interactive REPL; replace the cloud-era
storage subcommands with status/batch-delete; new types/validate commands.
- requireSubtype JSDoc now documents the 8.0 default-on contract; audit()
recommendation points at fillSubtypes.
- Docs: data-storage-architecture rewritten to the real 8.0 on-disk layout;
README storage section reflects filesystem+memory and snapshots; eli5 and
SEMANTIC_VFS /as-of/ semantics corrected; internal tracker IDs and
.strategy references scrubbed from published files.
This commit is contained in:
parent
9b0f4acd5b
commit
c44678390e
30 changed files with 1517 additions and 3226 deletions
|
|
@ -1528,10 +1528,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
const { limit, offset = 0, filter } = options
|
||||
const collectedNouns: HNSWNounWithMetadata[] = []
|
||||
// Collect ONE item past the requested window so `hasMore` is decidable:
|
||||
// stopping exactly at offset+limit cannot distinguish "page full, nothing
|
||||
// after it" from "page full, more behind it" — which made hasMore
|
||||
// permanently false and silently truncated every multi-page walk
|
||||
// (audit/migrateField/fillSubtypes, graph verb-id recovery, count rebuilds).
|
||||
const targetCount = offset + limit
|
||||
const peekCount = targetCount + 1
|
||||
|
||||
// Iterate by shards (0x00-0xFF) instead of types
|
||||
for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) {
|
||||
for (let shard = 0; shard < 256 && collectedNouns.length < peekCount; shard++) {
|
||||
const shardHex = shard.toString(16).padStart(2, '0')
|
||||
const shardDir = `entities/nouns/${shardHex}`
|
||||
|
||||
|
|
@ -1539,7 +1545,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const nounFiles = await this.listCanonicalObjects(shardDir)
|
||||
|
||||
for (const nounPath of nounFiles) {
|
||||
if (collectedNouns.length >= targetCount) break
|
||||
if (collectedNouns.length >= peekCount) break
|
||||
if (!nounPath.includes('/vectors.json')) continue
|
||||
|
||||
try {
|
||||
|
|
@ -1597,7 +1603,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// Apply pagination
|
||||
// Apply pagination. The peeked extra item (if any) is dropped by the slice;
|
||||
// its existence is exactly what makes hasMore true.
|
||||
const paginatedNouns = collectedNouns.slice(offset, offset + limit)
|
||||
const hasMore = collectedNouns.length > targetCount
|
||||
|
||||
|
|
@ -1647,7 +1654,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
const { limit, offset = 0, filter } = options // cursor intentionally not extracted (not yet implemented)
|
||||
const collectedVerbs: HNSWVerbWithMetadata[] = []
|
||||
const targetCount = offset + limit // Early termination target
|
||||
// Same peek-one-past-the-window strategy as getNounsWithPagination — see
|
||||
// the comment there. Without the extra item, hasMore is undecidable and
|
||||
// was permanently false (silent truncation of every multi-page walk).
|
||||
const targetCount = offset + limit // Requested window end
|
||||
const peekCount = targetCount + 1 // Early termination target (window + 1 peek)
|
||||
|
||||
// Prepare filter sets for efficient lookup
|
||||
const filterVerbTypes = filter?.verbType
|
||||
|
|
@ -1668,7 +1679,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
: null
|
||||
|
||||
// Iterate by shards (0x00-0xFF) instead of types - single pass!
|
||||
for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
|
||||
for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) {
|
||||
const shardHex = shard.toString(16).padStart(2, '0')
|
||||
const shardDir = `entities/verbs/${shardHex}`
|
||||
|
||||
|
|
@ -1676,7 +1687,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const verbFiles = await this.listCanonicalObjects(shardDir)
|
||||
|
||||
for (const verbPath of verbFiles) {
|
||||
if (collectedVerbs.length >= targetCount) break
|
||||
if (collectedVerbs.length >= peekCount) break
|
||||
if (!verbPath.includes('/vectors.json')) continue
|
||||
|
||||
try {
|
||||
|
|
@ -1737,9 +1748,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// Apply pagination (Efficient slicing after early termination)
|
||||
// Apply pagination. The peeked extra item (if any) is dropped by the slice;
|
||||
// its existence is exactly what makes hasMore true. `>` (not `>=`) keeps the
|
||||
// exact-boundary case (total == offset+limit) from looping forever.
|
||||
const paginatedVerbs = collectedVerbs.slice(offset, offset + limit)
|
||||
const hasMore = collectedVerbs.length > targetCount // Fixed >= to > (was causing infinite loop)
|
||||
const hasMore = collectedVerbs.length > targetCount
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
|
|
@ -3235,8 +3248,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* Before this override the Uint32Array counters were only persisted
|
||||
* on a heuristic schedule inside `saveNoun_internal` (first-of-type or
|
||||
* every-100th), which left readers seeing stale counts after a clean
|
||||
* writer flush — the same failure mode that BR-FIND-WHERE-ZERO surfaced
|
||||
* via `brain.stats()`.
|
||||
* writer flush — the same silent-stale failure mode that once produced
|
||||
* zero counts from `brain.stats()`.
|
||||
*/
|
||||
public async flushCounts(): Promise<void> {
|
||||
await super.flushCounts()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue