feat: subtype top-level field + trackField + migrateField
Promotes `subtype?: string` to a top-level standard field on every entity,
alongside `type` / `confidence` / `weight`. Flat string, no hierarchy — the
consumer-chosen vocabulary for sub-classifying entities within a NounType
(Person → employee/customer, Document → invoice/contract, etc.).
Layer 1 — subtype field + rollup
- HNSWNounWithMetadata.subtype + STANDARD_ENTITY_FIELDS entry
- Entity / Result / AddParams / UpdateParams / FindParams threading
- add()/update() persist subtype on storageMetadata + entityForIndexing
- get()/find() route through the standard-field fast path
- subtypeCountsByType (Map<NounTypeIdx, Map<subtype, count>>) on
BaseStorage, mirrored after nounCountsByType with the same self-heal
rebuild and persisted to _system/subtype-statistics.json
- brain.counts.bySubtype(type, subtype?) — O(1) point + breakdown
- brain.counts.topSubtypes(type, n) — top-N by count
- brain.subtypesOf(type) — distinct subtypes seen
- find({ type, subtype }) and find({ subtype: ['a','b'] }) on the fast path
Layer 2 — trackField for other facets
- brain.trackField(name, { perType?, values? }) registers a field for
cardinality + per-NounType breakdown stats. Backed by the aggregation
engine (auto-defines __fieldCounts__<name>), backfill-on-define applies.
- brain.counts.byField(name, { type? }) returns value frequencies
- Optional vocabulary whitelist rejects off-vocabulary writes at add/update
Layer 3 — generic migrateField
- brain.migrateField({ from, to, readBoth?, batchSize?, onProgress? })
streams every entity, copies the value from one path to another, and
(unless readBoth) clears the source. Supports top-level standard fields,
metadata.X, and data.X paths. Idempotent — safe to re-run.
Docs
- New guide: docs/guides/subtypes-and-facets.md (Layer 1 + 2 + 3)
- README, DATA_MODEL, QUERY_OPERATORS, api/README, finite-type-system,
quick-start all treat subtype as a core primitive with anonymous example
vocabularies (employee/customer/invoice/milestone).
Tests
- 26 new integration tests covering write/read/update/delete round-trips,
counts rollup decrement + re-route on mutation, trackField + byField
with and without perType, vocabulary whitelist enforcement, and
migrateField for metadata.X → subtype and data.X → subtype paths
including readBoth deprecation-window semantics.
Unit suite: 1468/1468 passing. Type-check + build clean.
This commit is contained in:
parent
e47fea0917
commit
2cdf70ee0f
14 changed files with 1698 additions and 21 deletions
|
|
@ -26,6 +26,13 @@ export interface Entity<T = any> {
|
|||
vector: Vector
|
||||
/** Entity type classification (NounType enum) */
|
||||
type: NounType
|
||||
/**
|
||||
* Per-product sub-classification within the NounType (e.g. a `Person` entity might
|
||||
* have `subtype: 'employee'` or `'customer'`; a `Document` might have `subtype: 'invoice'`).
|
||||
* Flat string, no hierarchy. Top-level standard field — indexed on the fast path and
|
||||
* rolled into per-NounType statistics.
|
||||
*/
|
||||
subtype?: string
|
||||
/** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */
|
||||
data?: any
|
||||
/** User-defined structured fields — indexed and queryable via `where` filters. */
|
||||
|
|
@ -105,6 +112,7 @@ export interface Result<T = any> {
|
|||
|
||||
// Convenience: Common entity fields flattened to top level
|
||||
type?: NounType // Entity type (from entity.type)
|
||||
subtype?: string // Per-product sub-classification (from entity.subtype)
|
||||
metadata?: T // Entity metadata (from entity.metadata)
|
||||
data?: any // Entity data (from entity.data)
|
||||
confidence?: number // Type classification confidence (from entity.confidence)
|
||||
|
|
@ -158,6 +166,14 @@ export interface AddParams<T = any> {
|
|||
data: any | Vector
|
||||
/** Entity type classification (required) */
|
||||
type: NounType
|
||||
/**
|
||||
* Per-product sub-classification within the NounType (e.g. a `Person` entity might have
|
||||
* `subtype: 'employee'` or `'customer'`; a `Document` might have `subtype: 'invoice'`).
|
||||
* Flat string, no hierarchy — consumers choose the vocabulary. Indexed and rolled up into
|
||||
* per-NounType statistics for fast `find({ type, subtype })` filtering and `groupBy:['subtype']`
|
||||
* aggregation on the standard-field fast path.
|
||||
*/
|
||||
subtype?: string
|
||||
/** Structured queryable fields — indexed by MetadataIndex, used in `where` filters */
|
||||
metadata?: T
|
||||
/** Custom entity ID (auto-generated UUID v4 if not provided) */
|
||||
|
|
@ -181,6 +197,7 @@ export interface UpdateParams<T = any> {
|
|||
id: string // Entity to update
|
||||
data?: any // New content to re-embed
|
||||
type?: NounType // Change type
|
||||
subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work)
|
||||
metadata?: Partial<T> // Metadata to update
|
||||
merge?: boolean // Merge or replace metadata (default: true)
|
||||
vector?: Vector // New pre-computed vector
|
||||
|
|
@ -252,6 +269,13 @@ export interface FindParams<T = any> {
|
|||
// Metadata Intelligence
|
||||
/** Filter by entity type(s). Alias for `where.noun`. */
|
||||
type?: NounType | NounType[]
|
||||
/**
|
||||
* Filter by per-product subtype (top-level standard field — uses the fast path, not the
|
||||
* metadata fallback). Pass a single string for equality, or an array for set membership
|
||||
* (e.g. `subtype: ['employee', 'contractor']`). For operator-form predicates (e.g.
|
||||
* `{ exists: true }`, `{ missing: true }`) use `where: { subtype: { …operators… } }`.
|
||||
*/
|
||||
subtype?: string | string[]
|
||||
/** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */
|
||||
where?: Partial<T>
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue