feat: enforce data/metadata separation, numeric range queries, improved docs

- Store data opaquely in add() and update() instead of spreading object
  properties into top-level metadata. data is for semantic search (HNSW),
  metadata is for structured where-filter queries (MetadataIndex).
- Fix numeric range queries in MetadataIndex — use numeric-aware comparison
  instead of lexicographic string comparison for normalized values.
- Add data field to RelateParams and Relation types for relationship content.
- Add where.type → where.noun alias in metadata-only find() path.
- Rewrite README: focused ~350 lines from 791, quick start first, feature
  showcase with mini-snippets, organized doc links, no version callouts.
- Add DATA_MODEL.md and QUERY_OPERATORS.md reference docs.
- Remove 10 outdated/redundant doc files consolidated into API reference.
- Improve JSDoc on Entity, Relation, AddParams, FindParams, and core methods.
- Fix tests asserting data properties appear in metadata (data model violation).
- Deprecate verb.source/target in favor of from/to (public) and sourceId/targetId (storage).
This commit is contained in:
David Snelling 2026-02-09 12:06:59 -08:00
parent edb5ec4696
commit 0ddc05a5bb
30 changed files with 1356 additions and 7001 deletions

View file

@ -21,7 +21,8 @@ import {
AdaptiveChunkingStrategy,
ChunkData,
ChunkDescriptor,
ZoneMap
ZoneMap,
compareNormalizedValues
} from './metadataIndexChunking.js'
import { EntityIdMapper } from './entityIdMapper.js'
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
@ -852,15 +853,18 @@ export class MetadataIndexManager {
const chunk = await this.chunkManager.loadChunk(field, chunkId)
if (chunk) {
for (const [value, bitmap] of chunk.entries) {
// Check if value is in range (both value and normalized bounds are now bucketed)
// Check if value is in range using numeric-aware comparison
// (normalizeValue converts numbers to strings, so we must compare numerically)
let inRange = true
if (normalizedMin !== undefined) {
inRange = inRange && (includeMin ? value >= normalizedMin : value > normalizedMin)
const cmp = compareNormalizedValues(value, normalizedMin)
inRange = inRange && (includeMin ? cmp >= 0 : cmp > 0)
}
if (normalizedMax !== undefined) {
inRange = inRange && (includeMax ? value <= normalizedMax : value < normalizedMax)
const cmp = compareNormalizedValues(value, normalizedMax)
inRange = inRange && (includeMax ? cmp <= 0 : cmp < 0)
}
if (inRange) {