feat: enforce data/metadata separation, numeric range queries, improved docs

- Store data opaquely in add() and update() instead of spreading object
  properties into top-level metadata. data is for semantic search (HNSW),
  metadata is for structured where-filter queries (MetadataIndex).
- Fix numeric range queries in MetadataIndex — use numeric-aware comparison
  instead of lexicographic string comparison for normalized values.
- Add data field to RelateParams and Relation types for relationship content.
- Add where.type → where.noun alias in metadata-only find() path.
- Rewrite README: focused ~350 lines from 791, quick start first, feature
  showcase with mini-snippets, organized doc links, no version callouts.
- Add DATA_MODEL.md and QUERY_OPERATORS.md reference docs.
- Remove 10 outdated/redundant doc files consolidated into API reference.
- Improve JSDoc on Entity, Relation, AddParams, FindParams, and core methods.
- Fix tests asserting data properties appear in metadata (data model violation).
- Deprecate verb.source/target in favor of from/to (public) and sourceId/targetId (storage).
This commit is contained in:
David Snelling 2026-02-09 12:06:59 -08:00
parent edb5ec4696
commit 0ddc05a5bb
30 changed files with 1356 additions and 7001 deletions

View file

@ -21,7 +21,8 @@ import {
AdaptiveChunkingStrategy,
ChunkData,
ChunkDescriptor,
ZoneMap
ZoneMap,
compareNormalizedValues
} from './metadataIndexChunking.js'
import { EntityIdMapper } from './entityIdMapper.js'
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
@ -852,15 +853,18 @@ export class MetadataIndexManager {
const chunk = await this.chunkManager.loadChunk(field, chunkId)
if (chunk) {
for (const [value, bitmap] of chunk.entries) {
// Check if value is in range (both value and normalized bounds are now bucketed)
// Check if value is in range using numeric-aware comparison
// (normalizeValue converts numbers to strings, so we must compare numerically)
let inRange = true
if (normalizedMin !== undefined) {
inRange = inRange && (includeMin ? value >= normalizedMin : value > normalizedMin)
const cmp = compareNormalizedValues(value, normalizedMin)
inRange = inRange && (includeMin ? cmp >= 0 : cmp > 0)
}
if (normalizedMax !== undefined) {
inRange = inRange && (includeMax ? value <= normalizedMax : value < normalizedMax)
const cmp = compareNormalizedValues(value, normalizedMax)
inRange = inRange && (includeMax ? cmp <= 0 : cmp < 0)
}
if (inRange) {

View file

@ -25,6 +25,31 @@ import { prodLog } from './logger.js'
import { RoaringBitmap32 } from './roaring/index.js'
import type { EntityIdMapper } from './entityIdMapper.js'
// ============================================================================
// Numeric-Aware Comparison
// ============================================================================
/**
* Compare two normalized string values with numeric awareness.
* Since normalizeValue() converts numbers to strings (e.g., 50 "50"),
* plain string comparison breaks numeric ordering ("50" > "100" is true
* lexicographically but wrong numerically). This function detects when
* both values are numeric strings and compares them as numbers.
*
* @returns negative if a < b, 0 if equal, positive if a > b
*/
export function compareNormalizedValues(a: string, b: string): number {
const numA = Number(a)
const numB = Number(b)
if (!isNaN(numA) && !isNaN(numB) && a !== '' && b !== '') {
return numA - numB
}
// Fall back to string comparison for non-numeric values
if (a < b) return -1
if (a > b) return 1
return 0
}
// ============================================================================
// Core Data Structures
// ============================================================================
@ -350,15 +375,10 @@ export class SparseIndex {
return zoneMap.hasNulls
}
// Handle different types
if (typeof value === 'number') {
return value >= zoneMap.min && value <= zoneMap.max
} else if (typeof value === 'string') {
return value >= zoneMap.min && value <= zoneMap.max
} else {
// For other types, conservatively check
return true
}
const strValue = String(value)
const strMin = String(zoneMap.min)
const strMax = String(zoneMap.max)
return compareNormalizedValues(strValue, strMin) >= 0 && compareNormalizedValues(strValue, strMax) <= 0
}
/**
@ -375,16 +395,19 @@ export class SparseIndex {
return true
}
// Check overlap
// Check overlap using numeric-aware comparison
const strZoneMin = String(zoneMap.min)
const strZoneMax = String(zoneMap.max)
if (min !== undefined && max !== undefined) {
// Range: [min, max] overlaps with [zoneMin, zoneMax]
return !(max < zoneMap.min || min > zoneMap.max)
return !(compareNormalizedValues(String(max), strZoneMin) < 0 || compareNormalizedValues(String(min), strZoneMax) > 0)
} else if (min !== undefined) {
// >= min
return zoneMap.max >= min
return compareNormalizedValues(strZoneMax, String(min)) >= 0
} else if (max !== undefined) {
// <= max
return zoneMap.min <= max
return compareNormalizedValues(strZoneMin, String(max)) <= 0
}
return true
@ -454,13 +477,7 @@ export class SparseIndex {
*/
private sortChunks(): void {
this.data.chunks.sort((a, b) => {
// Handle different types
if (typeof a.zoneMap.min === 'number' && typeof b.zoneMap.min === 'number') {
return a.zoneMap.min - b.zoneMap.min
} else if (typeof a.zoneMap.min === 'string' && typeof b.zoneMap.min === 'string') {
return a.zoneMap.min.localeCompare(b.zoneMap.min)
}
return 0
return compareNormalizedValues(String(a.zoneMap.min), String(b.zoneMap.min))
})
}
@ -716,8 +733,8 @@ export class ChunkManager {
if (value === '__NULL__' || value === null || value === undefined) {
hasNulls = true
} else {
if (value < min) min = value
if (value > max) max = value
if (compareNormalizedValues(value, min) < 0) min = value
if (compareNormalizedValues(value, max) > 0) max = value
}
// Get count from roaring bitmap

View file

@ -350,7 +350,7 @@ export function validateAddParams(params: AddParams): void {
throw new Error(
`Invalid NounType: '${params.type}'\n` +
`\nValid types: ${Object.values(NounType).join(', ')}\n` +
`\nExample: await brain.add({ data: 'text', type: NounType.Note })`
`\nExample: await brain.add({ data: 'text', type: NounType.Document })`
)
}