feat: enforce data/metadata separation, numeric range queries, improved docs
- Store data opaquely in add() and update() instead of spreading object properties into top-level metadata. data is for semantic search (HNSW), metadata is for structured where-filter queries (MetadataIndex). - Fix numeric range queries in MetadataIndex — use numeric-aware comparison instead of lexicographic string comparison for normalized values. - Add data field to RelateParams and Relation types for relationship content. - Add where.type → where.noun alias in metadata-only find() path. - Rewrite README: focused ~350 lines from 791, quick start first, feature showcase with mini-snippets, organized doc links, no version callouts. - Add DATA_MODEL.md and QUERY_OPERATORS.md reference docs. - Remove 10 outdated/redundant doc files consolidated into API reference. - Improve JSDoc on Entity, Relation, AddParams, FindParams, and core methods. - Fix tests asserting data properties appear in metadata (data model violation). - Deprecate verb.source/target in favor of from/to (public) and sourceId/targetId (storage).
This commit is contained in:
parent
edb5ec4696
commit
0ddc05a5bb
30 changed files with 1356 additions and 7001 deletions
|
|
@ -21,7 +21,8 @@ import {
|
|||
AdaptiveChunkingStrategy,
|
||||
ChunkData,
|
||||
ChunkDescriptor,
|
||||
ZoneMap
|
||||
ZoneMap,
|
||||
compareNormalizedValues
|
||||
} from './metadataIndexChunking.js'
|
||||
import { EntityIdMapper } from './entityIdMapper.js'
|
||||
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
|
||||
|
|
@ -852,15 +853,18 @@ export class MetadataIndexManager {
|
|||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||
if (chunk) {
|
||||
for (const [value, bitmap] of chunk.entries) {
|
||||
// Check if value is in range (both value and normalized bounds are now bucketed)
|
||||
// Check if value is in range using numeric-aware comparison
|
||||
// (normalizeValue converts numbers to strings, so we must compare numerically)
|
||||
let inRange = true
|
||||
|
||||
if (normalizedMin !== undefined) {
|
||||
inRange = inRange && (includeMin ? value >= normalizedMin : value > normalizedMin)
|
||||
const cmp = compareNormalizedValues(value, normalizedMin)
|
||||
inRange = inRange && (includeMin ? cmp >= 0 : cmp > 0)
|
||||
}
|
||||
|
||||
if (normalizedMax !== undefined) {
|
||||
inRange = inRange && (includeMax ? value <= normalizedMax : value < normalizedMax)
|
||||
const cmp = compareNormalizedValues(value, normalizedMax)
|
||||
inRange = inRange && (includeMax ? cmp <= 0 : cmp < 0)
|
||||
}
|
||||
|
||||
if (inRange) {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,31 @@ import { prodLog } from './logger.js'
|
|||
import { RoaringBitmap32 } from './roaring/index.js'
|
||||
import type { EntityIdMapper } from './entityIdMapper.js'
|
||||
|
||||
// ============================================================================
|
||||
// Numeric-Aware Comparison
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Compare two normalized string values with numeric awareness.
|
||||
* Since normalizeValue() converts numbers to strings (e.g., 50 → "50"),
|
||||
* plain string comparison breaks numeric ordering ("50" > "100" is true
|
||||
* lexicographically but wrong numerically). This function detects when
|
||||
* both values are numeric strings and compares them as numbers.
|
||||
*
|
||||
* @returns negative if a < b, 0 if equal, positive if a > b
|
||||
*/
|
||||
export function compareNormalizedValues(a: string, b: string): number {
|
||||
const numA = Number(a)
|
||||
const numB = Number(b)
|
||||
if (!isNaN(numA) && !isNaN(numB) && a !== '' && b !== '') {
|
||||
return numA - numB
|
||||
}
|
||||
// Fall back to string comparison for non-numeric values
|
||||
if (a < b) return -1
|
||||
if (a > b) return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Core Data Structures
|
||||
// ============================================================================
|
||||
|
|
@ -350,15 +375,10 @@ export class SparseIndex {
|
|||
return zoneMap.hasNulls
|
||||
}
|
||||
|
||||
// Handle different types
|
||||
if (typeof value === 'number') {
|
||||
return value >= zoneMap.min && value <= zoneMap.max
|
||||
} else if (typeof value === 'string') {
|
||||
return value >= zoneMap.min && value <= zoneMap.max
|
||||
} else {
|
||||
// For other types, conservatively check
|
||||
return true
|
||||
}
|
||||
const strValue = String(value)
|
||||
const strMin = String(zoneMap.min)
|
||||
const strMax = String(zoneMap.max)
|
||||
return compareNormalizedValues(strValue, strMin) >= 0 && compareNormalizedValues(strValue, strMax) <= 0
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -375,16 +395,19 @@ export class SparseIndex {
|
|||
return true
|
||||
}
|
||||
|
||||
// Check overlap
|
||||
// Check overlap using numeric-aware comparison
|
||||
const strZoneMin = String(zoneMap.min)
|
||||
const strZoneMax = String(zoneMap.max)
|
||||
|
||||
if (min !== undefined && max !== undefined) {
|
||||
// Range: [min, max] overlaps with [zoneMin, zoneMax]
|
||||
return !(max < zoneMap.min || min > zoneMap.max)
|
||||
return !(compareNormalizedValues(String(max), strZoneMin) < 0 || compareNormalizedValues(String(min), strZoneMax) > 0)
|
||||
} else if (min !== undefined) {
|
||||
// >= min
|
||||
return zoneMap.max >= min
|
||||
return compareNormalizedValues(strZoneMax, String(min)) >= 0
|
||||
} else if (max !== undefined) {
|
||||
// <= max
|
||||
return zoneMap.min <= max
|
||||
return compareNormalizedValues(strZoneMin, String(max)) <= 0
|
||||
}
|
||||
|
||||
return true
|
||||
|
|
@ -454,13 +477,7 @@ export class SparseIndex {
|
|||
*/
|
||||
private sortChunks(): void {
|
||||
this.data.chunks.sort((a, b) => {
|
||||
// Handle different types
|
||||
if (typeof a.zoneMap.min === 'number' && typeof b.zoneMap.min === 'number') {
|
||||
return a.zoneMap.min - b.zoneMap.min
|
||||
} else if (typeof a.zoneMap.min === 'string' && typeof b.zoneMap.min === 'string') {
|
||||
return a.zoneMap.min.localeCompare(b.zoneMap.min)
|
||||
}
|
||||
return 0
|
||||
return compareNormalizedValues(String(a.zoneMap.min), String(b.zoneMap.min))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -716,8 +733,8 @@ export class ChunkManager {
|
|||
if (value === '__NULL__' || value === null || value === undefined) {
|
||||
hasNulls = true
|
||||
} else {
|
||||
if (value < min) min = value
|
||||
if (value > max) max = value
|
||||
if (compareNormalizedValues(value, min) < 0) min = value
|
||||
if (compareNormalizedValues(value, max) > 0) max = value
|
||||
}
|
||||
|
||||
// Get count from roaring bitmap
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ export function validateAddParams(params: AddParams): void {
|
|||
throw new Error(
|
||||
`Invalid NounType: '${params.type}'\n` +
|
||||
`\nValid types: ${Object.values(NounType).join(', ')}\n` +
|
||||
`\nExample: await brain.add({ data: 'text', type: NounType.Note })`
|
||||
`\nExample: await brain.add({ data: 'text', type: NounType.Document })`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue