soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: add orderBy sorting and fix timestamp queries

Browse source 
Add production-scale sorting support with orderBy/order parameters:
- Sort by any field including createdAt, updatedAt, timestamps
- Works in both metadata-only and vector+metadata query paths
- O(k) memory complexity where k = filtered results

Fix timestamp sorting precision issue:
- Load actual timestamp values from entity metadata for sorting
- Avoids 1-minute bucketing precision loss
- Maintains bucketing for efficient range queries

Fix range query operators:
- Normalize min/max bounds before comparison with bucketed index
- Ensures gte, lte, gt, lt work correctly with timestamps

Standardize operator syntax:
- Canonical: eq, ne, gt, gte, lt, lte, in, between, contains, exists
- Deprecate: is, isNot, greaterEqual, lessEqual (remove in v5.0.0)
- Maintain backward compatibility with aliases

Test results: All sorting and range query tests pass, no regressions
This commit is contained in:
David Snelling 2025-10-27 09:14:10 -07:00
parent a7948d2f11
commit 97cc8fd1c7
3 changed files with 329 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

299
src/utils/metadataIndex.ts
View file

@ -631,6 +631,7 @@ export class MetadataIndexManager {
/**
* Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps (v3.43.0)
* v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
* v4.5.4: Normalize min/max for timestamp bucketing before comparison
*/
private async getIdsFromChunksForRange(
field: string,
@ -645,8 +646,13 @@ export class MetadataIndexManager {
return [] // No chunked index exists yet
}
// v4.5.4: Normalize min/max for consistent comparison with indexed values
// (indexed values are bucketed for timestamps, so we must bucket the query bounds too)
const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined
const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined
// Find candidate chunks using zone maps
const candidateChunkIds = sparseIndex.findChunksForRange(min, max)
const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax)
if (candidateChunkIds.length === 0) {
return []
@ -658,15 +664,15 @@ export class MetadataIndexManager {
const chunk = await this.chunkManager.loadChunk(field, chunkId)
if (chunk) {
for (const [value, bitmap] of chunk.entries) {
// Check if value is in range
// Check if value is in range (both value and normalized bounds are now bucketed)
let inRange = true
if (min !== undefined) {
inRange = inRange && (includeMin ? value >= min : value > min)
if (normalizedMin !== undefined) {
inRange = inRange && (includeMin ? value >= normalizedMin : value > normalizedMin)
}
if (max !== undefined) {
inRange = inRange && (includeMax ? value <= max : value < max)
if (normalizedMax !== undefined) {
inRange = inRange && (includeMax ? value <= normalizedMax : value < normalizedMax)
}
if (inRange) {
@ -1486,18 +1492,41 @@ export class MetadataIndexManager {
let fieldResults: string[] = []
if (condition && typeof condition === 'object' && !Array.isArray(condition)) {
// Handle Brainy Field Operators
// Handle Brainy Field Operators (v4.5.4: canonical operators defined)
// See docs/api/README.md for complete operator reference
for (const [op, operand] of Object.entries(condition)) {
switch (op) {
// Exact match operators
case 'equals':
case 'is':
// ===== EQUALITY OPERATORS =====
// Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is' (remove in v5.0.0)
case 'is': // DEPRECATED (v4.5.4): Use 'eq' instead
case 'equals': // Alias for 'eq'
case 'eq':
fieldResults = await this.getIds(field, operand)
break
// Multiple value operators
case 'oneOf':
// ===== NEGATION OPERATORS =====
// Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot' (remove in v5.0.0)
case 'isNot': // DEPRECATED (v4.5.4): Use 'ne' instead
case 'notEquals': // Alias for 'ne'
case 'ne':
// For notEquals, we need all IDs EXCEPT those matching the value
// This is especially important for soft delete: deleted !== true
// should include items without a deleted field
// First, get all IDs in the database
const allItemIds = await this.getAllIds()
// Then get IDs that match the value we want to exclude
const excludeIds = await this.getIds(field, operand)
const excludeSet = new Set(excludeIds)
// Return all IDs except those to exclude
fieldResults = allItemIds.filter(id => !excludeSet.has(id))
break
// ===== MULTI-VALUE OPERATORS =====
// Canonical: 'in' | Alias: 'oneOf'
case 'oneOf': // Alias for 'in'
case 'in':
if (Array.isArray(operand)) {
const unionIds = new Set<string>()
@ -1508,42 +1537,53 @@ export class MetadataIndexManager {
fieldResults = Array.from(unionIds)
}
break
// Range operators
case 'greaterThan':
// ===== GREATER THAN OPERATORS =====
// Canonical: 'gt' | Alias: 'greaterThan'
case 'greaterThan': // Alias for 'gt'
case 'gt':
fieldResults = await this.getIdsForRange(field, operand, undefined, false, true)
break
case 'greaterEqual':
// ===== GREATER THAN OR EQUAL OPERATORS =====
// Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual' (remove in v5.0.0)
case 'greaterEqual': // DEPRECATED (v4.5.4): Use 'gte' instead
case 'greaterThanOrEqual': // Alias for 'gte'
case 'gte':
case 'greaterThanOrEqual':
fieldResults = await this.getIdsForRange(field, operand, undefined, true, true)
break
case 'lessThan':
// ===== LESS THAN OPERATORS =====
// Canonical: 'lt' | Alias: 'lessThan'
case 'lessThan': // Alias for 'lt'
case 'lt':
fieldResults = await this.getIdsForRange(field, undefined, operand, true, false)
break
case 'lessEqual':
// ===== LESS THAN OR EQUAL OPERATORS =====
// Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual' (remove in v5.0.0)
case 'lessEqual': // DEPRECATED (v4.5.4): Use 'lte' instead
case 'lessThanOrEqual': // Alias for 'lte'
case 'lte':
case 'lessThanOrEqual':
fieldResults = await this.getIdsForRange(field, undefined, operand, true, true)
break
// ===== RANGE OPERATOR =====
// between: [min, max] - inclusive range query
case 'between':
if (Array.isArray(operand) && operand.length === 2) {
fieldResults = await this.getIdsForRange(field, operand[0], operand[1], true, true)
}
break
// Array contains operator
// ===== ARRAY CONTAINS OPERATOR =====
// contains: value - check if array field contains value
case 'contains':
fieldResults = await this.getIds(field, operand)
break
// Existence operator
// ===== EXISTENCE OPERATOR =====
// exists: boolean - check if field exists (any value)
case 'exists':
if (operand) {
// Get all IDs that have this field (any value) from chunked sparse index with roaring bitmaps (v3.43.0)
@ -1571,29 +1611,10 @@ export class MetadataIndexManager {
fieldResults = this.idMapper.intsIterableToUuids(allIntIds)
}
break
// Negation operators
case 'notEquals':
case 'isNot':
case 'ne':
// For notEquals, we need all IDs EXCEPT those matching the value
// This is especially important for soft delete: deleted !== true
// should include items without a deleted field
// First, get all IDs in the database
const allItemIds = await this.getAllIds()
// Then get IDs that match the value we want to exclude
const excludeIds = await this.getIds(field, operand)
const excludeSet = new Set(excludeIds)
// Return all IDs except those to exclude
fieldResults = allItemIds.filter(id => !excludeSet.has(id))
break
}
}
} else {
// Direct value match (shorthand for equals)
// Direct value match (shorthand for 'eq' operator)
fieldResults = await this.getIds(field, condition)
}
@ -1609,11 +1630,187 @@ export class MetadataIndexManager {
if (idSets.length === 1) return idSets[0]
// Intersection of all field criteria (implicit AND)
return idSets.reduce((intersection, currentSet) =>
return idSets.reduce((intersection, currentSet) =>
intersection.filter(id => currentSet.includes(id))
)
}
/**
* Get filtered IDs sorted by a field (production-scale sorting)
*
* **Performance Characteristics** (designed for billions of entities):
* - **Filtering**: O(log n) using roaring bitmaps with SIMD acceleration
* - **Field Loading**: O(k) where k = filtered result count (NOT O(n))
* - **Sorting**: O(k log k) in-memory (IDs + sort values only, NOT full entities)
* - **Memory**: O(k) for k filtered results, independent of total entity count
*
* **Scalability**:
* - Total entities: Billions (memory usage unaffected)
* - Filtered set: Up to 10M (reasonable for in-memory sort of ID+value pairs)
* - Pagination: Happens AFTER sorting, so only page entities are loaded
*
* **Example**:
* ```typescript
* // Production-scale: 1B entities, 100K match filter, sort by createdAt
* const sortedIds = await metadataIndex.getSortedIdsForFilter(
* { status: 'published', category: 'AI' },
* 'createdAt',
* 'desc'
* )
* // Returns: 100K sorted IDs
* // Memory: ~5MB (100K IDs + 100K timestamps)
* // Then caller paginates: sortedIds.slice(0, 20) and loads only 20 entities
* ```
*
* @param filter - Metadata filter criteria (uses roaring bitmaps)
* @param orderBy - Field name to sort by (e.g., 'createdAt', 'title')
* @param order - Sort direction: 'asc' (default) or 'desc'
* @returns Promise<string[]> - Entity IDs sorted by specified field
*
* @since v4.5.4
*/
async getSortedIdsForFilter(
filter: any,
orderBy: string,
order: 'asc' | 'desc' = 'asc'
): Promise<string[]> {
// 1. Get filtered IDs using existing roaring bitmap implementation (fast!)
const filteredIds = await this.getIdsForFilter(filter)
if (filteredIds.length === 0) {
return []
}
// 2. Load sort field values for filtered IDs ONLY
// This is O(k) not O(n) where k = filtered count
// We only load the ONE field needed for sorting, not full entities
const idValuePairs: Array<{ id: string, value: any }> = []
for (const id of filteredIds) {
const value = await this.getFieldValueForEntity(id, orderBy)
idValuePairs.push({ id, value })
}
// 3. Sort by value (in-memory BUT only IDs + sort values)
// This is acceptable because we're sorting the FILTERED set, not all entities
// Even 1M filtered results = ~50MB (IDs + values), manageable in-memory
idValuePairs.sort((a, b) => {
// Handle null/undefined (always sort to end)
if (a.value == null && b.value == null) return 0
if (a.value == null) return order === 'asc' ? 1 : -1
if (b.value == null) return order === 'asc' ? -1 : 1
// Compare values
if (a.value === b.value) return 0
const comparison = a.value < b.value ? -1 : 1
return order === 'asc' ? comparison : -comparison
})
// 4. Return sorted IDs (caller handles pagination BEFORE loading entities)
return idValuePairs.map(p => p.id)
}
/**
* Get field value for a specific entity (helper for sorted queries)
*
* **IMPORTANT**: For timestamp fields (createdAt, updatedAt), this loads
* the ACTUAL value from entity metadata, NOT the bucketed index value.
* This is required because timestamp bucketing (1-minute precision) loses
* precision needed for accurate sorting.
*
* For non-timestamp fields, loads from the chunked sparse index without
* loading the full entity. This is critical for production-scale sorting.
*
* **Performance**:
* - Timestamp fields: O(1) metadata load from storage (cached)
* - Other fields: O(chunks) roaring bitmap lookup (typically 1-10 chunks)
*
* @param entityId - Entity UUID to get field value for
* @param field - Field name to retrieve (e.g., 'createdAt', 'title')
* @returns Promise<any> - Field value or undefined if not found
*
* @public (called from brainy.ts for sorted queries)
* @since v4.5.4
*/
async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
// For timestamp fields, load ACTUAL value from entity metadata
// (index has bucketed values which lose precision for sorting)
if (field === 'createdAt' || field === 'updatedAt' || field === 'accessed' || field === 'modified') {
try {
const noun = await this.storage.getNoun(entityId)
if (noun && noun.metadata) {
return noun.metadata[field]
}
} catch (err) {
// If metadata load fails, fall back to index (bucketed value)
console.warn(`[MetadataIndex] Failed to load ${field} from metadata for ${entityId}, using bucketed value`)
}
}
// For non-timestamp fields, use the sparse index (no bucketing issues)
const intId = this.idMapper.getInt(entityId)
if (intId === undefined) {
return undefined
}
// Load sparse index for this field (cached via UnifiedCache)
const sparseIndex = await this.loadSparseIndex(field)
if (!sparseIndex) {
return undefined
}
// Search through chunks to find which value this entity has
// Typically 1-10 chunks per field, so this is fast
for (const chunkId of sparseIndex.getAllChunkIds()) {
const chunk = await this.chunkManager.loadChunk(field, chunkId)
if (!chunk) continue
// Check each value's roaring bitmap for our entity ID
// Roaring bitmap .has() is O(1) with SIMD optimization
for (const [value, bitmap] of chunk.entries) {
if (bitmap.has(intId)) {
// Found it! Denormalize the value (no bucketing for non-timestamps)
return this.denormalizeValue(value, field)
}
}
}
return undefined
}
/**
* Denormalize a value (reverse of normalizeValue)
*
* Converts normalized/stringified values back to their original type.
* For most fields, this just parses numbers or returns strings as-is.
*
* **NOTE**: This is NOT used for timestamp sorting! Timestamp fields
* (createdAt, updatedAt) are loaded directly from entity metadata by
* getFieldValueForEntity() to avoid precision loss from bucketing.
*
* **Timestamp Bucketing (for range queries only)**:
* - Indexed as: Math.floor(timestamp / 60000) * 60000
* - Used for: Range queries (gte, lte) where 1-minute precision is acceptable
* - NOT used for: Sorting (requires exact millisecond precision)
*
* @param normalized - Normalized value string from index
* @param field - Field name (used for type inference)
* @returns Denormalized value in original type
*
* @private
* @since v4.5.4
*/
private denormalizeValue(normalized: string, field: string): any {
// Try parsing as number (timestamps, integers, floats)
const asNumber = Number(normalized)
if (!isNaN(asNumber)) {
return asNumber
}
// For strings, return as-is (already denormalized)
return normalized
}
/**
* DEPRECATED - Old implementation for backward compatibility
*/