diff --git a/src/brainy.ts b/src/brainy.ts index b15aa919..c6d4c163 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -561,7 +561,13 @@ export class Brainy implements BrainyInterface { id: noun.id, vector: noun.vector, type: (nounType as NounType) || NounType.Thing, - metadata: userMetadata as T, + // Preserve timestamps in metadata for indexing (v4.5.4 fix) + // Metadata index needs these fields to enable sorting and range queries + metadata: { + ...userMetadata, + ...(createdAt !== undefined && { createdAt }), + ...(updatedAt !== undefined && { updatedAt }) + } as T, service: service as string, createdAt: (createdAt as number) || Date.now(), updatedAt: updatedAt as number @@ -1289,8 +1295,21 @@ export class Brainy implements BrainyInterface { } } - // Get filtered IDs and paginate BEFORE loading entities - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + // v4.5.4: Apply sorting if requested, otherwise just filter + let filteredIds: string[] + if (params.orderBy) { + // Get sorted IDs using production-scale sorted filtering + filteredIds = await this.metadataIndex.getSortedIdsForFilter( + filter, + params.orderBy, + params.order || 'asc' + ) + } else { + // Just filter without sorting + filteredIds = await this.metadataIndex.getIdsForFilter(filter) + } + + // Paginate BEFORE loading entities (production-scale!) const limit = params.limit || 10 const offset = params.offset || 0 const pageIds = filteredIds.slice(offset, offset + limit) @@ -1457,6 +1476,31 @@ export class Brainy implements BrainyInterface { // Early return for metadata-only queries with pagination applied if (!params.query && !params.connected) { + // v4.5.4: Apply sorting if requested for metadata-only queries + if (params.orderBy) { + const sortedIds = await this.metadataIndex.getSortedIdsForFilter( + filter, + params.orderBy, + params.order || 'asc' + ) + + // Paginate sorted IDs BEFORE loading entities (production-scale!) + const limit = params.limit || 10 + const offset = params.offset || 0 + const pageIds = sortedIds.slice(offset, offset + limit) + + // Load entities for paginated results only + const sortedResults: Result[] = [] + for (const id of pageIds) { + const entity = await this.get(id) + if (entity) { + sortedResults.push(this.createResult(id, 1.0, entity)) + } + } + + return sortedResults + } + return results } } @@ -1466,14 +1510,41 @@ export class Brainy implements BrainyInterface { if (params.connected) { results = await this.executeGraphSearch(params, results) } - + // Apply fusion scoring if requested if (params.fusion && results.length > 0) { results = this.applyFusionScoring(results, params.fusion) } // OPTIMIZED: Sort first, then apply efficient pagination - results.sort((a, b) => b.score - a.score) + // v4.5.4: Support custom orderBy for vector + metadata queries + if (params.orderBy && results.length > 0) { + // For vector + metadata queries, sort by specified field instead of score + // Load sort field values for all results (small set, already filtered) + const resultsWithValues = await Promise.all(results.map(async (r) => ({ + result: r, + value: await this.metadataIndex.getFieldValueForEntity(r.id, params.orderBy!) + }))) + + // Sort by field value + resultsWithValues.sort((a, b) => { + // Handle null/undefined + if (a.value == null && b.value == null) return 0 + if (a.value == null) return (params.order || 'asc') === 'asc' ? 1 : -1 + if (b.value == null) return (params.order || 'asc') === 'asc' ? -1 : 1 + + // Compare values + if (a.value === b.value) return 0 + const comparison = a.value < b.value ? -1 : 1 + return (params.order || 'asc') === 'asc' ? comparison : -comparison + }) + + results = resultsWithValues.map(({ result }) => result) + } else { + // Default: sort by relevance score + results.sort((a, b) => b.score - a.score) + } + const limit = params.limit || 10 const offset = params.offset || 0 diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index ec80630c..d80c147f 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -180,7 +180,11 @@ export interface FindParams { limit?: number // Max results (default: 10) offset?: number // Skip N results cursor?: string // Cursor-based pagination - + + // Sorting (v4.5.4) + orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority') + order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' + // Advanced options mode?: SearchMode // Search strategy explain?: boolean // Return scoring explanation diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 0b7f0067..34a5c028 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -631,6 +631,7 @@ export class MetadataIndexManager { /** * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps (v3.43.0) * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * v4.5.4: Normalize min/max for timestamp bucketing before comparison */ private async getIdsFromChunksForRange( field: string, @@ -645,8 +646,13 @@ export class MetadataIndexManager { return [] // No chunked index exists yet } + // v4.5.4: Normalize min/max for consistent comparison with indexed values + // (indexed values are bucketed for timestamps, so we must bucket the query bounds too) + const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined + const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined + // Find candidate chunks using zone maps - const candidateChunkIds = sparseIndex.findChunksForRange(min, max) + const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax) if (candidateChunkIds.length === 0) { return [] @@ -658,15 +664,15 @@ export class MetadataIndexManager { const chunk = await this.chunkManager.loadChunk(field, chunkId) if (chunk) { for (const [value, bitmap] of chunk.entries) { - // Check if value is in range + // Check if value is in range (both value and normalized bounds are now bucketed) let inRange = true - if (min !== undefined) { - inRange = inRange && (includeMin ? value >= min : value > min) + if (normalizedMin !== undefined) { + inRange = inRange && (includeMin ? value >= normalizedMin : value > normalizedMin) } - if (max !== undefined) { - inRange = inRange && (includeMax ? value <= max : value < max) + if (normalizedMax !== undefined) { + inRange = inRange && (includeMax ? value <= normalizedMax : value < normalizedMax) } if (inRange) { @@ -1486,18 +1492,41 @@ export class MetadataIndexManager { let fieldResults: string[] = [] if (condition && typeof condition === 'object' && !Array.isArray(condition)) { - // Handle Brainy Field Operators + // Handle Brainy Field Operators (v4.5.4: canonical operators defined) + // See docs/api/README.md for complete operator reference for (const [op, operand] of Object.entries(condition)) { switch (op) { - // Exact match operators - case 'equals': - case 'is': + // ===== EQUALITY OPERATORS ===== + // Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is' (remove in v5.0.0) + case 'is': // DEPRECATED (v4.5.4): Use 'eq' instead + case 'equals': // Alias for 'eq' case 'eq': fieldResults = await this.getIds(field, operand) break - - // Multiple value operators - case 'oneOf': + + // ===== NEGATION OPERATORS ===== + // Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot' (remove in v5.0.0) + case 'isNot': // DEPRECATED (v4.5.4): Use 'ne' instead + case 'notEquals': // Alias for 'ne' + case 'ne': + // For notEquals, we need all IDs EXCEPT those matching the value + // This is especially important for soft delete: deleted !== true + // should include items without a deleted field + + // First, get all IDs in the database + const allItemIds = await this.getAllIds() + + // Then get IDs that match the value we want to exclude + const excludeIds = await this.getIds(field, operand) + const excludeSet = new Set(excludeIds) + + // Return all IDs except those to exclude + fieldResults = allItemIds.filter(id => !excludeSet.has(id)) + break + + // ===== MULTI-VALUE OPERATORS ===== + // Canonical: 'in' | Alias: 'oneOf' + case 'oneOf': // Alias for 'in' case 'in': if (Array.isArray(operand)) { const unionIds = new Set() @@ -1508,42 +1537,53 @@ export class MetadataIndexManager { fieldResults = Array.from(unionIds) } break - - // Range operators - case 'greaterThan': + + // ===== GREATER THAN OPERATORS ===== + // Canonical: 'gt' | Alias: 'greaterThan' + case 'greaterThan': // Alias for 'gt' case 'gt': fieldResults = await this.getIdsForRange(field, operand, undefined, false, true) break - - case 'greaterEqual': + + // ===== GREATER THAN OR EQUAL OPERATORS ===== + // Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual' (remove in v5.0.0) + case 'greaterEqual': // DEPRECATED (v4.5.4): Use 'gte' instead + case 'greaterThanOrEqual': // Alias for 'gte' case 'gte': - case 'greaterThanOrEqual': fieldResults = await this.getIdsForRange(field, operand, undefined, true, true) break - - case 'lessThan': + + // ===== LESS THAN OPERATORS ===== + // Canonical: 'lt' | Alias: 'lessThan' + case 'lessThan': // Alias for 'lt' case 'lt': fieldResults = await this.getIdsForRange(field, undefined, operand, true, false) break - - case 'lessEqual': + + // ===== LESS THAN OR EQUAL OPERATORS ===== + // Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual' (remove in v5.0.0) + case 'lessEqual': // DEPRECATED (v4.5.4): Use 'lte' instead + case 'lessThanOrEqual': // Alias for 'lte' case 'lte': - case 'lessThanOrEqual': fieldResults = await this.getIdsForRange(field, undefined, operand, true, true) break - + + // ===== RANGE OPERATOR ===== + // between: [min, max] - inclusive range query case 'between': if (Array.isArray(operand) && operand.length === 2) { fieldResults = await this.getIdsForRange(field, operand[0], operand[1], true, true) } break - - // Array contains operator + + // ===== ARRAY CONTAINS OPERATOR ===== + // contains: value - check if array field contains value case 'contains': fieldResults = await this.getIds(field, operand) break - - // Existence operator + + // ===== EXISTENCE OPERATOR ===== + // exists: boolean - check if field exists (any value) case 'exists': if (operand) { // Get all IDs that have this field (any value) from chunked sparse index with roaring bitmaps (v3.43.0) @@ -1571,29 +1611,10 @@ export class MetadataIndexManager { fieldResults = this.idMapper.intsIterableToUuids(allIntIds) } break - - // Negation operators - case 'notEquals': - case 'isNot': - case 'ne': - // For notEquals, we need all IDs EXCEPT those matching the value - // This is especially important for soft delete: deleted !== true - // should include items without a deleted field - - // First, get all IDs in the database - const allItemIds = await this.getAllIds() - - // Then get IDs that match the value we want to exclude - const excludeIds = await this.getIds(field, operand) - const excludeSet = new Set(excludeIds) - - // Return all IDs except those to exclude - fieldResults = allItemIds.filter(id => !excludeSet.has(id)) - break } } } else { - // Direct value match (shorthand for equals) + // Direct value match (shorthand for 'eq' operator) fieldResults = await this.getIds(field, condition) } @@ -1609,11 +1630,187 @@ export class MetadataIndexManager { if (idSets.length === 1) return idSets[0] // Intersection of all field criteria (implicit AND) - return idSets.reduce((intersection, currentSet) => + return idSets.reduce((intersection, currentSet) => intersection.filter(id => currentSet.includes(id)) ) } - + + /** + * Get filtered IDs sorted by a field (production-scale sorting) + * + * **Performance Characteristics** (designed for billions of entities): + * - **Filtering**: O(log n) using roaring bitmaps with SIMD acceleration + * - **Field Loading**: O(k) where k = filtered result count (NOT O(n)) + * - **Sorting**: O(k log k) in-memory (IDs + sort values only, NOT full entities) + * - **Memory**: O(k) for k filtered results, independent of total entity count + * + * **Scalability**: + * - Total entities: Billions (memory usage unaffected) + * - Filtered set: Up to 10M (reasonable for in-memory sort of ID+value pairs) + * - Pagination: Happens AFTER sorting, so only page entities are loaded + * + * **Example**: + * ```typescript + * // Production-scale: 1B entities, 100K match filter, sort by createdAt + * const sortedIds = await metadataIndex.getSortedIdsForFilter( + * { status: 'published', category: 'AI' }, + * 'createdAt', + * 'desc' + * ) + * // Returns: 100K sorted IDs + * // Memory: ~5MB (100K IDs + 100K timestamps) + * // Then caller paginates: sortedIds.slice(0, 20) and loads only 20 entities + * ``` + * + * @param filter - Metadata filter criteria (uses roaring bitmaps) + * @param orderBy - Field name to sort by (e.g., 'createdAt', 'title') + * @param order - Sort direction: 'asc' (default) or 'desc' + * @returns Promise - Entity IDs sorted by specified field + * + * @since v4.5.4 + */ + async getSortedIdsForFilter( + filter: any, + orderBy: string, + order: 'asc' | 'desc' = 'asc' + ): Promise { + // 1. Get filtered IDs using existing roaring bitmap implementation (fast!) + const filteredIds = await this.getIdsForFilter(filter) + + if (filteredIds.length === 0) { + return [] + } + + // 2. Load sort field values for filtered IDs ONLY + // This is O(k) not O(n) where k = filtered count + // We only load the ONE field needed for sorting, not full entities + const idValuePairs: Array<{ id: string, value: any }> = [] + + for (const id of filteredIds) { + const value = await this.getFieldValueForEntity(id, orderBy) + idValuePairs.push({ id, value }) + } + + // 3. Sort by value (in-memory BUT only IDs + sort values) + // This is acceptable because we're sorting the FILTERED set, not all entities + // Even 1M filtered results = ~50MB (IDs + values), manageable in-memory + idValuePairs.sort((a, b) => { + // Handle null/undefined (always sort to end) + if (a.value == null && b.value == null) return 0 + if (a.value == null) return order === 'asc' ? 1 : -1 + if (b.value == null) return order === 'asc' ? -1 : 1 + + // Compare values + if (a.value === b.value) return 0 + const comparison = a.value < b.value ? -1 : 1 + return order === 'asc' ? comparison : -comparison + }) + + // 4. Return sorted IDs (caller handles pagination BEFORE loading entities) + return idValuePairs.map(p => p.id) + } + + /** + * Get field value for a specific entity (helper for sorted queries) + * + * **IMPORTANT**: For timestamp fields (createdAt, updatedAt), this loads + * the ACTUAL value from entity metadata, NOT the bucketed index value. + * This is required because timestamp bucketing (1-minute precision) loses + * precision needed for accurate sorting. + * + * For non-timestamp fields, loads from the chunked sparse index without + * loading the full entity. This is critical for production-scale sorting. + * + * **Performance**: + * - Timestamp fields: O(1) metadata load from storage (cached) + * - Other fields: O(chunks) roaring bitmap lookup (typically 1-10 chunks) + * + * @param entityId - Entity UUID to get field value for + * @param field - Field name to retrieve (e.g., 'createdAt', 'title') + * @returns Promise - Field value or undefined if not found + * + * @public (called from brainy.ts for sorted queries) + * @since v4.5.4 + */ + async getFieldValueForEntity(entityId: string, field: string): Promise { + // For timestamp fields, load ACTUAL value from entity metadata + // (index has bucketed values which lose precision for sorting) + if (field === 'createdAt' || field === 'updatedAt' || field === 'accessed' || field === 'modified') { + try { + const noun = await this.storage.getNoun(entityId) + if (noun && noun.metadata) { + return noun.metadata[field] + } + } catch (err) { + // If metadata load fails, fall back to index (bucketed value) + console.warn(`[MetadataIndex] Failed to load ${field} from metadata for ${entityId}, using bucketed value`) + } + } + + // For non-timestamp fields, use the sparse index (no bucketing issues) + const intId = this.idMapper.getInt(entityId) + if (intId === undefined) { + return undefined + } + + // Load sparse index for this field (cached via UnifiedCache) + const sparseIndex = await this.loadSparseIndex(field) + if (!sparseIndex) { + return undefined + } + + // Search through chunks to find which value this entity has + // Typically 1-10 chunks per field, so this is fast + for (const chunkId of sparseIndex.getAllChunkIds()) { + const chunk = await this.chunkManager.loadChunk(field, chunkId) + if (!chunk) continue + + // Check each value's roaring bitmap for our entity ID + // Roaring bitmap .has() is O(1) with SIMD optimization + for (const [value, bitmap] of chunk.entries) { + if (bitmap.has(intId)) { + // Found it! Denormalize the value (no bucketing for non-timestamps) + return this.denormalizeValue(value, field) + } + } + } + + return undefined + } + + /** + * Denormalize a value (reverse of normalizeValue) + * + * Converts normalized/stringified values back to their original type. + * For most fields, this just parses numbers or returns strings as-is. + * + * **NOTE**: This is NOT used for timestamp sorting! Timestamp fields + * (createdAt, updatedAt) are loaded directly from entity metadata by + * getFieldValueForEntity() to avoid precision loss from bucketing. + * + * **Timestamp Bucketing (for range queries only)**: + * - Indexed as: Math.floor(timestamp / 60000) * 60000 + * - Used for: Range queries (gte, lte) where 1-minute precision is acceptable + * - NOT used for: Sorting (requires exact millisecond precision) + * + * @param normalized - Normalized value string from index + * @param field - Field name (used for type inference) + * @returns Denormalized value in original type + * + * @private + * @since v4.5.4 + */ + private denormalizeValue(normalized: string, field: string): any { + // Try parsing as number (timestamps, integers, floats) + const asNumber = Number(normalized) + if (!isNaN(asNumber)) { + return asNumber + } + + // For strings, return as-is (already denormalized) + return normalized + } + /** * DEPRECATED - Old implementation for backward compatibility */