fix: correct typo in README major updates section
🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
d2ddb9199e
commit
1a4f035ffc
22 changed files with 4423 additions and 142 deletions
542
docs/api-reference/search-methods.md
Normal file
542
docs/api-reference/search-methods.md
Normal file
|
|
@ -0,0 +1,542 @@
|
|||
# Search Methods API Reference
|
||||
|
||||
Complete API documentation for Brainy's search methods with MongoDB-style metadata filtering.
|
||||
|
||||
## Overview
|
||||
|
||||
Brainy provides powerful search capabilities that combine vector similarity with sophisticated metadata filtering. All search methods support the new `metadata` parameter for advanced filtering using MongoDB-style operators.
|
||||
|
||||
## Core Search Methods
|
||||
|
||||
### `search(queryVector, k, options)`
|
||||
|
||||
**Vector-based search with metadata filtering**
|
||||
|
||||
```typescript
|
||||
async search<T = any>(
|
||||
queryVector: Vector | any,
|
||||
k: number = 10,
|
||||
options: SearchOptions<T> = {}
|
||||
): Promise<SearchResult<T>[]>
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `queryVector` | `Vector \| any` | Required | Query vector or data to be embedded |
|
||||
| `k` | `number` | `10` | Maximum number of results to return |
|
||||
| `options` | `SearchOptions<T>` | `{}` | Search configuration options |
|
||||
|
||||
#### Options
|
||||
|
||||
```typescript
|
||||
interface SearchOptions<T> {
|
||||
nounTypes?: string[] // Filter by entity types
|
||||
includeVerbs?: boolean // Include relationships in results
|
||||
searchMode?: 'local' | 'remote' | 'combined'
|
||||
metadata?: MetadataFilter // MongoDB-style metadata filtering 🆕
|
||||
service?: string // Filter by service that created the data
|
||||
offset?: number // Pagination offset
|
||||
forceEmbed?: boolean // Force embedding even if input is a vector
|
||||
}
|
||||
```
|
||||
|
||||
#### Returns
|
||||
|
||||
```typescript
|
||||
interface SearchResult<T> {
|
||||
id: string
|
||||
score: number // Similarity score (0-1, higher = more similar)
|
||||
vector: Vector
|
||||
metadata: T
|
||||
nounType?: string
|
||||
createdBy?: any
|
||||
}
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```javascript
|
||||
const results = await brainy.search("smartphone", 10, {
|
||||
metadata: {
|
||||
category: { $in: ["electronics", "mobile"] },
|
||||
brand: "Apple",
|
||||
price: { $lt: 1000 }
|
||||
},
|
||||
nounTypes: ["product"],
|
||||
includeVerbs: true
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `searchText(query, k, options)`
|
||||
|
||||
**Text-based search with automatic embedding**
|
||||
|
||||
```typescript
|
||||
async searchText(
|
||||
query: string,
|
||||
k: number = 10,
|
||||
options: TextSearchOptions = {}
|
||||
): Promise<SearchResult<T>[]>
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `query` | `string` | Required | Text query to search for |
|
||||
| `k` | `number` | `10` | Maximum number of results to return |
|
||||
| `options` | `TextSearchOptions` | `{}` | Search configuration options |
|
||||
|
||||
#### Options
|
||||
|
||||
```typescript
|
||||
interface TextSearchOptions {
|
||||
nounTypes?: string[] // Filter by entity types
|
||||
includeVerbs?: boolean // Include relationships in results
|
||||
searchMode?: 'local' | 'remote' | 'combined'
|
||||
metadata?: MetadataFilter // MongoDB-style metadata filtering 🆕
|
||||
}
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```javascript
|
||||
const results = await brainy.searchText("gaming laptop", 15, {
|
||||
metadata: {
|
||||
availability: { $in: ["in_stock", "preorder"] },
|
||||
price: { $lte: 2000 },
|
||||
specs: { $all: ["RTX4060", "16GB RAM"] }
|
||||
},
|
||||
nounTypes: ["product"]
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `searchByNounTypes(queryVector, k, nounTypes, options)`
|
||||
|
||||
**Search within specific entity types with metadata filtering**
|
||||
|
||||
```typescript
|
||||
async searchByNounTypes(
|
||||
queryVectorOrData: Vector | any,
|
||||
k: number = 10,
|
||||
nounTypes: string[] | null = null,
|
||||
options: SearchByNounTypesOptions = {}
|
||||
): Promise<SearchResult<T>[]>
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `queryVectorOrData` | `Vector \| any` | Required | Query vector or data to be embedded |
|
||||
| `k` | `number` | `10` | Maximum number of results to return |
|
||||
| `nounTypes` | `string[] \| null` | `null` | Specific entity types to search within |
|
||||
| `options` | `SearchByNounTypesOptions` | `{}` | Search configuration options |
|
||||
|
||||
#### Options
|
||||
|
||||
```typescript
|
||||
interface SearchByNounTypesOptions {
|
||||
forceEmbed?: boolean // Force embedding even if input is a vector
|
||||
service?: string // Filter by service that created the data
|
||||
metadata?: MetadataFilter // MongoDB-style metadata filtering 🆕
|
||||
offset?: number // Pagination offset
|
||||
}
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```javascript
|
||||
const results = await brainy.searchByNounTypes(
|
||||
"organic coffee beans",
|
||||
20,
|
||||
["product", "food_item"],
|
||||
{
|
||||
metadata: {
|
||||
$and: [
|
||||
{ certification: "organic" },
|
||||
{ origin: { $includes: "Ethiopia" } },
|
||||
{ availability: { $ne: "out_of_stock" } }
|
||||
]
|
||||
},
|
||||
service: "ecommerce_platform"
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Metadata Filtering
|
||||
|
||||
### `MetadataFilter` Interface
|
||||
|
||||
The metadata filtering system supports MongoDB-style operators for complex queries:
|
||||
|
||||
```typescript
|
||||
type MetadataFilter = {
|
||||
[field: string]: any | {
|
||||
// Comparison operators
|
||||
$eq?: any
|
||||
$ne?: any
|
||||
$gt?: number | string | Date
|
||||
$gte?: number | string | Date
|
||||
$lt?: number | string | Date
|
||||
$lte?: number | string | Date
|
||||
|
||||
// Array operators
|
||||
$in?: any[]
|
||||
$nin?: any[]
|
||||
$all?: any[]
|
||||
$includes?: any
|
||||
$size?: number
|
||||
|
||||
// String operators
|
||||
$regex?: string
|
||||
$startsWith?: string
|
||||
$endsWith?: string
|
||||
$contains?: string
|
||||
|
||||
// Existence operators
|
||||
$exists?: boolean
|
||||
$type?: 'string' | 'number' | 'boolean' | 'object' | 'array'
|
||||
}
|
||||
|
||||
// Logical operators
|
||||
$and?: MetadataFilter[]
|
||||
$or?: MetadataFilter[]
|
||||
$not?: MetadataFilter
|
||||
$nor?: MetadataFilter[]
|
||||
}
|
||||
```
|
||||
|
||||
### Operator Examples
|
||||
|
||||
#### Comparison Operators
|
||||
|
||||
```javascript
|
||||
// Equal (implicit)
|
||||
{ category: "books" }
|
||||
|
||||
// Explicit comparison
|
||||
{
|
||||
price: { $lte: 50 },
|
||||
pages: { $gt: 200 },
|
||||
rating: { $ne: null }
|
||||
}
|
||||
```
|
||||
|
||||
#### Array Operators
|
||||
|
||||
```javascript
|
||||
{
|
||||
genres: { $in: ["Fiction", "Mystery", "Thriller"] }, // Has any of these genres
|
||||
awards: { $all: ["Hugo", "Nebula"] }, // Has all awards
|
||||
chapters: { $size: 12 }, // Exactly 12 chapters
|
||||
tags: { $includes: "bestseller" } // Array contains "bestseller"
|
||||
}
|
||||
```
|
||||
|
||||
#### String Operators
|
||||
|
||||
```javascript
|
||||
{
|
||||
isbn: { $regex: "^978-" }, // ISBN starts with 978
|
||||
title: { $startsWith: "The" }, // Title starts with "The"
|
||||
description: { $contains: "adventure" }, // Description contains "adventure"
|
||||
publisher: { $endsWith: "Press" } // Publisher ends with "Press"
|
||||
}
|
||||
```
|
||||
|
||||
#### Logical Operators
|
||||
|
||||
```javascript
|
||||
{
|
||||
$and: [
|
||||
{ category: "electronics" },
|
||||
{ warranty: true }
|
||||
],
|
||||
$or: [
|
||||
{ brand: "Apple" },
|
||||
{ brand: "Samsung" }
|
||||
],
|
||||
$not: { status: "discontinued" }
|
||||
}
|
||||
```
|
||||
|
||||
#### Nested Fields (Dot Notation)
|
||||
|
||||
```javascript
|
||||
{
|
||||
"specs.display.size": { $gte: 15 },
|
||||
"specs.processor.brand": "Intel",
|
||||
"ratings.average": { $gt: 4.5 }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Advanced Search Options
|
||||
|
||||
### Combining Multiple Filters
|
||||
|
||||
You can combine `metadata` filtering with other search options:
|
||||
|
||||
```javascript
|
||||
const results = await brainy.search("science textbook", 10, {
|
||||
// Entity type filtering
|
||||
nounTypes: ["book"],
|
||||
|
||||
// Service filtering
|
||||
service: "academic_platform",
|
||||
|
||||
// Metadata filtering
|
||||
metadata: {
|
||||
subject: { $in: ["physics", "chemistry"] },
|
||||
format: { $includes: "hardcover" },
|
||||
condition: { $ne: "damaged" }
|
||||
},
|
||||
|
||||
// Include relationships
|
||||
includeVerbs: true,
|
||||
|
||||
// Pagination
|
||||
offset: 20
|
||||
})
|
||||
```
|
||||
|
||||
### Search Modes
|
||||
|
||||
Control how search is performed:
|
||||
|
||||
```javascript
|
||||
const results = await brainy.searchText("cooking recipes", 10, {
|
||||
searchMode: "local", // Search only local data
|
||||
// searchMode: "remote", // Search only remote instances
|
||||
// searchMode: "combined", // Search local + remote (default)
|
||||
|
||||
metadata: {
|
||||
cuisine: { $includes: "italian" }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### Index Optimization
|
||||
|
||||
Metadata filtering uses automatic indexing for optimal performance:
|
||||
|
||||
- **Pre-filtering**: Uses indexes to identify candidates before vector search
|
||||
- **Automatic maintenance**: Indexes update when data changes
|
||||
- **Memory efficient**: LRU caching with configurable limits
|
||||
|
||||
### Query Performance Tips
|
||||
|
||||
1. **Use specific filters first**:
|
||||
```javascript
|
||||
// Faster: specific equality
|
||||
{ category: "books" }
|
||||
|
||||
// Slower: negation
|
||||
{ category: { $ne: "magazines" } }
|
||||
```
|
||||
|
||||
2. **Combine filters efficiently**:
|
||||
```javascript
|
||||
// Faster: most selective filter first
|
||||
{
|
||||
isbn: "978-0123456789", // High selectivity
|
||||
genre: { $includes: "mystery" }, // Medium selectivity
|
||||
in_stock: true // Low selectivity
|
||||
}
|
||||
```
|
||||
|
||||
3. **Use appropriate operators**:
|
||||
```javascript
|
||||
// For arrays, use array operators
|
||||
{ genres: { $includes: "mystery" } } // ✅ Correct
|
||||
{ genres: "mystery" } // ❌ Won't match arrays
|
||||
```
|
||||
|
||||
### Index Configuration
|
||||
|
||||
Configure indexing behavior:
|
||||
|
||||
```javascript
|
||||
const brainy = new BrainyData({
|
||||
metadataIndex: {
|
||||
maxIndexSize: 50000, // Max entries per field+value
|
||||
rebuildThreshold: 0.05, // Rebuild when 5% stale
|
||||
autoOptimize: true, // Auto-cleanup unused entries
|
||||
indexedFields: ["category", "brand"], // Only index specific fields
|
||||
excludeFields: ["internal_id", "temp"] // Never index sensitive fields
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Common Errors
|
||||
|
||||
```javascript
|
||||
try {
|
||||
const results = await brainy.search("query", 10, {
|
||||
metadata: { level: "senior" }
|
||||
})
|
||||
} catch (error) {
|
||||
if (error.message.includes('MetadataIndexError')) {
|
||||
// Index-related error
|
||||
console.error('Metadata index error:', error)
|
||||
} else if (error.message.includes('ValidationError')) {
|
||||
// Invalid query format
|
||||
console.error('Invalid metadata query:', error)
|
||||
} else {
|
||||
// Other search errors
|
||||
console.error('Search error:', error)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Query Validation
|
||||
|
||||
Brainy validates metadata queries and provides helpful error messages:
|
||||
|
||||
```javascript
|
||||
// Invalid operator
|
||||
{ price: { $invalid: 25 } }
|
||||
// Error: Unknown operator '$invalid'
|
||||
|
||||
// Type mismatch
|
||||
{ pages: { $gt: "not_a_number" } }
|
||||
// Error: $gt operator requires number, got string
|
||||
|
||||
// Invalid regex
|
||||
{ title: { $regex: "[invalid" } }
|
||||
// Error: Invalid regular expression
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### From Simple Filtering
|
||||
|
||||
```javascript
|
||||
// Before (v0.47.x and earlier)
|
||||
const results = await brainy.search("laptop", 10, {
|
||||
filter: { brand: "Apple" } // Simple object matching
|
||||
})
|
||||
|
||||
// After (v0.48.x+) - Backward compatible!
|
||||
const results = await brainy.search("laptop", 10, {
|
||||
metadata: {
|
||||
brand: "Apple", // Same simple matching
|
||||
price: { $lte: 2000 }, // Plus MongoDB operators
|
||||
specs: { $includes: "SSD" } // Plus array operations
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### Automatic Migration
|
||||
|
||||
- **No code changes required** - existing searches continue to work
|
||||
- **Indexes build automatically** - on first startup after upgrade
|
||||
- **Performance improves gradually** - as indexes populate
|
||||
|
||||
---
|
||||
|
||||
## Examples by Use Case
|
||||
|
||||
### Academic Library
|
||||
|
||||
```javascript
|
||||
// Find advanced textbooks
|
||||
const textbooks = await brainy.searchText("mathematics textbook", 20, {
|
||||
metadata: {
|
||||
level: { $in: ["undergraduate", "graduate", "advanced"] },
|
||||
subject: { $in: ["calculus", "algebra", "statistics"] },
|
||||
format: { $all: ["hardcover", "solutions_manual"] },
|
||||
availability: "in_stock"
|
||||
}
|
||||
})
|
||||
|
||||
// Find research papers with specific citations
|
||||
const researchPapers = await brainy.search(queryVector, 15, {
|
||||
metadata: {
|
||||
citations: { $gte: 10, $lte: 1000 },
|
||||
$or: [
|
||||
{ "author.degree": { $in: ["PhD", "Masters"] } },
|
||||
{ awards: { $size: { $gte: 1 } } }
|
||||
]
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### E-commerce Platform
|
||||
|
||||
```javascript
|
||||
// Product search with filters
|
||||
const products = await brainy.searchText("wireless headphones", 25, {
|
||||
metadata: {
|
||||
category: "electronics",
|
||||
price: { $lte: 200 },
|
||||
rating: { $gte: 4.0 },
|
||||
availability: { $ne: "out_of_stock" },
|
||||
features: { $all: ["bluetooth", "noise_canceling"] }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### Content Management
|
||||
|
||||
```javascript
|
||||
// Find published articles
|
||||
const articles = await brainy.searchText("climate change", 10, {
|
||||
metadata: {
|
||||
status: "published",
|
||||
publish_date: { $gte: "2023-01-01" },
|
||||
author: { $in: ["Dr. Green", "Prof. Earth"] },
|
||||
tags: { $includes: "environment" },
|
||||
word_count: { $gte: 1000 }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance Monitoring
|
||||
|
||||
### Index Statistics
|
||||
|
||||
```javascript
|
||||
// Get index performance metrics
|
||||
const stats = await brainy.metadataIndex?.getStats()
|
||||
console.log(`Index entries: ${stats.totalEntries}`)
|
||||
console.log(`Fields indexed: ${stats.fieldsIndexed}`)
|
||||
console.log(`Memory usage: ${stats.indexSize} bytes`)
|
||||
```
|
||||
|
||||
### Search Performance
|
||||
|
||||
```javascript
|
||||
// Monitor search performance
|
||||
const start = Date.now()
|
||||
const results = await brainy.search("query", 10, {
|
||||
metadata: { category: "electronics" }
|
||||
})
|
||||
const duration = Date.now() - start
|
||||
console.log(`Search completed in ${duration}ms`)
|
||||
console.log(`Found ${results.length} results`)
|
||||
```
|
||||
|
||||
This API reference provides complete documentation for all search methods with metadata filtering. The MongoDB-style operators give you powerful querying capabilities while maintaining high performance through automatic indexing.
|
||||
385
docs/guides/metadata-filtering.md
Normal file
385
docs/guides/metadata-filtering.md
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
# MongoDB-Style Metadata Filtering 🆕
|
||||
|
||||
**Advanced filtering for vector search with MongoDB-style query operators**
|
||||
|
||||
Brainy now supports sophisticated metadata filtering using familiar MongoDB query syntax. Filter your search results with complex criteria while maintaining high performance through automatic indexing.
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
```javascript
|
||||
import { BrainyData } from '@soulcraft/brainy'
|
||||
|
||||
const brainy = new BrainyData()
|
||||
await brainy.init()
|
||||
|
||||
// Add some data with metadata
|
||||
await brainy.add("Premium Wireless Headphones", {
|
||||
category: "electronics",
|
||||
brand: "Sony",
|
||||
price: 299,
|
||||
rating: 4.8,
|
||||
features: ["noise_canceling", "bluetooth"]
|
||||
})
|
||||
|
||||
await brainy.add("Budget Bluetooth Speaker", {
|
||||
category: "electronics",
|
||||
brand: "Anker",
|
||||
price: 49,
|
||||
rating: 4.2,
|
||||
features: ["bluetooth", "waterproof"]
|
||||
})
|
||||
|
||||
// Search with metadata filtering
|
||||
const results = await brainy.search("audio device", 10, {
|
||||
metadata: {
|
||||
category: "electronics",
|
||||
price: { $lte: 300 },
|
||||
features: { $in: ["bluetooth", "wireless"] }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## 📋 Query Operators
|
||||
|
||||
### Comparison Operators
|
||||
|
||||
| Operator | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$eq` | Equal (default) | `{ level: "senior" }` or `{ level: { $eq: "senior" } }` |
|
||||
| `$ne` | Not equal | `{ status: { $ne: "inactive" } }` |
|
||||
| `$gt` | Greater than | `{ salary: { $gt: 100000 } }` |
|
||||
| `$gte` | Greater than or equal | `{ experience: { $gte: 5 } }` |
|
||||
| `$lt` | Less than | `{ age: { $lt: 30 } }` |
|
||||
| `$lte` | Less than or equal | `{ rating: { $lte: 4.5 } }` |
|
||||
|
||||
### Array Operators
|
||||
|
||||
| Operator | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$in` | Value in array | `{ department: { $in: ["engineering", "product"] } }` |
|
||||
| `$nin` | Value not in array | `{ status: { $nin: ["fired", "inactive"] } }` |
|
||||
| `$all` | Array contains all values | `{ skills: { $all: ["React", "TypeScript"] } }` |
|
||||
| `$includes` | Array includes value | `{ tags: { $includes: "featured" } }` |
|
||||
| `$size` | Array has specific length | `{ projects: { $size: 3 } }` |
|
||||
|
||||
### String Operators
|
||||
|
||||
| Operator | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$regex` | Regular expression | `{ email: { $regex: ".*@company\\.com$" } }` |
|
||||
| `$startsWith` | String starts with | `{ name: { $startsWith: "John" } }` |
|
||||
| `$endsWith` | String ends with | `{ domain: { $endsWith: ".edu" } }` |
|
||||
| `$contains` | String contains | `{ bio: { $contains: "machine learning" } }` |
|
||||
|
||||
### Existence Operators
|
||||
|
||||
| Operator | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$exists` | Field exists | `{ linkedin: { $exists: true } }` |
|
||||
| `$type` | Field has specific type | `{ rating: { $type: "number" } }` |
|
||||
|
||||
### Logical Operators
|
||||
|
||||
| Operator | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `$and` | Logical AND (default) | `{ $and: [{ level: "senior" }, { remote: true }] }` |
|
||||
| `$or` | Logical OR | `{ $or: [{ location: "SF" }, { remote: true }] }` |
|
||||
| `$not` | Logical NOT | `{ $not: { status: "inactive" } }` |
|
||||
| `$nor` | Logical NOR | `{ $nor: [{ fired: true }, { resigned: true }] }` |
|
||||
|
||||
## 🎯 Real-World Examples
|
||||
|
||||
### E-commerce Platform
|
||||
|
||||
```javascript
|
||||
// Find premium electronics in specific price ranges
|
||||
const premiumProducts = await brainy.search("smartphone", 20, {
|
||||
metadata: {
|
||||
category: { $in: ["electronics", "mobile", "phones"] },
|
||||
brand: { $in: ["Apple", "Samsung", "Google"] },
|
||||
price: { $gte: 800 },
|
||||
features: { $all: ["5G", "wireless_charging"] },
|
||||
availability: { $ne: "out_of_stock" }
|
||||
}
|
||||
})
|
||||
|
||||
// Find budget-friendly options
|
||||
const budgetOptions = await brainy.search("laptop", 15, {
|
||||
metadata: {
|
||||
$or: [
|
||||
{ category: "refurbished" },
|
||||
{ discount: true },
|
||||
{ price: { $lte: 500 } }
|
||||
],
|
||||
rating: { $gte: 4.0 }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### E-commerce Product Search
|
||||
|
||||
```javascript
|
||||
// Electronics under $500 with good ratings
|
||||
const products = await brainy.search("laptop computer", 10, {
|
||||
metadata: {
|
||||
category: "electronics",
|
||||
price: { $lte: 500 },
|
||||
rating: { $gte: 4.0 },
|
||||
availability: { $ne: "out_of_stock" },
|
||||
tags: { $includes: "bestseller" }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### Academic Research
|
||||
|
||||
```javascript
|
||||
// Recent AI papers from top venues
|
||||
const papers = await brainy.search("machine learning", 25, {
|
||||
metadata: {
|
||||
type: "academic_paper",
|
||||
year: { $gte: 2022 },
|
||||
venue: { $in: ["NeurIPS", "ICML", "ICLR", "AAAI"] },
|
||||
citations: { $gt: 10 },
|
||||
open_access: true
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### Content Management
|
||||
|
||||
```javascript
|
||||
// Published blog posts by specific authors
|
||||
const posts = await brainy.search("artificial intelligence", 10, {
|
||||
metadata: {
|
||||
status: "published",
|
||||
author: { $in: ["John Smith", "Jane Doe"] },
|
||||
publish_date: { $gte: "2023-01-01" },
|
||||
tags: { $all: ["AI", "technology"] },
|
||||
word_count: { $gte: 1000, $lte: 5000 }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## 🌳 Nested Fields (Dot Notation)
|
||||
|
||||
Access nested object fields using dot notation:
|
||||
|
||||
```javascript
|
||||
await brainy.add("Gaming Laptop", {
|
||||
specs: {
|
||||
display: { size: 17.3, resolution: "4K" },
|
||||
processor: { brand: "Intel", model: "i9-12900H" }
|
||||
},
|
||||
ratings: { average: 4.8, total_reviews: 342 }
|
||||
})
|
||||
|
||||
// Search using nested fields
|
||||
const results = await brainy.search("laptop", 5, {
|
||||
metadata: {
|
||||
"specs.display.size": { $gte: 15 },
|
||||
"specs.processor.brand": "Intel",
|
||||
"ratings.average": { $gt: 4.5 }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## 🚀 Performance Features
|
||||
|
||||
### Automatic Indexing
|
||||
|
||||
- **Zero Configuration**: Indexes are built automatically when you add data
|
||||
- **Smart Field Selection**: Common fields like `id`, `createdAt`, `updatedAt` are excluded by default
|
||||
- **Incremental Updates**: Indexes update automatically when data changes
|
||||
- **Memory Efficient**: LRU caching with automatic cleanup
|
||||
|
||||
### Pre-filtering Optimization
|
||||
|
||||
Brainy uses metadata indexes to pre-filter candidates before vector search:
|
||||
|
||||
```javascript
|
||||
// This is FAST! Pre-filters using indexes, then searches only matching vectors
|
||||
const results = await brainy.search("electronics", 10, {
|
||||
metadata: { category: "smartphones" } // Only searches smartphone vectors
|
||||
})
|
||||
```
|
||||
|
||||
### Index Statistics
|
||||
|
||||
Monitor your metadata indexes:
|
||||
|
||||
```javascript
|
||||
const stats = await brainy.metadataIndex.getStats()
|
||||
console.log(`Index entries: ${stats.totalEntries}`)
|
||||
console.log(`Fields indexed: ${stats.fieldsIndexed.join(', ')}`)
|
||||
console.log(`Memory usage: ${stats.indexSize} bytes`)
|
||||
```
|
||||
|
||||
## ⚙️ Configuration Options
|
||||
|
||||
Customize metadata indexing behavior:
|
||||
|
||||
```javascript
|
||||
const brainy = new BrainyData({
|
||||
metadataIndex: {
|
||||
maxIndexSize: 50000, // Max entries per field+value
|
||||
rebuildThreshold: 0.05, // Rebuild when 5% stale
|
||||
autoOptimize: true, // Auto-cleanup unused entries
|
||||
indexedFields: ["category", "brand"], // Only index these fields
|
||||
excludeFields: ["internal_id", "temp"] // Never index these fields
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## 🔧 Advanced Patterns
|
||||
|
||||
### Complex Logical Queries
|
||||
|
||||
```javascript
|
||||
// Find products matching complex criteria
|
||||
const results = await brainy.search("kitchen appliances", 10, {
|
||||
metadata: {
|
||||
$and: [
|
||||
{
|
||||
$or: [
|
||||
{ brand: "KitchenAid" },
|
||||
{ warranty_years: { $gte: 2 } }
|
||||
]
|
||||
},
|
||||
{
|
||||
rating: { $gte: 4.0 }
|
||||
},
|
||||
{
|
||||
features: { $all: ["dishwasher_safe", "BPA_free"] }
|
||||
},
|
||||
{
|
||||
$not: { status: "discontinued" }
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### Dynamic Query Building
|
||||
|
||||
```javascript
|
||||
function buildProductQuery(filters) {
|
||||
const query = {}
|
||||
|
||||
if (filters.maxPrice) {
|
||||
query.price = { $lte: filters.maxPrice }
|
||||
}
|
||||
|
||||
if (filters.brands?.length) {
|
||||
query.brand = { $in: filters.brands }
|
||||
}
|
||||
|
||||
if (filters.requiredFeatures?.length) {
|
||||
query.features = { $all: filters.requiredFeatures }
|
||||
}
|
||||
|
||||
if (filters.excludeOutOfStock) {
|
||||
query.availability = { $ne: "out_of_stock" }
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// Use dynamic query
|
||||
const searchQuery = buildProductQuery({
|
||||
maxPrice: 1000,
|
||||
brands: ["Apple", "Samsung"],
|
||||
requiredFeatures: ["5G", "wireless_charging"],
|
||||
excludeOutOfStock: true
|
||||
})
|
||||
|
||||
const results = await brainy.search("smartphone", 10, {
|
||||
metadata: searchQuery
|
||||
})
|
||||
```
|
||||
|
||||
## 📈 Best Practices
|
||||
|
||||
### 1. Index Strategy
|
||||
- **Include searchable fields**: Category, brand, price, features
|
||||
- **Exclude volatile fields**: Last viewed, view count, temporary flags
|
||||
- **Use consistent naming**: Prefer `snake_case` or `camelCase` consistently
|
||||
|
||||
### 2. Query Optimization
|
||||
- **Use specific filters**: `{ category: "books" }` is faster than `{ category: { $ne: "magazines" } }`
|
||||
- **Combine with other filters**: Use `nounTypes` and `metadata` together for best performance
|
||||
- **Avoid regex on large datasets**: Pre-process text fields when possible
|
||||
|
||||
### 3. Data Modeling
|
||||
```javascript
|
||||
// Good: Structured metadata
|
||||
await brainy.add("Smartphone", {
|
||||
category: "electronics", // String enum
|
||||
price: 899, // Number for range queries
|
||||
features: ["5G", "wireless"], // Array for $in/$all queries
|
||||
in_stock: true, // Boolean for exact matching
|
||||
brand: "Apple" // String for exact/regex matching
|
||||
})
|
||||
|
||||
// Avoid: Unstructured metadata
|
||||
await brainy.add("Smartphone", {
|
||||
description: "Premium smartphone with 5G and wireless charging features from Apple"
|
||||
})
|
||||
```
|
||||
|
||||
## 🔄 Migration from Simple Filtering
|
||||
|
||||
If you were using basic filtering, upgrading is seamless:
|
||||
|
||||
```javascript
|
||||
// Before (still works!)
|
||||
const results = await brainy.search("laptop", 10, {
|
||||
filter: { category: "electronics" }
|
||||
})
|
||||
|
||||
// After (more powerful!)
|
||||
const results = await brainy.search("laptop", 10, {
|
||||
metadata: {
|
||||
category: "electronics",
|
||||
brand: { $in: ["Apple", "Dell"] },
|
||||
features: { $includes: "SSD" }
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## 🚨 Common Gotchas
|
||||
|
||||
1. **Case Sensitivity**: String matching is case-sensitive by default
|
||||
```javascript
|
||||
// Won't match "Electronics"
|
||||
{ category: "electronics" }
|
||||
|
||||
// Use regex for case-insensitive
|
||||
{ category: { $regex: "electronics", $options: "i" } }
|
||||
```
|
||||
|
||||
2. **Array vs Single Values**:
|
||||
```javascript
|
||||
// If features is ["bluetooth", "wireless"]
|
||||
{ features: "bluetooth" } // ❌ Won't match
|
||||
{ features: { $includes: "bluetooth" } } // ✅ Matches
|
||||
```
|
||||
|
||||
3. **Nested Field Access**:
|
||||
```javascript
|
||||
// Use dot notation for nested fields
|
||||
{ "specs.display": "4K" } // ✅ Correct
|
||||
{ specs: { display: "4K" } } // ❌ Won't work as expected
|
||||
```
|
||||
|
||||
## 🎉 What's Next?
|
||||
|
||||
This powerful filtering system opens up possibilities for:
|
||||
- **Advanced search UIs** with multiple filter controls
|
||||
- **Personalized recommendations** based on user preferences
|
||||
- **Complex business logic** in search applications
|
||||
- **Multi-tenant filtering** by organization or user
|
||||
|
||||
The filtering happens **during the vector search** (not after), ensuring maximum performance even with complex queries!
|
||||
|
||||
Ready to build something amazing? Check out the [API Reference](../api-reference/search-methods.md) for complete method signatures and options.
|
||||
Loading…
Add table
Add a link
Reference in a new issue