**feat(search): enhance JSON document search with field-level filtering and prioritization**
- Added support for field-specific and prioritized searches in `brainyData`: - Introduced `searchField` option to enable targeted field-level searches. - Implemented `priorityFields` option for weighted vectorization and query relevance. - Developed utilities in `jsonProcessing.ts` and `fieldNameTracking.ts`: - `extractTextFromJson` for text extraction with customizable depth and field prioritization. - `extractFieldFromJson` to target specific fields in JSON documents. - `prepareJsonForVectorization` for optimized JSON vectorization. - Enhanced management of field names and mappings: - Integrated `trackFieldNames` to associate fields with their services. - Supported cross-service consistency through `standardFieldMappings`. - Updated documentation: - Added detailed guides for JSON search enhancements and HNSW limitations. - Extended usage examples in `README.md` and `json-search-test.js`. - Verified improvements with comprehensive tests: - Created unit and integration tests demonstrating search behavior improvements. - Addressed previous TypeScript errors related to search parameters. **Purpose**: Improve search accuracy and usability when working with complex JSON documents by enabling field-specific searches and enhancing contextual relevance.
This commit is contained in:
parent
81d32e0662
commit
f86295eab8
11 changed files with 1088 additions and 10 deletions
114
src/utils/fieldNameTracking.ts
Normal file
114
src/utils/fieldNameTracking.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/**
|
||||
* Utility functions for tracking and managing field names in JSON documents
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extracts field names from a JSON document
|
||||
* @param jsonObject The JSON object to extract field names from
|
||||
* @param options Configuration options
|
||||
* @returns An array of field paths (e.g., "user.name", "addresses[0].city")
|
||||
*/
|
||||
export function extractFieldNamesFromJson(
|
||||
jsonObject: any,
|
||||
options: {
|
||||
maxDepth?: number
|
||||
currentDepth?: number
|
||||
currentPath?: string
|
||||
fieldNames?: Set<string>
|
||||
} = {}
|
||||
): string[] {
|
||||
const {
|
||||
maxDepth = 5,
|
||||
currentDepth = 0,
|
||||
currentPath = '',
|
||||
fieldNames = new Set<string>()
|
||||
} = options
|
||||
|
||||
if (
|
||||
jsonObject === null ||
|
||||
jsonObject === undefined ||
|
||||
typeof jsonObject !== 'object' ||
|
||||
currentDepth >= maxDepth
|
||||
) {
|
||||
return Array.from(fieldNames)
|
||||
}
|
||||
|
||||
if (Array.isArray(jsonObject)) {
|
||||
// For arrays, we'll just check the first item to avoid explosion of paths
|
||||
if (jsonObject.length > 0) {
|
||||
const arrayPath = currentPath ? `${currentPath}[0]` : '[0]'
|
||||
extractFieldNamesFromJson(jsonObject[0], {
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
currentPath: arrayPath,
|
||||
fieldNames
|
||||
})
|
||||
}
|
||||
} else {
|
||||
// For objects, process each property
|
||||
for (const key of Object.keys(jsonObject)) {
|
||||
const value = jsonObject[key]
|
||||
const fieldPath = currentPath ? `${currentPath}.${key}` : key
|
||||
|
||||
// Add this field path
|
||||
fieldNames.add(fieldPath)
|
||||
|
||||
// Recursively process nested objects
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
extractFieldNamesFromJson(value, {
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
currentPath: fieldPath,
|
||||
fieldNames
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(fieldNames)
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps field names to standard field names based on common patterns
|
||||
* @param fieldName The field name to map
|
||||
* @returns The standard field name if a match is found, or null if no match
|
||||
*/
|
||||
export function mapToStandardField(fieldName: string): string | null {
|
||||
// Standard field mappings
|
||||
const standardMappings: Record<string, string[]> = {
|
||||
'title': ['title', 'name', 'headline', 'subject'],
|
||||
'description': ['description', 'summary', 'content', 'text', 'body'],
|
||||
'author': ['author', 'creator', 'user', 'owner', 'by'],
|
||||
'date': ['date', 'created', 'createdAt', 'timestamp', 'published'],
|
||||
'url': ['url', 'link', 'href', 'source'],
|
||||
'image': ['image', 'thumbnail', 'photo', 'picture'],
|
||||
'tags': ['tags', 'categories', 'keywords', 'topics']
|
||||
}
|
||||
|
||||
// Check for matches
|
||||
for (const [standardField, possibleMatches] of Object.entries(standardMappings)) {
|
||||
// Exact match
|
||||
if (possibleMatches.includes(fieldName)) {
|
||||
return standardField
|
||||
}
|
||||
|
||||
// Path match (e.g., "user.name" matches "name")
|
||||
const parts = fieldName.split('.')
|
||||
const lastPart = parts[parts.length - 1]
|
||||
if (possibleMatches.includes(lastPart)) {
|
||||
return standardField
|
||||
}
|
||||
|
||||
// Array match (e.g., "items[0].name" matches "name")
|
||||
if (fieldName.includes('[')) {
|
||||
for (const part of parts) {
|
||||
const cleanPart = part.split('[')[0]
|
||||
if (possibleMatches.includes(cleanPart)) {
|
||||
return standardField
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
|
@ -2,3 +2,5 @@ export * from './distance.js'
|
|||
export * from './embedding.js'
|
||||
export * from './workerUtils.js'
|
||||
export * from './statistics.js'
|
||||
export * from './jsonProcessing.js'
|
||||
export * from './fieldNameTracking.js'
|
||||
|
|
|
|||
226
src/utils/jsonProcessing.ts
Normal file
226
src/utils/jsonProcessing.ts
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* Utility functions for processing JSON documents for vectorization and search
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extracts text from a JSON object for vectorization
|
||||
* This function recursively processes the JSON object and extracts text from all fields
|
||||
* It can also prioritize specific fields if provided
|
||||
*
|
||||
* @param jsonObject The JSON object to extract text from
|
||||
* @param options Configuration options for text extraction
|
||||
* @returns A string containing the extracted text
|
||||
*/
|
||||
export function extractTextFromJson(
|
||||
jsonObject: any,
|
||||
options: {
|
||||
priorityFields?: string[] // Fields to prioritize (will be repeated for emphasis)
|
||||
excludeFields?: string[] // Fields to exclude from extraction
|
||||
includeFieldNames?: boolean // Whether to include field names in the extracted text
|
||||
maxDepth?: number // Maximum depth to recurse into nested objects
|
||||
currentDepth?: number // Current recursion depth (internal use)
|
||||
fieldPath?: string[] // Current field path (internal use)
|
||||
} = {}
|
||||
): string {
|
||||
// Set default options
|
||||
const {
|
||||
priorityFields = [],
|
||||
excludeFields = [],
|
||||
includeFieldNames = true,
|
||||
maxDepth = 5,
|
||||
currentDepth = 0,
|
||||
fieldPath = []
|
||||
} = options
|
||||
|
||||
// If input is not an object or array, or we've reached max depth, return as string
|
||||
if (
|
||||
jsonObject === null ||
|
||||
jsonObject === undefined ||
|
||||
typeof jsonObject !== 'object' ||
|
||||
currentDepth >= maxDepth
|
||||
) {
|
||||
return String(jsonObject || '')
|
||||
}
|
||||
|
||||
const extractedText: string[] = []
|
||||
const priorityText: string[] = []
|
||||
|
||||
// Process arrays
|
||||
if (Array.isArray(jsonObject)) {
|
||||
for (let i = 0; i < jsonObject.length; i++) {
|
||||
const value = jsonObject[i]
|
||||
const newPath = [...fieldPath, i.toString()]
|
||||
|
||||
// Recursively extract text from array items
|
||||
const itemText = extractTextFromJson(value, {
|
||||
priorityFields,
|
||||
excludeFields,
|
||||
includeFieldNames,
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
fieldPath: newPath
|
||||
})
|
||||
|
||||
if (itemText) {
|
||||
extractedText.push(itemText)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process objects
|
||||
else {
|
||||
for (const [key, value] of Object.entries(jsonObject)) {
|
||||
// Skip excluded fields
|
||||
if (excludeFields.includes(key)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const newPath = [...fieldPath, key]
|
||||
const fullPath = newPath.join('.')
|
||||
|
||||
// Check if this is a priority field
|
||||
const isPriority = priorityFields.some(field => {
|
||||
// Exact match
|
||||
if (field === key) return true
|
||||
// Path match
|
||||
if (field === fullPath) return true
|
||||
// Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
|
||||
if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2))) return true
|
||||
return false
|
||||
})
|
||||
|
||||
// Get the field value as text
|
||||
let fieldText: string
|
||||
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
// Recursively extract text from nested objects
|
||||
fieldText = extractTextFromJson(value, {
|
||||
priorityFields,
|
||||
excludeFields,
|
||||
includeFieldNames,
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
fieldPath: newPath
|
||||
})
|
||||
} else {
|
||||
fieldText = String(value || '')
|
||||
}
|
||||
|
||||
// Add field name if requested
|
||||
if (includeFieldNames && fieldText) {
|
||||
fieldText = `${key}: ${fieldText}`
|
||||
}
|
||||
|
||||
// Add to appropriate collection
|
||||
if (fieldText) {
|
||||
if (isPriority) {
|
||||
priorityText.push(fieldText)
|
||||
} else {
|
||||
extractedText.push(fieldText)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combine priority text (repeated for emphasis) and regular text
|
||||
return [...priorityText, ...priorityText, ...extractedText].join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares a JSON document for vectorization
|
||||
* This function extracts text from the JSON document and formats it for optimal vectorization
|
||||
*
|
||||
* @param jsonDocument The JSON document to prepare
|
||||
* @param options Configuration options for preparation
|
||||
* @returns A string ready for vectorization
|
||||
*/
|
||||
export function prepareJsonForVectorization(
|
||||
jsonDocument: any,
|
||||
options: {
|
||||
priorityFields?: string[]
|
||||
excludeFields?: string[]
|
||||
includeFieldNames?: boolean
|
||||
maxDepth?: number
|
||||
} = {}
|
||||
): string {
|
||||
// If input is a string, try to parse it as JSON
|
||||
let document = jsonDocument
|
||||
if (typeof jsonDocument === 'string') {
|
||||
try {
|
||||
document = JSON.parse(jsonDocument)
|
||||
} catch (e) {
|
||||
// If parsing fails, treat it as a plain string
|
||||
return jsonDocument
|
||||
}
|
||||
}
|
||||
|
||||
// If not an object after parsing, return as is
|
||||
if (typeof document !== 'object' || document === null) {
|
||||
return String(document || '')
|
||||
}
|
||||
|
||||
// Extract text from the document
|
||||
return extractTextFromJson(document, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts text from a specific field in a JSON document
|
||||
* This is useful for searching within specific fields
|
||||
*
|
||||
* @param jsonDocument The JSON document to extract from
|
||||
* @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
|
||||
* @returns The extracted text or empty string if field not found
|
||||
*/
|
||||
export function extractFieldFromJson(
|
||||
jsonDocument: any,
|
||||
fieldPath: string
|
||||
): string {
|
||||
// If input is a string, try to parse it as JSON
|
||||
let document = jsonDocument
|
||||
if (typeof jsonDocument === 'string') {
|
||||
try {
|
||||
document = JSON.parse(jsonDocument)
|
||||
} catch (e) {
|
||||
// If parsing fails, return empty string
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// If not an object after parsing, return empty string
|
||||
if (typeof document !== 'object' || document === null) {
|
||||
return ''
|
||||
}
|
||||
|
||||
// Parse the field path
|
||||
const parts = fieldPath.split('.')
|
||||
let current = document
|
||||
|
||||
// Navigate through the path
|
||||
for (const part of parts) {
|
||||
// Handle array indexing (e.g., "addresses[0]")
|
||||
const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/)
|
||||
if (!match) {
|
||||
return ''
|
||||
}
|
||||
|
||||
const [, key, indexStr] = match
|
||||
|
||||
// Move to the next level
|
||||
current = current[key]
|
||||
|
||||
// If we have an array index, access that element
|
||||
if (indexStr !== undefined && Array.isArray(current)) {
|
||||
const index = parseInt(indexStr, 10)
|
||||
current = current[index]
|
||||
}
|
||||
|
||||
// If we've reached a null or undefined value, return empty string
|
||||
if (current === null || current === undefined) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// Convert the final value to string
|
||||
return typeof current === 'object'
|
||||
? JSON.stringify(current)
|
||||
: String(current)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue