**feat(search): enhance JSON document search with field-level filtering and prioritization**
- Added support for field-specific and prioritized searches in `brainyData`: - Introduced `searchField` option to enable targeted field-level searches. - Implemented `priorityFields` option for weighted vectorization and query relevance. - Developed utilities in `jsonProcessing.ts` and `fieldNameTracking.ts`: - `extractTextFromJson` for text extraction with customizable depth and field prioritization. - `extractFieldFromJson` to target specific fields in JSON documents. - `prepareJsonForVectorization` for optimized JSON vectorization. - Enhanced management of field names and mappings: - Integrated `trackFieldNames` to associate fields with their services. - Supported cross-service consistency through `standardFieldMappings`. - Updated documentation: - Added detailed guides for JSON search enhancements and HNSW limitations. - Extended usage examples in `README.md` and `json-search-test.js`. - Verified improvements with comprehensive tests: - Created unit and integration tests demonstrating search behavior improvements. - Addressed previous TypeScript errors related to search parameters. **Purpose**: Improve search accuracy and usability when working with complex JSON documents by enabling field-specific searches and enhancing contextual relevance.
This commit is contained in:
parent
ed417c9393
commit
a288ad9434
11 changed files with 1088 additions and 10 deletions
|
|
@ -42,6 +42,10 @@ import {
|
|||
} from './types/augmentations.js'
|
||||
import { BrainyDataInterface } from './types/brainyDataInterface.js'
|
||||
import { augmentationPipeline } from './augmentationPipeline.js'
|
||||
import {
|
||||
prepareJsonForVectorization,
|
||||
extractFieldFromJson
|
||||
} from './utils/jsonProcessing.js'
|
||||
|
||||
export interface BrainyDataConfig {
|
||||
/**
|
||||
|
|
@ -317,7 +321,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
if (config.storageAdapter) {
|
||||
hnswConfig.useDiskBasedIndex = true
|
||||
}
|
||||
|
||||
|
||||
this.index = new HNSWIndexOptimized(
|
||||
hnswConfig,
|
||||
this.distanceFunction,
|
||||
|
|
@ -755,6 +759,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the service name from options or fallback to current augmentation
|
||||
* This provides a consistent way to handle service names across all methods
|
||||
* @param options Options object that may contain a service property
|
||||
* @returns The service name to use for operations
|
||||
*/
|
||||
private getServiceName(options?: { service?: string }): string {
|
||||
if (options?.service) {
|
||||
return options.service
|
||||
}
|
||||
return this.getCurrentAugmentation()
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the database
|
||||
* Loads existing data from storage if available
|
||||
|
|
@ -1000,7 +1017,35 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
} else {
|
||||
// Input needs to be vectorized
|
||||
try {
|
||||
vector = await this.embeddingFunction(vectorOrData)
|
||||
// Check if input is a JSON object and process it specially
|
||||
if (
|
||||
typeof vectorOrData === 'object' &&
|
||||
vectorOrData !== null &&
|
||||
!Array.isArray(vectorOrData)
|
||||
) {
|
||||
// Process JSON object for better vectorization
|
||||
const preparedText = prepareJsonForVectorization(vectorOrData, {
|
||||
// Prioritize common name/title fields if they exist
|
||||
priorityFields: [
|
||||
'name',
|
||||
'title',
|
||||
'company',
|
||||
'organization',
|
||||
'description',
|
||||
'summary'
|
||||
]
|
||||
})
|
||||
vector = await this.embeddingFunction(preparedText)
|
||||
|
||||
// Track field names for this JSON document
|
||||
const service = this.getServiceName(options)
|
||||
if (this.storage) {
|
||||
await this.storage.trackFieldNames(vectorOrData, service)
|
||||
}
|
||||
} else {
|
||||
// Use standard embedding for non-JSON data
|
||||
vector = await this.embeddingFunction(vectorOrData)
|
||||
}
|
||||
} catch (embedError) {
|
||||
throw new Error(`Failed to vectorize data: ${embedError}`)
|
||||
}
|
||||
|
|
@ -1039,7 +1084,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
await this.storage!.saveNoun(noun)
|
||||
|
||||
// Track noun statistics
|
||||
const service = options.service || this.getCurrentAugmentation()
|
||||
const service = this.getServiceName(options)
|
||||
await this.storage!.incrementStatistic('noun', service)
|
||||
|
||||
// Save metadata if provided and not empty
|
||||
|
|
@ -1107,8 +1152,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
await this.storage!.saveMetadata(id, metadataToSave)
|
||||
|
||||
// Track metadata statistics
|
||||
const metadataService =
|
||||
options.service || this.getCurrentAugmentation()
|
||||
const metadataService = this.getServiceName(options)
|
||||
await this.storage!.incrementStatistic('metadata', metadataService)
|
||||
}
|
||||
}
|
||||
|
|
@ -1629,6 +1673,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
searchConnectedNouns?: boolean // Whether to search for nouns connected by verbs
|
||||
verbDirection?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider when searching connected nouns
|
||||
service?: string // Filter results by the service that created the data
|
||||
searchField?: string // Optional specific field to search within JSON documents
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
// Validate input is not null or undefined
|
||||
|
|
@ -1707,6 +1752,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
nounTypes?: string[] // Optional array of noun types to search within
|
||||
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
|
||||
service?: string // Filter results by the service that created the data
|
||||
searchField?: string // Optional specific field to search within JSON documents
|
||||
priorityFields?: string[] // Fields to prioritize when searching JSON documents
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
if (!this.isInitialized) {
|
||||
|
|
@ -1717,12 +1764,49 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
// If input is a string and not a vector, automatically vectorize it
|
||||
// Process the query input for vectorization
|
||||
let queryToUse = queryVectorOrData
|
||||
|
||||
// Handle string queries
|
||||
if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
|
||||
queryToUse = await this.embed(queryVectorOrData)
|
||||
options.forceEmbed = false // Already embedded, don't force again
|
||||
}
|
||||
// Handle JSON object queries with special processing
|
||||
else if (
|
||||
typeof queryVectorOrData === 'object' &&
|
||||
queryVectorOrData !== null &&
|
||||
!Array.isArray(queryVectorOrData) &&
|
||||
!options.forceEmbed
|
||||
) {
|
||||
// If searching within a specific field
|
||||
if (options.searchField) {
|
||||
// Extract text from the specific field
|
||||
const fieldText = extractFieldFromJson(
|
||||
queryVectorOrData,
|
||||
options.searchField
|
||||
)
|
||||
if (fieldText) {
|
||||
queryToUse = await this.embeddingFunction(fieldText)
|
||||
options.forceEmbed = false // Already embedded, don't force again
|
||||
}
|
||||
}
|
||||
// Otherwise process the entire object with priority fields
|
||||
else {
|
||||
const preparedText = prepareJsonForVectorization(queryVectorOrData, {
|
||||
priorityFields: options.priorityFields || [
|
||||
'name',
|
||||
'title',
|
||||
'company',
|
||||
'organization',
|
||||
'description',
|
||||
'summary'
|
||||
]
|
||||
})
|
||||
queryToUse = await this.embeddingFunction(preparedText)
|
||||
options.forceEmbed = false // Already embedded, don't force again
|
||||
}
|
||||
}
|
||||
|
||||
// If noun types are specified, use searchByNounTypes
|
||||
let searchResults
|
||||
|
|
@ -2101,7 +2185,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
await this.storage!.deleteNoun(actualId)
|
||||
|
||||
// Track deletion statistics
|
||||
const service = options.service || 'default'
|
||||
const service = this.getServiceName(options)
|
||||
await this.storage!.decrementStatistic('noun', service)
|
||||
|
||||
// Try to remove metadata (ignore errors)
|
||||
|
|
@ -2528,7 +2612,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
await this.storage!.saveVerb(verb)
|
||||
|
||||
// Track verb statistics
|
||||
const serviceForStats = options.service || 'default'
|
||||
const serviceForStats = this.getServiceName(options)
|
||||
await this.storage!.incrementStatistic('verb', serviceForStats)
|
||||
|
||||
// Update HNSW index size (excluding verbs)
|
||||
|
|
@ -2715,7 +2799,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
await this.storage!.deleteVerb(id)
|
||||
|
||||
// Track deletion statistics
|
||||
const service = options.service || 'default'
|
||||
const service = this.getServiceName(options)
|
||||
await this.storage!.decrementStatistic('verb', service)
|
||||
|
||||
return true
|
||||
|
|
@ -3373,6 +3457,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
|
||||
storeResults?: boolean // Whether to store the results in the local database (default: true)
|
||||
service?: string // Filter results by the service that created the data
|
||||
searchField?: string // Optional specific field to search within JSON documents
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -3438,6 +3523,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
|
||||
localFirst?: boolean // Whether to search local first (default: true)
|
||||
service?: string // Filter results by the service that created the data
|
||||
searchField?: string // Optional specific field to search within JSON documents
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -3967,7 +4053,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
if (this.storage) {
|
||||
hnswConfig.useDiskBasedIndex = true
|
||||
}
|
||||
|
||||
|
||||
this.index = new HNSWIndexOptimized(
|
||||
hnswConfig,
|
||||
this.distanceFunction,
|
||||
|
|
@ -4164,6 +4250,109 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
throw new Error(`Failed to generate random graph: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available field names by service
|
||||
* This helps users understand what fields are available for searching from different data sources
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
public async getAvailableFieldNames(): Promise<Record<string, string[]>> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.storage) {
|
||||
return {}
|
||||
}
|
||||
|
||||
return this.storage.getAvailableFieldNames()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* This helps users understand how fields from different services map to standard field names
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
public async getStandardFieldMappings(): Promise<
|
||||
Record<string, Record<string, string[]>>
|
||||
> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.storage) {
|
||||
return {}
|
||||
}
|
||||
|
||||
return this.storage.getStandardFieldMappings()
|
||||
}
|
||||
|
||||
/**
|
||||
* Search using a standard field name
|
||||
* This allows searching across multiple services using a standardized field name
|
||||
* @param standardField The standard field name to search in
|
||||
* @param searchTerm The term to search for
|
||||
* @param k Number of results to return
|
||||
* @param options Additional search options
|
||||
* @returns Array of search results
|
||||
*/
|
||||
public async searchByStandardField(
|
||||
standardField: string,
|
||||
searchTerm: string,
|
||||
k: number = 10,
|
||||
options: {
|
||||
services?: string[]
|
||||
includeVerbs?: boolean
|
||||
searchMode?: 'local' | 'remote' | 'combined'
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Check if database is in write-only mode
|
||||
this.checkWriteOnly()
|
||||
|
||||
// Get standard field mappings
|
||||
const standardFieldMappings = await this.getStandardFieldMappings()
|
||||
|
||||
// If the standard field doesn't exist, return empty results
|
||||
if (!standardFieldMappings[standardField]) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Filter by services if specified
|
||||
let serviceFieldMappings = standardFieldMappings[standardField]
|
||||
if (options.services && options.services.length > 0) {
|
||||
const filteredMappings: Record<string, string[]> = {}
|
||||
for (const service of options.services) {
|
||||
if (serviceFieldMappings[service]) {
|
||||
filteredMappings[service] = serviceFieldMappings[service]
|
||||
}
|
||||
}
|
||||
serviceFieldMappings = filteredMappings
|
||||
}
|
||||
|
||||
// If no mappings after filtering, return empty results
|
||||
if (Object.keys(serviceFieldMappings).length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Search in each service's fields and combine results
|
||||
const allResults: SearchResult<T>[] = []
|
||||
|
||||
for (const [service, fieldNames] of Object.entries(serviceFieldMappings)) {
|
||||
for (const fieldName of fieldNames) {
|
||||
// Search using the specific field name for this service
|
||||
const results = await this.search(searchTerm, k, {
|
||||
searchField: fieldName,
|
||||
service,
|
||||
includeVerbs: options.includeVerbs,
|
||||
searchMode: options.searchMode
|
||||
})
|
||||
|
||||
// Add results to the combined list
|
||||
allResults.push(...results)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score and limit to k results
|
||||
return allResults.sort((a, b) => b.score - a.score).slice(0, k)
|
||||
}
|
||||
}
|
||||
|
||||
// Export distance functions for convenience
|
||||
|
|
|
|||
|
|
@ -155,6 +155,18 @@ export interface StatisticsData {
|
|||
total: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Field names available for searching, organized by service
|
||||
* This helps users understand what fields are available from different data sources
|
||||
*/
|
||||
fieldNames?: Record<string, string[]>
|
||||
|
||||
/**
|
||||
* Standard field mappings for common field names across services
|
||||
* Maps standard field names to the actual field names used by each service
|
||||
*/
|
||||
standardFieldMappings?: Record<string, Record<string, string[]>>
|
||||
|
||||
/**
|
||||
* Last updated timestamp
|
||||
*/
|
||||
|
|
@ -350,6 +362,25 @@ export interface StorageAdapter {
|
|||
*/
|
||||
flushStatisticsToStorage(): Promise<void>
|
||||
|
||||
/**
|
||||
* Track field names from a JSON document
|
||||
* @param jsonDocument The JSON document to extract field names from
|
||||
* @param service The service that inserted the data
|
||||
*/
|
||||
trackFieldNames(jsonDocument: any, service: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Get available field names by service
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
getAvailableFieldNames(): Promise<Record<string, string[]>>
|
||||
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>
|
||||
|
||||
/**
|
||||
* Get changes since a specific timestamp
|
||||
* @param timestamp The timestamp to get changes since
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
*/
|
||||
|
||||
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
|
||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||
|
||||
/**
|
||||
* Base class for storage adapters that implements statistics tracking
|
||||
|
|
@ -372,6 +373,124 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
await this.flushStatistics()
|
||||
}
|
||||
|
||||
/**
|
||||
* Track field names from a JSON document
|
||||
* @param jsonDocument The JSON document to extract field names from
|
||||
* @param service The service that inserted the data
|
||||
*/
|
||||
async trackFieldNames(jsonDocument: any, service: string): Promise<void> {
|
||||
// Skip if not a JSON object
|
||||
if (typeof jsonDocument !== 'object' || jsonDocument === null || Array.isArray(jsonDocument)) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
statistics = this.createDefaultStatistics()
|
||||
}
|
||||
|
||||
// Update the cache
|
||||
this.statisticsCache = {
|
||||
...statistics,
|
||||
nounCount: { ...statistics.nounCount },
|
||||
verbCount: { ...statistics.verbCount },
|
||||
metadataCount: { ...statistics.metadataCount },
|
||||
fieldNames: { ...statistics.fieldNames },
|
||||
standardFieldMappings: { ...statistics.standardFieldMappings }
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure fieldNames exists
|
||||
if (!this.statisticsCache!.fieldNames) {
|
||||
this.statisticsCache!.fieldNames = {}
|
||||
}
|
||||
|
||||
// Ensure standardFieldMappings exists
|
||||
if (!this.statisticsCache!.standardFieldMappings) {
|
||||
this.statisticsCache!.standardFieldMappings = {}
|
||||
}
|
||||
|
||||
// Extract field names from the JSON document
|
||||
const fieldNames = extractFieldNamesFromJson(jsonDocument)
|
||||
|
||||
// Initialize service entry if it doesn't exist
|
||||
if (!this.statisticsCache!.fieldNames[service]) {
|
||||
this.statisticsCache!.fieldNames[service] = []
|
||||
}
|
||||
|
||||
// Add new field names to the service's list
|
||||
for (const fieldName of fieldNames) {
|
||||
if (!this.statisticsCache!.fieldNames[service].includes(fieldName)) {
|
||||
this.statisticsCache!.fieldNames[service].push(fieldName)
|
||||
}
|
||||
|
||||
// Map to standard field if possible
|
||||
const standardField = mapToStandardField(fieldName)
|
||||
if (standardField) {
|
||||
// Initialize standard field entry if it doesn't exist
|
||||
if (!this.statisticsCache!.standardFieldMappings[standardField]) {
|
||||
this.statisticsCache!.standardFieldMappings[standardField] = {}
|
||||
}
|
||||
|
||||
// Initialize service entry if it doesn't exist
|
||||
if (!this.statisticsCache!.standardFieldMappings[standardField][service]) {
|
||||
this.statisticsCache!.standardFieldMappings[standardField][service] = []
|
||||
}
|
||||
|
||||
// Add field name to standard field mapping if not already there
|
||||
if (!this.statisticsCache!.standardFieldMappings[standardField][service].includes(fieldName)) {
|
||||
this.statisticsCache!.standardFieldMappings[standardField][service].push(fieldName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update timestamp
|
||||
this.statisticsCache!.lastUpdated = new Date().toISOString()
|
||||
|
||||
// Schedule a batch update
|
||||
this.statisticsModified = true
|
||||
this.scheduleBatchUpdate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available field names by service
|
||||
* @returns Record of field names by service
|
||||
*/
|
||||
async getAvailableFieldNames(): Promise<Record<string, string[]>> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
// Return field names by service
|
||||
return statistics.fieldNames || {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get standard field mappings
|
||||
* @returns Record of standard field mappings
|
||||
*/
|
||||
async getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>> {
|
||||
// Get current statistics from cache or storage
|
||||
let statistics = this.statisticsCache
|
||||
if (!statistics) {
|
||||
statistics = await this.getStatisticsData()
|
||||
if (!statistics) {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
// Return standard field mappings
|
||||
return statistics.standardFieldMappings || {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create default statistics data
|
||||
* @returns Default statistics data
|
||||
|
|
@ -382,6 +501,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
fieldNames: {},
|
||||
standardFieldMappings: {},
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
114
src/utils/fieldNameTracking.ts
Normal file
114
src/utils/fieldNameTracking.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/**
|
||||
* Utility functions for tracking and managing field names in JSON documents
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extracts field names from a JSON document
|
||||
* @param jsonObject The JSON object to extract field names from
|
||||
* @param options Configuration options
|
||||
* @returns An array of field paths (e.g., "user.name", "addresses[0].city")
|
||||
*/
|
||||
export function extractFieldNamesFromJson(
|
||||
jsonObject: any,
|
||||
options: {
|
||||
maxDepth?: number
|
||||
currentDepth?: number
|
||||
currentPath?: string
|
||||
fieldNames?: Set<string>
|
||||
} = {}
|
||||
): string[] {
|
||||
const {
|
||||
maxDepth = 5,
|
||||
currentDepth = 0,
|
||||
currentPath = '',
|
||||
fieldNames = new Set<string>()
|
||||
} = options
|
||||
|
||||
if (
|
||||
jsonObject === null ||
|
||||
jsonObject === undefined ||
|
||||
typeof jsonObject !== 'object' ||
|
||||
currentDepth >= maxDepth
|
||||
) {
|
||||
return Array.from(fieldNames)
|
||||
}
|
||||
|
||||
if (Array.isArray(jsonObject)) {
|
||||
// For arrays, we'll just check the first item to avoid explosion of paths
|
||||
if (jsonObject.length > 0) {
|
||||
const arrayPath = currentPath ? `${currentPath}[0]` : '[0]'
|
||||
extractFieldNamesFromJson(jsonObject[0], {
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
currentPath: arrayPath,
|
||||
fieldNames
|
||||
})
|
||||
}
|
||||
} else {
|
||||
// For objects, process each property
|
||||
for (const key of Object.keys(jsonObject)) {
|
||||
const value = jsonObject[key]
|
||||
const fieldPath = currentPath ? `${currentPath}.${key}` : key
|
||||
|
||||
// Add this field path
|
||||
fieldNames.add(fieldPath)
|
||||
|
||||
// Recursively process nested objects
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
extractFieldNamesFromJson(value, {
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
currentPath: fieldPath,
|
||||
fieldNames
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(fieldNames)
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps field names to standard field names based on common patterns
|
||||
* @param fieldName The field name to map
|
||||
* @returns The standard field name if a match is found, or null if no match
|
||||
*/
|
||||
export function mapToStandardField(fieldName: string): string | null {
|
||||
// Standard field mappings
|
||||
const standardMappings: Record<string, string[]> = {
|
||||
'title': ['title', 'name', 'headline', 'subject'],
|
||||
'description': ['description', 'summary', 'content', 'text', 'body'],
|
||||
'author': ['author', 'creator', 'user', 'owner', 'by'],
|
||||
'date': ['date', 'created', 'createdAt', 'timestamp', 'published'],
|
||||
'url': ['url', 'link', 'href', 'source'],
|
||||
'image': ['image', 'thumbnail', 'photo', 'picture'],
|
||||
'tags': ['tags', 'categories', 'keywords', 'topics']
|
||||
}
|
||||
|
||||
// Check for matches
|
||||
for (const [standardField, possibleMatches] of Object.entries(standardMappings)) {
|
||||
// Exact match
|
||||
if (possibleMatches.includes(fieldName)) {
|
||||
return standardField
|
||||
}
|
||||
|
||||
// Path match (e.g., "user.name" matches "name")
|
||||
const parts = fieldName.split('.')
|
||||
const lastPart = parts[parts.length - 1]
|
||||
if (possibleMatches.includes(lastPart)) {
|
||||
return standardField
|
||||
}
|
||||
|
||||
// Array match (e.g., "items[0].name" matches "name")
|
||||
if (fieldName.includes('[')) {
|
||||
for (const part of parts) {
|
||||
const cleanPart = part.split('[')[0]
|
||||
if (possibleMatches.includes(cleanPart)) {
|
||||
return standardField
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
|
@ -2,3 +2,5 @@ export * from './distance.js'
|
|||
export * from './embedding.js'
|
||||
export * from './workerUtils.js'
|
||||
export * from './statistics.js'
|
||||
export * from './jsonProcessing.js'
|
||||
export * from './fieldNameTracking.js'
|
||||
|
|
|
|||
226
src/utils/jsonProcessing.ts
Normal file
226
src/utils/jsonProcessing.ts
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* Utility functions for processing JSON documents for vectorization and search
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extracts text from a JSON object for vectorization
|
||||
* This function recursively processes the JSON object and extracts text from all fields
|
||||
* It can also prioritize specific fields if provided
|
||||
*
|
||||
* @param jsonObject The JSON object to extract text from
|
||||
* @param options Configuration options for text extraction
|
||||
* @returns A string containing the extracted text
|
||||
*/
|
||||
export function extractTextFromJson(
|
||||
jsonObject: any,
|
||||
options: {
|
||||
priorityFields?: string[] // Fields to prioritize (will be repeated for emphasis)
|
||||
excludeFields?: string[] // Fields to exclude from extraction
|
||||
includeFieldNames?: boolean // Whether to include field names in the extracted text
|
||||
maxDepth?: number // Maximum depth to recurse into nested objects
|
||||
currentDepth?: number // Current recursion depth (internal use)
|
||||
fieldPath?: string[] // Current field path (internal use)
|
||||
} = {}
|
||||
): string {
|
||||
// Set default options
|
||||
const {
|
||||
priorityFields = [],
|
||||
excludeFields = [],
|
||||
includeFieldNames = true,
|
||||
maxDepth = 5,
|
||||
currentDepth = 0,
|
||||
fieldPath = []
|
||||
} = options
|
||||
|
||||
// If input is not an object or array, or we've reached max depth, return as string
|
||||
if (
|
||||
jsonObject === null ||
|
||||
jsonObject === undefined ||
|
||||
typeof jsonObject !== 'object' ||
|
||||
currentDepth >= maxDepth
|
||||
) {
|
||||
return String(jsonObject || '')
|
||||
}
|
||||
|
||||
const extractedText: string[] = []
|
||||
const priorityText: string[] = []
|
||||
|
||||
// Process arrays
|
||||
if (Array.isArray(jsonObject)) {
|
||||
for (let i = 0; i < jsonObject.length; i++) {
|
||||
const value = jsonObject[i]
|
||||
const newPath = [...fieldPath, i.toString()]
|
||||
|
||||
// Recursively extract text from array items
|
||||
const itemText = extractTextFromJson(value, {
|
||||
priorityFields,
|
||||
excludeFields,
|
||||
includeFieldNames,
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
fieldPath: newPath
|
||||
})
|
||||
|
||||
if (itemText) {
|
||||
extractedText.push(itemText)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process objects
|
||||
else {
|
||||
for (const [key, value] of Object.entries(jsonObject)) {
|
||||
// Skip excluded fields
|
||||
if (excludeFields.includes(key)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const newPath = [...fieldPath, key]
|
||||
const fullPath = newPath.join('.')
|
||||
|
||||
// Check if this is a priority field
|
||||
const isPriority = priorityFields.some(field => {
|
||||
// Exact match
|
||||
if (field === key) return true
|
||||
// Path match
|
||||
if (field === fullPath) return true
|
||||
// Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
|
||||
if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2))) return true
|
||||
return false
|
||||
})
|
||||
|
||||
// Get the field value as text
|
||||
let fieldText: string
|
||||
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
// Recursively extract text from nested objects
|
||||
fieldText = extractTextFromJson(value, {
|
||||
priorityFields,
|
||||
excludeFields,
|
||||
includeFieldNames,
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
fieldPath: newPath
|
||||
})
|
||||
} else {
|
||||
fieldText = String(value || '')
|
||||
}
|
||||
|
||||
// Add field name if requested
|
||||
if (includeFieldNames && fieldText) {
|
||||
fieldText = `${key}: ${fieldText}`
|
||||
}
|
||||
|
||||
// Add to appropriate collection
|
||||
if (fieldText) {
|
||||
if (isPriority) {
|
||||
priorityText.push(fieldText)
|
||||
} else {
|
||||
extractedText.push(fieldText)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combine priority text (repeated for emphasis) and regular text
|
||||
return [...priorityText, ...priorityText, ...extractedText].join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares a JSON document for vectorization
|
||||
* This function extracts text from the JSON document and formats it for optimal vectorization
|
||||
*
|
||||
* @param jsonDocument The JSON document to prepare
|
||||
* @param options Configuration options for preparation
|
||||
* @returns A string ready for vectorization
|
||||
*/
|
||||
export function prepareJsonForVectorization(
|
||||
jsonDocument: any,
|
||||
options: {
|
||||
priorityFields?: string[]
|
||||
excludeFields?: string[]
|
||||
includeFieldNames?: boolean
|
||||
maxDepth?: number
|
||||
} = {}
|
||||
): string {
|
||||
// If input is a string, try to parse it as JSON
|
||||
let document = jsonDocument
|
||||
if (typeof jsonDocument === 'string') {
|
||||
try {
|
||||
document = JSON.parse(jsonDocument)
|
||||
} catch (e) {
|
||||
// If parsing fails, treat it as a plain string
|
||||
return jsonDocument
|
||||
}
|
||||
}
|
||||
|
||||
// If not an object after parsing, return as is
|
||||
if (typeof document !== 'object' || document === null) {
|
||||
return String(document || '')
|
||||
}
|
||||
|
||||
// Extract text from the document
|
||||
return extractTextFromJson(document, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts text from a specific field in a JSON document
|
||||
* This is useful for searching within specific fields
|
||||
*
|
||||
* @param jsonDocument The JSON document to extract from
|
||||
* @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
|
||||
* @returns The extracted text or empty string if field not found
|
||||
*/
|
||||
export function extractFieldFromJson(
|
||||
jsonDocument: any,
|
||||
fieldPath: string
|
||||
): string {
|
||||
// If input is a string, try to parse it as JSON
|
||||
let document = jsonDocument
|
||||
if (typeof jsonDocument === 'string') {
|
||||
try {
|
||||
document = JSON.parse(jsonDocument)
|
||||
} catch (e) {
|
||||
// If parsing fails, return empty string
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// If not an object after parsing, return empty string
|
||||
if (typeof document !== 'object' || document === null) {
|
||||
return ''
|
||||
}
|
||||
|
||||
// Parse the field path
|
||||
const parts = fieldPath.split('.')
|
||||
let current = document
|
||||
|
||||
// Navigate through the path
|
||||
for (const part of parts) {
|
||||
// Handle array indexing (e.g., "addresses[0]")
|
||||
const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/)
|
||||
if (!match) {
|
||||
return ''
|
||||
}
|
||||
|
||||
const [, key, indexStr] = match
|
||||
|
||||
// Move to the next level
|
||||
current = current[key]
|
||||
|
||||
// If we have an array index, access that element
|
||||
if (indexStr !== undefined && Array.isArray(current)) {
|
||||
const index = parseInt(indexStr, 10)
|
||||
current = current[index]
|
||||
}
|
||||
|
||||
// If we've reached a null or undefined value, return empty string
|
||||
if (current === null || current === undefined) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// Convert the final value to string
|
||||
return typeof current === 'object'
|
||||
? JSON.stringify(current)
|
||||
: String(current)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue