**feat(search): enhance JSON document search with field-level filtering and prioritization**

- Added support for field-specific and prioritized searches in `brainyData`:
  - Introduced `searchField` option to enable targeted field-level searches.
  - Implemented `priorityFields` option for weighted vectorization and query relevance.
- Developed utilities in `jsonProcessing.ts` and `fieldNameTracking.ts`:
  - `extractTextFromJson` for text extraction with customizable depth and field prioritization.
  - `extractFieldFromJson` to target specific fields in JSON documents.
  - `prepareJsonForVectorization` for optimized JSON vectorization.
- Enhanced management of field names and mappings:
  - Integrated `trackFieldNames` to associate fields with their services.
  - Supported cross-service consistency through `standardFieldMappings`.
- Updated documentation:
  - Added detailed guides for JSON search enhancements and HNSW limitations.
  - Extended usage examples in `README.md` and `json-search-test.js`.
- Verified improvements with comprehensive tests:
  - Created unit and integration tests demonstrating search behavior improvements.
  - Addressed previous TypeScript errors related to search parameters.

**Purpose**: Improve search accuracy and usability when working with complex JSON documents by enabling field-specific searches and enhancing contextual relevance.

src/utils/jsonProcessing.ts

@@ -0,0 +1,226 @@
+/**
+ * Utility functions for processing JSON documents for vectorization and search
+ */
+
+/**
+ * Extracts text from a JSON object for vectorization
+ * This function recursively processes the JSON object and extracts text from all fields
+ * It can also prioritize specific fields if provided
+ * 
+ * @param jsonObject The JSON object to extract text from
+ * @param options Configuration options for text extraction
+ * @returns A string containing the extracted text
+ */
+export function extractTextFromJson(
+  jsonObject: any,
+  options: {
+    priorityFields?: string[] // Fields to prioritize (will be repeated for emphasis)
+    excludeFields?: string[] // Fields to exclude from extraction
+    includeFieldNames?: boolean // Whether to include field names in the extracted text
+    maxDepth?: number // Maximum depth to recurse into nested objects
+    currentDepth?: number // Current recursion depth (internal use)
+    fieldPath?: string[] // Current field path (internal use)
+  } = {}
+): string {
+  // Set default options
+  const {
+    priorityFields = [],
+    excludeFields = [],
+    includeFieldNames = true,
+    maxDepth = 5,
+    currentDepth = 0,
+    fieldPath = []
+  } = options
+
+  // If input is not an object or array, or we've reached max depth, return as string
+  if (
+    jsonObject === null ||
+    jsonObject === undefined ||
+    typeof jsonObject !== 'object' ||
+    currentDepth >= maxDepth
+  ) {
+    return String(jsonObject || '')
+  }
+
+  const extractedText: string[] = []
+  const priorityText: string[] = []
+
+  // Process arrays
+  if (Array.isArray(jsonObject)) {
+    for (let i = 0; i < jsonObject.length; i++) {
+      const value = jsonObject[i]
+      const newPath = [...fieldPath, i.toString()]
+      
+      // Recursively extract text from array items
+      const itemText = extractTextFromJson(value, {
+        priorityFields,
+        excludeFields,
+        includeFieldNames,
+        maxDepth,
+        currentDepth: currentDepth + 1,
+        fieldPath: newPath
+      })
+      
+      if (itemText) {
+        extractedText.push(itemText)
+      }
+    }
+  } 
+  // Process objects
+  else {
+    for (const [key, value] of Object.entries(jsonObject)) {
+      // Skip excluded fields
+      if (excludeFields.includes(key)) {
+        continue
+      }
+
+      const newPath = [...fieldPath, key]
+      const fullPath = newPath.join('.')
+      
+      // Check if this is a priority field
+      const isPriority = priorityFields.some(field => {
+        // Exact match
+        if (field === key) return true
+        // Path match
+        if (field === fullPath) return true
+        // Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
+        if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2))) return true
+        return false
+      })
+
+      // Get the field value as text
+      let fieldText: string
+      
+      if (typeof value === 'object' && value !== null) {
+        // Recursively extract text from nested objects
+        fieldText = extractTextFromJson(value, {
+          priorityFields,
+          excludeFields,
+          includeFieldNames,
+          maxDepth,
+          currentDepth: currentDepth + 1,
+          fieldPath: newPath
+        })
+      } else {
+        fieldText = String(value || '')
+      }
+
+      // Add field name if requested
+      if (includeFieldNames && fieldText) {
+        fieldText = `${key}: ${fieldText}`
+      }
+
+      // Add to appropriate collection
+      if (fieldText) {
+        if (isPriority) {
+          priorityText.push(fieldText)
+        } else {
+          extractedText.push(fieldText)
+        }
+      }
+    }
+  }
+
+  // Combine priority text (repeated for emphasis) and regular text
+  return [...priorityText, ...priorityText, ...extractedText].join(' ')
+}
+
+/**
+ * Prepares a JSON document for vectorization
+ * This function extracts text from the JSON document and formats it for optimal vectorization
+ * 
+ * @param jsonDocument The JSON document to prepare
+ * @param options Configuration options for preparation
+ * @returns A string ready for vectorization
+ */
+export function prepareJsonForVectorization(
+  jsonDocument: any,
+  options: {
+    priorityFields?: string[]
+    excludeFields?: string[]
+    includeFieldNames?: boolean
+    maxDepth?: number
+  } = {}
+): string {
+  // If input is a string, try to parse it as JSON
+  let document = jsonDocument
+  if (typeof jsonDocument === 'string') {
+    try {
+      document = JSON.parse(jsonDocument)
+    } catch (e) {
+      // If parsing fails, treat it as a plain string
+      return jsonDocument
+    }
+  }
+
+  // If not an object after parsing, return as is
+  if (typeof document !== 'object' || document === null) {
+    return String(document || '')
+  }
+
+  // Extract text from the document
+  return extractTextFromJson(document, options)
+}
+
+/**
+ * Extracts text from a specific field in a JSON document
+ * This is useful for searching within specific fields
+ * 
+ * @param jsonDocument The JSON document to extract from
+ * @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
+ * @returns The extracted text or empty string if field not found
+ */
+export function extractFieldFromJson(
+  jsonDocument: any,
+  fieldPath: string
+): string {
+  // If input is a string, try to parse it as JSON
+  let document = jsonDocument
+  if (typeof jsonDocument === 'string') {
+    try {
+      document = JSON.parse(jsonDocument)
+    } catch (e) {
+      // If parsing fails, return empty string
+      return ''
+    }
+  }
+
+  // If not an object after parsing, return empty string
+  if (typeof document !== 'object' || document === null) {
+    return ''
+  }
+
+  // Parse the field path
+  const parts = fieldPath.split('.')
+  let current = document
+
+  // Navigate through the path
+  for (const part of parts) {
+    // Handle array indexing (e.g., "addresses[0]")
+    const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/)
+    if (!match) {
+      return ''
+    }
+
+    const [, key, indexStr] = match
+    
+    // Move to the next level
+    current = current[key]
+    
+    // If we have an array index, access that element
+    if (indexStr !== undefined && Array.isArray(current)) {
+      const index = parseInt(indexStr, 10)
+      current = current[index]
+    }
+
+    // If we've reached a null or undefined value, return empty string
+    if (current === null || current === undefined) {
+      return ''
+    }
+  }
+
+  // Convert the final value to string
+  return typeof current === 'object' 
+    ? JSON.stringify(current) 
+    : String(current)
+}