**feat(docs): add comprehensive Search and Metadata Guide with metadata handling updates**

- **Documentation Additions**:
  - Introduced `SEARCH_AND_METADATA_GUIDE.md` to provide an in-depth guide on Brainy's search and metadata retrieval system:
    - Detailed explanation of search workflows, metadata structures (`GraphNoun`, `GraphVerb`), and core components like `SearchResult`.
    - Usage examples showcasing search queries, filtering by noun/verb types, and advanced features like multi-modal search.
    - Included performance tips on caching, HNSW indexing, lazy loading, and augmentation pipeline.

- **Storage System Updates**:
  - Enhanced memory and file storage adapters to support dedicated noun and verb metadata handling:
    - Added methods `saveN
This commit is contained in:
David Snelling 2025-08-02 16:41:30 -07:00
parent 3892399bab
commit cc6c75befb
5 changed files with 472 additions and 4 deletions

View file

@ -9,6 +9,8 @@ import {
NOUNS_DIR,
VERBS_DIR,
METADATA_DIR,
NOUN_METADATA_DIR,
VERB_METADATA_DIR,
INDEX_DIR,
STATISTICS_KEY
} from '../baseStorage.js'
@ -53,6 +55,8 @@ export class FileSystemStorage extends BaseStorage {
private nounsDir!: string
private verbsDir!: string
private metadataDir!: string
private nounMetadataDir!: string
private verbMetadataDir!: string
private indexDir!: string
private lockDir!: string
private activeLocks: Set<string> = new Set()
@ -98,6 +102,8 @@ export class FileSystemStorage extends BaseStorage {
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
this.nounMetadataDir = path.join(this.rootDir, NOUN_METADATA_DIR)
this.verbMetadataDir = path.join(this.rootDir, VERB_METADATA_DIR)
this.indexDir = path.join(this.rootDir, INDEX_DIR)
this.lockDir = path.join(this.rootDir, 'locks')
@ -113,6 +119,12 @@ export class FileSystemStorage extends BaseStorage {
// Create the metadata directory if it doesn't exist
await this.ensureDirectoryExists(this.metadataDir)
// Create the noun metadata directory if it doesn't exist
await this.ensureDirectoryExists(this.nounMetadataDir)
// Create the verb metadata directory if it doesn't exist
await this.ensureDirectoryExists(this.verbMetadataDir)
// Create the index directory if it doesn't exist
await this.ensureDirectoryExists(this.indexDir)
@ -460,6 +472,62 @@ export class FileSystemStorage extends BaseStorage {
}
}
/**
* Save noun metadata to storage
*/
public async saveNounMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const filePath = path.join(this.nounMetadataDir, `${id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
}
/**
* Get noun metadata from storage
*/
public async getNounMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
const filePath = path.join(this.nounMetadataDir, `${id}.json`)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading noun metadata ${id}:`, error)
}
return null
}
}
/**
* Save verb metadata to storage
*/
public async saveVerbMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const filePath = path.join(this.verbMetadataDir, `${id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
}
/**
* Get verb metadata from storage
*/
public async getVerbMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
const filePath = path.join(this.verbMetadataDir, `${id}.json`)
try {
const data = await fs.promises.readFile(filePath, 'utf-8')
return JSON.parse(data)
} catch (error: any) {
if (error.code !== 'ENOENT') {
console.error(`Error reading verb metadata ${id}:`, error)
}
return null
}
}
/**
* Clear all data from storage
*/
@ -503,6 +571,12 @@ export class FileSystemStorage extends BaseStorage {
// Remove all files in the metadata directory
await removeDirectoryContents(this.metadataDir)
// Remove all files in the noun metadata directory
await removeDirectoryContents(this.nounMetadataDir)
// Remove all files in the verb metadata directory
await removeDirectoryContents(this.verbMetadataDir)
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir)
}

View file

@ -18,6 +18,8 @@ export class MemoryStorage extends BaseStorage {
private nouns: Map<string, HNSWNoun> = new Map()
private verbs: Map<string, GraphVerb> = new Map()
private metadata: Map<string, any> = new Map()
private nounMetadata: Map<string, any> = new Map()
private verbMetadata: Map<string, any> = new Map()
private statistics: StatisticsData | null = null
constructor() {
@ -557,6 +559,44 @@ export class MemoryStorage extends BaseStorage {
return JSON.parse(JSON.stringify(metadata))
}
/**
* Save noun metadata to storage
*/
public async saveNounMetadata(id: string, metadata: any): Promise<void> {
this.nounMetadata.set(id, JSON.parse(JSON.stringify(metadata)))
}
/**
* Get noun metadata from storage
*/
public async getNounMetadata(id: string): Promise<any | null> {
const metadata = this.nounMetadata.get(id)
if (!metadata) {
return null
}
return JSON.parse(JSON.stringify(metadata))
}
/**
* Save verb metadata to storage
*/
public async saveVerbMetadata(id: string, metadata: any): Promise<void> {
this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)))
}
/**
* Get verb metadata from storage
*/
public async getVerbMetadata(id: string): Promise<any | null> {
const metadata = this.verbMetadata.get(id)
if (!metadata) {
return null
}
return JSON.parse(JSON.stringify(metadata))
}
/**
* Clear all data from storage
*/
@ -564,6 +604,8 @@ export class MemoryStorage extends BaseStorage {
this.nouns.clear()
this.verbs.clear()
this.metadata.clear()
this.nounMetadata.clear()
this.verbMetadata.clear()
this.statistics = null
}