Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
83
scripts/maintenance/check-database.js
Normal file
83
scripts/maintenance/check-database.js
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// Script to check if there's any data in the database
|
||||
import { BrainyData } from './dist/brainyData.js';
|
||||
|
||||
async function checkDatabase() {
|
||||
try {
|
||||
console.log('Initializing BrainyData...');
|
||||
const db = new BrainyData();
|
||||
await db.init();
|
||||
|
||||
console.log('Getting database status...');
|
||||
const status = await db.status();
|
||||
console.log('Database status:', JSON.stringify(status, null, 2));
|
||||
|
||||
console.log('Getting statistics...');
|
||||
const stats = await db.getStatistics();
|
||||
console.log('Statistics:', JSON.stringify(stats, null, 2));
|
||||
|
||||
console.log('Getting all nouns...');
|
||||
const nouns = await db.getAllNouns();
|
||||
console.log(`Found ${nouns.length} nouns in the database.`);
|
||||
|
||||
if (nouns.length > 0) {
|
||||
console.log('Sample of nouns:');
|
||||
for (let i = 0; i < Math.min(5, nouns.length); i++) {
|
||||
console.log(`Noun ${i + 1}:`, JSON.stringify(nouns[i], null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Getting all verbs...');
|
||||
const verbs = await db.getAllVerbs();
|
||||
console.log(`Found ${verbs.length} verbs in the database.`);
|
||||
|
||||
if (verbs.length > 0) {
|
||||
console.log('Sample of verbs:');
|
||||
for (let i = 0; i < Math.min(5, verbs.length); i++) {
|
||||
console.log(`Verb ${i + 1}:`, JSON.stringify(verbs[i], null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// Try a simple search to see if it returns any results
|
||||
console.log('Trying a simple search...');
|
||||
const searchResults = await db.searchText('test', 10);
|
||||
console.log(`Search returned ${searchResults.length} results.`);
|
||||
|
||||
if (searchResults.length > 0) {
|
||||
console.log('Sample of search results:');
|
||||
for (let i = 0; i < Math.min(5, searchResults.length); i++) {
|
||||
console.log(`Result ${i + 1}:`, JSON.stringify({
|
||||
id: searchResults[i].id,
|
||||
score: searchResults[i].score,
|
||||
metadata: searchResults[i].metadata
|
||||
}, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// If no results, try adding a test item and searching again
|
||||
if (searchResults.length === 0 && nouns.length === 0) {
|
||||
console.log('No data found. Adding a test item...');
|
||||
const id = await db.add('This is a test item for searching', { noun: 'Thing', category: 'test' });
|
||||
console.log(`Added test item with ID: ${id}`);
|
||||
|
||||
console.log('Trying search again...');
|
||||
const newSearchResults = await db.searchText('test', 10);
|
||||
console.log(`Search returned ${newSearchResults.length} results.`);
|
||||
|
||||
if (newSearchResults.length > 0) {
|
||||
console.log('Sample of search results:');
|
||||
for (let i = 0; i < Math.min(5, newSearchResults.length); i++) {
|
||||
console.log(`Result ${i + 1}:`, JSON.stringify({
|
||||
id: newSearchResults[i].id,
|
||||
score: newSearchResults[i].score,
|
||||
metadata: newSearchResults[i].metadata
|
||||
}, null, 2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error checking database:', error);
|
||||
}
|
||||
}
|
||||
|
||||
checkDatabase().catch(console.error);
|
||||
163
scripts/maintenance/fix-dimension-mismatch.js
Normal file
163
scripts/maintenance/fix-dimension-mismatch.js
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
// Script to fix dimension mismatch by re-embedding existing data
|
||||
import { BrainyData } from './dist/brainyData.js';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
async function fixDimensionMismatch() {
|
||||
try {
|
||||
console.log('Starting dimension mismatch fix...');
|
||||
|
||||
// Create a backup of the existing data
|
||||
const backupDir = './brainy-data-backup-' + Date.now();
|
||||
console.log(`Creating backup of existing data in ${backupDir}...`);
|
||||
|
||||
// Copy the entire brainy-data directory to the backup directory
|
||||
await fs.promises.mkdir(backupDir, { recursive: true });
|
||||
await copyDirectory('./brainy-data', backupDir);
|
||||
console.log('Backup created successfully.');
|
||||
|
||||
// Initialize BrainyData with the current embedding function
|
||||
console.log('Initializing BrainyData...');
|
||||
const db = new BrainyData();
|
||||
await db.init();
|
||||
|
||||
// Get database status to check if there's any data
|
||||
const status = await db.status();
|
||||
console.log('Database status:', JSON.stringify(status, null, 2));
|
||||
|
||||
// Read all noun files directly from the filesystem
|
||||
console.log('Reading noun files directly from filesystem...');
|
||||
const nounsDir = './brainy-data/nouns';
|
||||
const files = await fs.promises.readdir(nounsDir);
|
||||
|
||||
// Process each noun file
|
||||
const processedNouns = [];
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(nounsDir, file);
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8');
|
||||
const parsedNoun = JSON.parse(data);
|
||||
|
||||
// Get the metadata for this noun
|
||||
const metadataPath = path.join('./brainy-data/metadata', `${parsedNoun.id}.json`);
|
||||
let metadata = {};
|
||||
try {
|
||||
const metadataData = await fs.promises.readFile(metadataPath, 'utf-8');
|
||||
metadata = JSON.parse(metadataData);
|
||||
} catch (error) {
|
||||
console.warn(`No metadata found for noun ${parsedNoun.id}`);
|
||||
}
|
||||
|
||||
// Extract text from metadata if available
|
||||
let text = '';
|
||||
if (metadata.text) {
|
||||
text = metadata.text;
|
||||
} else if (metadata.description) {
|
||||
text = metadata.description;
|
||||
} else {
|
||||
// If no text is available, use a placeholder
|
||||
text = `Noun ${parsedNoun.id}`;
|
||||
console.warn(`No text found for noun ${parsedNoun.id}, using placeholder`);
|
||||
}
|
||||
|
||||
// Re-embed the text using the current embedding function
|
||||
console.log(`Re-embedding noun ${parsedNoun.id}...`);
|
||||
try {
|
||||
// Delete the existing noun first
|
||||
await db.delete(parsedNoun.id);
|
||||
|
||||
// Add the noun with the same ID but new vector
|
||||
const newId = await db.add(text, metadata, { id: parsedNoun.id });
|
||||
processedNouns.push({ id: newId, originalId: parsedNoun.id });
|
||||
console.log(`Successfully re-embedded noun ${parsedNoun.id}`);
|
||||
} catch (error) {
|
||||
console.error(`Error re-embedding noun ${parsedNoun.id}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Processed ${processedNouns.length} nouns.`);
|
||||
|
||||
// Recreate verbs
|
||||
console.log('Reading verb files directly from filesystem...');
|
||||
const verbsDir = './brainy-data/verbs';
|
||||
const verbFiles = await fs.promises.readdir(verbsDir);
|
||||
|
||||
// Process each verb file
|
||||
const processedVerbs = [];
|
||||
for (const file of verbFiles) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(verbsDir, file);
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8');
|
||||
const parsedVerb = JSON.parse(data);
|
||||
|
||||
// Check if both source and target nouns exist
|
||||
const sourceExists = processedNouns.some(n => n.originalId === parsedVerb.sourceId);
|
||||
const targetExists = processedNouns.some(n => n.originalId === parsedVerb.targetId);
|
||||
|
||||
if (sourceExists && targetExists) {
|
||||
console.log(`Re-creating verb ${parsedVerb.id} between ${parsedVerb.sourceId} and ${parsedVerb.targetId}...`);
|
||||
try {
|
||||
// Delete the existing verb first
|
||||
await db.deleteVerb(parsedVerb.id);
|
||||
|
||||
// Add the verb with the same relationship
|
||||
await db.addVerb(parsedVerb.sourceId, parsedVerb.targetId, {
|
||||
verb: parsedVerb.type || 'RelatedTo',
|
||||
...parsedVerb.metadata
|
||||
});
|
||||
|
||||
processedVerbs.push(parsedVerb.id);
|
||||
console.log(`Successfully re-created verb ${parsedVerb.id}`);
|
||||
} catch (error) {
|
||||
console.error(`Error re-creating verb ${parsedVerb.id}:`, error);
|
||||
}
|
||||
} else {
|
||||
console.warn(`Skipping verb ${parsedVerb.id} because source or target noun doesn't exist`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Processed ${processedVerbs.length} verbs.`);
|
||||
|
||||
// Try a search to verify it works
|
||||
console.log('Trying a search to verify it works...');
|
||||
const searchResults = await db.searchText('test', 10);
|
||||
console.log(`Search returned ${searchResults.length} results.`);
|
||||
|
||||
if (searchResults.length > 0) {
|
||||
console.log('Sample of search results:');
|
||||
for (let i = 0; i < Math.min(5, searchResults.length); i++) {
|
||||
console.log(`Result ${i + 1}:`, JSON.stringify({
|
||||
id: searchResults[i].id,
|
||||
score: searchResults[i].score,
|
||||
metadata: searchResults[i].metadata
|
||||
}, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Dimension mismatch fix completed successfully.');
|
||||
} catch (error) {
|
||||
console.error('Error fixing dimension mismatch:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to copy a directory recursively
|
||||
async function copyDirectory(source, destination) {
|
||||
const entries = await fs.promises.readdir(source, { withFileTypes: true });
|
||||
|
||||
await fs.promises.mkdir(destination, { recursive: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const srcPath = path.join(source, entry.name);
|
||||
const destPath = path.join(destination, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await copyDirectory(srcPath, destPath);
|
||||
} else {
|
||||
await fs.promises.copyFile(srcPath, destPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fixDimensionMismatch().catch(console.error);
|
||||
Loading…
Add table
Add a link
Reference in a new issue