Initial commit: Brainy - Multi-Dimensional AI Database

Open source vector database with HNSW indexing, graph relationships,
and metadata facets. Features CLI with professional augmentation registry
integration for discovering extensions and capabilities.
This commit is contained in:
David Snelling 2025-08-18 17:35:06 -07:00
commit f8c45f2d8d
448 changed files with 103294 additions and 0 deletions

View file

@ -0,0 +1,187 @@
#!/usr/bin/env node
/**
* Metadata Performance Analysis Script
* Quick performance analysis of metadata filtering system without full test suite
*/
import { BrainyData } from '../dist/brainyData.js'
const measureTime = async (fn) => {
const start = performance.now()
const result = await fn()
const end = performance.now()
return { result, time: end - start }
}
const generateTestData = (count) => {
const departments = ['Engineering', 'Marketing', 'Sales', 'HR']
const levels = ['junior', 'senior', 'staff', 'principal']
const locations = ['SF', 'NYC', 'LA', 'Seattle']
return Array.from({ length: count }, (_, i) => ({
text: `Profile ${i}: Professional with experience in software development`,
metadata: {
id: `profile-${i}`,
department: departments[i % departments.length],
level: levels[i % levels.length],
location: locations[i % locations.length],
salary: 50000 + (i % 10) * 10000,
remote: i % 3 === 0,
active: i % 5 !== 0
}
}))
}
async function analyzePerformance() {
console.log('=== Metadata Performance Analysis ===\n')
// Test 1: Initialization with vs without metadata indexing
console.log('1. INITIALIZATION COMPARISON')
const testData = generateTestData(100)
// Without indexing
const withoutIndex = await measureTime(async () => {
const brainy = new BrainyData({
storage: { forceMemoryStorage: true },
logging: { verbose: false }
})
await brainy.init()
for (const item of testData) {
await brainy.add(item.text, item.metadata)
}
return brainy
})
console.log(`WITHOUT indexing: ${withoutIndex.time.toFixed(2)}ms for 100 items`)
// With indexing
const withIndex = await measureTime(async () => {
const brainy = new BrainyData({
storage: { forceMemoryStorage: true },
logging: { verbose: false },
metadataIndex: { autoOptimize: true }
})
await brainy.init()
for (const item of testData) {
await brainy.add(item.text, item.metadata)
}
return brainy
})
console.log(`WITH indexing: ${withIndex.time.toFixed(2)}ms for 100 items`)
const overhead = ((withIndex.time - withoutIndex.time) / withoutIndex.time) * 100
console.log(`Index overhead: ${overhead.toFixed(1)}%\n`)
// Test 2: Search Performance Comparison
console.log('2. SEARCH PERFORMANCE COMPARISON')
const brainy = withIndex.result
const searchQuery = 'Professional software development'
const numSearches = 5
// No filtering
let totalNoFilter = 0
for (let i = 0; i < numSearches; i++) {
const { time } = await measureTime(async () => {
return await brainy.search(searchQuery, 10)
})
totalNoFilter += time
}
const avgNoFilter = totalNoFilter / numSearches
console.log(`No filtering: ${avgNoFilter.toFixed(2)}ms average`)
// Simple filtering
let totalSimpleFilter = 0
for (let i = 0; i < numSearches; i++) {
const { time } = await measureTime(async () => {
return await brainy.search(searchQuery, 10, {
metadata: { department: 'Engineering' }
})
})
totalSimpleFilter += time
}
const avgSimpleFilter = totalSimpleFilter / numSearches
console.log(`Simple filter: ${avgSimpleFilter.toFixed(2)}ms average`)
// Complex filtering
let totalComplexFilter = 0
for (let i = 0; i < numSearches; i++) {
const { time } = await measureTime(async () => {
return await brainy.search(searchQuery, 10, {
metadata: {
department: { $in: ['Engineering', 'Marketing'] },
level: { $in: ['senior', 'staff'] },
salary: { $gte: 80000 }
}
})
})
totalComplexFilter += time
}
const avgComplexFilter = totalComplexFilter / numSearches
console.log(`Complex filter: ${avgComplexFilter.toFixed(2)}ms average`)
console.log('\nSearch Performance Impact:')
console.log(`Simple filter overhead: ${((avgSimpleFilter / avgNoFilter - 1) * 100).toFixed(1)}%`)
console.log(`Complex filter overhead: ${((avgComplexFilter / avgNoFilter - 1) * 100).toFixed(1)}%\n`)
// Test 3: Index Statistics
console.log('3. INDEX STATISTICS')
if (brainy.metadataIndex) {
const stats = await brainy.metadataIndex.getStats()
console.log(`Total index entries: ${stats.totalEntries}`)
console.log(`Total indexed IDs: ${stats.totalIds}`)
console.log(`Fields indexed: ${stats.fieldsIndexed.join(', ')}`)
console.log(`Estimated index size: ${stats.indexSize} bytes`)
console.log(`Storage overhead per item: ${(stats.indexSize / 100).toFixed(2)} bytes\n`)
}
// Test 4: Write Performance
console.log('4. WRITE PERFORMANCE ANALYSIS')
const newTestData = generateTestData(50)
// Add performance
const { time: addTime } = await measureTime(async () => {
for (const item of newTestData) {
await brainy.add(item.text, item.metadata)
}
})
console.log(`ADD: 50 items in ${addTime.toFixed(2)}ms (${(addTime / 50).toFixed(2)}ms per item)`)
// Update performance
const updateData = newTestData.slice(0, 20).map(item => ({
...item,
metadata: { ...item.metadata, level: 'updated', salary: item.metadata.salary + 10000 }
}))
const { time: updateTime } = await measureTime(async () => {
for (const item of updateData) {
await brainy.updateMetadata(item.metadata.id, item.metadata)
}
})
console.log(`UPDATE: 20 items in ${updateTime.toFixed(2)}ms (${(updateTime / 20).toFixed(2)}ms per item)`)
// Delete performance
const idsToDelete = newTestData.slice(30, 40).map(item => item.metadata.id)
const { time: deleteTime } = await measureTime(async () => {
for (const id of idsToDelete) {
await brainy.delete(id)
}
})
console.log(`DELETE: 10 items in ${deleteTime.toFixed(2)}ms (${(deleteTime / 10).toFixed(2)}ms per item)\n`)
// Cleanup
await withoutIndex.result.shutDown()
await withIndex.result.shutDown()
console.log('Analysis complete!')
}
analyzePerformance().catch(console.error)

93
scripts/check-code-style.js Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env node
/**
* Script to check code style and enforce no-semicolon rule
* This script runs eslint and prettier checks on the codebase
*/
const { execSync } = require('child_process')
const path = require('path')
const fs = require('fs')
// ANSI color codes for terminal output
const colors = {
reset: '\x1b[0m',
red: '\x1b[31m',
green: '\x1b[32m',
yellow: '\x1b[33m',
blue: '\x1b[34m',
magenta: '\x1b[35m',
cyan: '\x1b[36m',
white: '\x1b[37m'
}
console.log(`${colors.cyan}Checking code style...${colors.reset}`)
console.log(`${colors.cyan}====================${colors.reset}`)
// Run eslint
try {
console.log(`${colors.blue}Running ESLint...${colors.reset}`)
execSync('npm run lint', { stdio: 'inherit' })
console.log(`${colors.green}ESLint check passed!${colors.reset}`)
} catch (error) {
console.error(`${colors.red}ESLint check failed!${colors.reset}`)
console.log(`${colors.yellow}Run 'npm run lint:fix' to automatically fix some issues.${colors.reset}`)
process.exit(1)
}
// Run prettier check
try {
console.log(`${colors.blue}Running Prettier check...${colors.reset}`)
execSync('npm run check-format', { stdio: 'inherit' })
console.log(`${colors.green}Prettier check passed!${colors.reset}`)
} catch (error) {
console.error(`${colors.red}Prettier check failed!${colors.reset}`)
console.log(`${colors.yellow}Run 'npm run format' to automatically format your code.${colors.reset}`)
process.exit(1)
}
// Specific check for semicolons
console.log(`${colors.blue}Checking for semicolons in code...${colors.reset}`)
try {
// Find all .ts and .js files in src directory
const findCommand = "find src -type f -name '*.ts' -o -name '*.js'"
const files = execSync(findCommand, { encoding: 'utf8' }).trim().split('\n')
let semicolonFound = false
for (const file of files) {
if (!file) continue
const content = fs.readFileSync(file, 'utf8')
const lines = content.split('\n')
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
// Skip comments and strings
if (line.trim().startsWith('//') || line.trim().startsWith('/*') ||
line.trim().startsWith('*') || line.trim().startsWith('*/')) {
continue
}
// Check for semicolons at the end of lines (excluding in string literals and comments)
if (line.trim().endsWith(';') && !line.includes('//') && !line.includes('/*')) {
console.error(`${colors.red}Semicolon found in ${file}:${i+1}${colors.reset}`)
console.error(`${colors.yellow}${line}${colors.reset}`)
semicolonFound = true
}
}
}
if (semicolonFound) {
console.error(`${colors.red}Semicolons found in code! Please remove them.${colors.reset}`)
process.exit(1)
} else {
console.log(`${colors.green}No semicolons found in code!${colors.reset}`)
}
} catch (error) {
console.error(`${colors.red}Error checking for semicolons: ${error}${colors.reset}`)
process.exit(1)
}
console.log(`${colors.green}All code style checks passed!${colors.reset}`)
console.log(`${colors.cyan}Remember: No semicolons in code!${colors.reset}`)

15
scripts/claude-commit.sh Executable file
View file

@ -0,0 +1,15 @@
#!/bin/bash
# This script now calls the global claude-commit command
# The global version is located at ~/.local/bin/claude-commit
# Or can be installed from docs/tools/claude-commit/setup.sh
if command -v claude-commit >/dev/null 2>&1; then
exec claude-commit "$@"
elif [ -x "$HOME/.local/bin/claude-commit" ]; then
exec "$HOME/.local/bin/claude-commit" "$@"
else
echo "claude-commit not found. Please run:"
echo " ./docs/tools/claude-commit/setup.sh"
exit 1
fi

20
scripts/create-favicon.js Normal file
View file

@ -0,0 +1,20 @@
// Create a simple favicon.ico file
import { writeFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
// Get the directory name of the current module
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// This is a base64-encoded 16x16 transparent favicon
const faviconBase64 = 'AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABILAAASCwAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAA==';
// Path to save the favicon
const faviconPath = join(__dirname, '..', 'favicon.ico');
// Convert base64 to binary and save
const faviconBuffer = Buffer.from(faviconBase64, 'base64');
writeFileSync(faviconPath, faviconBuffer);
console.log(`Favicon created at ${faviconPath}`);

View file

@ -0,0 +1,98 @@
#!/usr/bin/env node
/**
* Create GitHub Release Script
*
* This script creates a GitHub release with auto-generated release notes
* for the current version of the package.
*
* It uses the GitHub CLI (gh) to create the release, so the gh CLI must be installed
* and authenticated with appropriate permissions.
*
* The script:
* 1. Gets the current version from package.json
* 2. Creates a GitHub release for that version
* 3. Auto-generates release notes based on commits since the last release
*
* This ensures that each npm release has a corresponding GitHub release with notes.
*/
import { execSync } from 'child_process'
import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
// Path to the root directory
const rootDir = path.join(__dirname, '..')
// Path to package.json
const packageJsonPath = path.join(rootDir, 'package.json')
// Read package.json
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
const version = packageJson.version
// Check if GitHub CLI is installed
try {
execSync('gh --version', { stdio: 'ignore' })
} catch (error) {
console.error('Error: GitHub CLI (gh) is not installed or not in PATH')
console.error('Please install it from https://cli.github.com/ and authenticate with `gh auth login`')
process.exit(1)
}
// Check if the tag exists locally
let tagExistsLocally = false
try {
execSync(`git tag -l v${version}`, { stdio: 'pipe', cwd: rootDir }).toString().trim() === `v${version}` ? tagExistsLocally = true : tagExistsLocally = false
} catch (error) {
console.log(`Error checking if tag exists: ${error.message}`)
tagExistsLocally = false
}
// Push the tag to remote if it exists locally
if (tagExistsLocally) {
try {
console.log(`Pushing tag v${version} to remote...`)
execSync(`git push origin v${version}`, { stdio: 'inherit', cwd: rootDir })
console.log(`Successfully pushed tag v${version} to remote`)
} catch (error) {
console.error(`Error pushing tag to remote: ${error.message}`)
// Continue with release creation even if tag push fails
}
} else {
console.log(`Tag v${version} does not exist locally, skipping tag push`)
}
// Create the GitHub release
try {
console.log(`Creating GitHub release for v${version}...`)
// Create a release with auto-generated notes
// The --generate-notes flag automatically generates release notes based on PRs and commits
execSync(
`gh release create v${version} --title "v${version}" --generate-notes`,
{ stdio: 'inherit', cwd: rootDir }
)
console.log(`GitHub release v${version} created successfully!`)
// GitHub will automatically handle the changelog
console.log('GitHub release created with auto-generated notes')
} catch (error) {
// If the release already exists, this is not a fatal error
if (error.message.includes('already exists')) {
console.log(`GitHub release v${version} already exists, skipping creation.`)
// GitHub will automatically handle the changelog
console.log('GitHub release already exists with auto-generated notes')
} else {
console.error('Error creating GitHub release:', error.message)
// Don't exit with error to allow the npm publish to continue
// process.exit(1)
}
}

View file

@ -0,0 +1,85 @@
#!/usr/bin/env node
/**
* CLI Wrapper Script
*
* This script serves as a wrapper for the Brainy CLI, ensuring that command-line arguments
* are properly passed to the CLI when invoked through npm scripts.
*/
import { spawn, execSync } from 'child_process'
import { fileURLToPath } from 'url'
import { dirname, join } from 'path'
import fs from 'fs'
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
// Path to the actual CLI script
const cliPath = join(__dirname, 'dist', 'cli.js')
// Check if the CLI script exists
if (!fs.existsSync(cliPath)) {
// Check if we're running in a global installation context
const isGlobalInstall = __dirname.includes('node_modules') && !__dirname.includes('node_modules/.')
if (isGlobalInstall) {
console.error(`Error: CLI script not found at ${cliPath}`)
console.error('This is likely because the CLI was not built during package installation.')
console.error('Please reinstall the package with:')
console.error('npm uninstall -g @soulcraft/brainy')
console.error('npm install -g @soulcraft/brainy --legacy-peer-deps')
process.exit(1)
} else {
// In a local development context, try to build the CLI
console.log(`CLI script not found at ${cliPath}. Building CLI...`)
try {
// Run the build:cli script
execSync('npm run build:cli', { stdio: 'inherit' })
// Check again if the CLI script exists after building
if (!fs.existsSync(cliPath)) {
console.error(`Error: Failed to build CLI script at ${cliPath}`)
process.exit(1)
}
console.log('CLI built successfully.')
} catch (error) {
console.error(`Error building CLI: ${error.message}`)
console.error('Make sure you have the necessary dependencies installed.')
process.exit(1)
}
}
}
// Special handling for version flags
if (process.argv.includes('--version') || process.argv.includes('-V')) {
// Read version directly from package.json to ensure it's always correct
try {
const packageJsonPath = join(__dirname, 'package.json')
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
console.log(packageJson.version)
process.exit(0)
} catch (error) {
console.error('Error loading version information:', error.message)
process.exit(1)
}
}
// Forward all arguments to the CLI script
const args = process.argv.slice(2)
// Check if npm is passing --force flag
// When npm runs with --force, it sets the npm_config_force environment variable
if (process.env.npm_config_force === 'true' && args.includes('clear') && !args.includes('--force') && !args.includes('-f')) {
args.push('--force')
}
const cli = spawn('node', [cliPath, ...args], { stdio: 'inherit' })
cli.on('close', (code) => {
process.exit(code)
})

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,17 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Brainy Interactive Demo - Redirecting...</title>
<link rel="icon" href="brainy.png" type="image/png">
<meta http-equiv="refresh" content="0;url=demo/index.html">
<script>
window.location.href = 'demo/index.html'
</script>
</head>
<body>
<p>Redirecting to <a href="demo/index.html">Brainy Interactive Demo</a>...</p>
<p>If you are not redirected automatically, please click the link above.</p>
</body>
</html>

View file

@ -0,0 +1,78 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Brainy Cache Detection Browser Test</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
}
#results {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
border-radius: 5px;
background-color: #f9f9f9;
}
.success {
color: green;
font-weight: bold;
}
.error {
color: red;
font-weight: bold;
}
</style>
</head>
<body>
<h1>Brainy Cache Detection Browser Test</h1>
<p>This page tests if Brainy's cache detection works properly in browser environments.</p>
<button id="runTest">Run Test</button>
<div id="results">
<p>Test results will appear here...</p>
</div>
<script type="module">
import { BrainyData } from './dist/unified.js';
document.getElementById('runTest').addEventListener('click', async () => {
const resultsDiv = document.getElementById('results');
resultsDiv.innerHTML = '<p>Running test...</p>';
try {
console.log('Creating BrainyData instance...');
resultsDiv.innerHTML += '<p>Creating BrainyData instance...</p>';
const brainy = new BrainyData();
console.log('BrainyData instance created successfully!');
resultsDiv.innerHTML += '<p class="success">BrainyData instance created successfully!</p>';
// Initialize to make sure all components are created
await brainy.init();
console.log('BrainyData initialized successfully!');
resultsDiv.innerHTML += '<p class="success">BrainyData initialized successfully!</p>';
// Add a simple item to verify everything works
const id = await brainy.add("Test item for browser cache detection");
console.log('Added test item with ID:', id);
resultsDiv.innerHTML += `<p class="success">Added test item with ID: ${id}</p>`;
resultsDiv.innerHTML += '<p class="success">✅ Test completed successfully! Cache detection works in browser environment.</p>';
} catch (error) {
console.error('Error during cache detection test:', error);
resultsDiv.innerHTML += `<p class="error">❌ Error during test: ${error.message}</p>`;
resultsDiv.innerHTML += `<p class="error">Stack trace: ${error.stack}</p>`;
}
});
</script>
</body>
</html>

View file

@ -0,0 +1,130 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Brainy Cache Detection Worker Test</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
}
#results {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
border-radius: 5px;
background-color: #f9f9f9;
}
.success {
color: green;
font-weight: bold;
}
.error {
color: red;
font-weight: bold;
}
.log {
color: #333;
margin: 5px 0;
}
</style>
</head>
<body>
<h1>Brainy Cache Detection Worker Test</h1>
<p>This page tests if Brainy's cache detection works properly in Web Worker environments.</p>
<button id="runTest">Run Test</button>
<div id="results">
<p>Test results will appear here...</p>
</div>
<script type="module">
document.getElementById('runTest').addEventListener('click', () => {
const resultsDiv = document.getElementById('results');
resultsDiv.innerHTML = '<p>Starting worker test...</p>';
try {
// Create a blob URL for the worker script
const workerScript = `
// Worker script for testing cache detection
import { BrainyData } from './dist/unified.js';
// Listen for messages from the main thread
self.onmessage = async (event) => {
if (event.data === 'start-test') {
try {
self.postMessage({ type: 'log', message: 'Creating BrainyData instance...' });
const brainy = new BrainyData();
self.postMessage({ type: 'log', message: 'BrainyData instance created successfully!' });
// Initialize to make sure all components are created
await brainy.init();
self.postMessage({ type: 'log', message: 'BrainyData initialized successfully!' });
// Add a simple item to verify everything works
const id = await brainy.add("Test item for worker cache detection");
self.postMessage({ type: 'log', message: \`Added test item with ID: \${id}\` });
self.postMessage({
type: 'success',
message: 'Test completed successfully! Cache detection works in worker environment.'
});
} catch (error) {
self.postMessage({
type: 'error',
message: \`Error during test: \${error.message}\`,
stack: error.stack
});
}
}
};
// Notify that the worker is ready
self.postMessage({ type: 'ready' });
`;
const blob = new Blob([workerScript], { type: 'application/javascript' });
const workerUrl = URL.createObjectURL(blob);
// Create the worker
const worker = new Worker(workerUrl, { type: 'module' });
// Handle messages from the worker
worker.onmessage = (event) => {
const data = event.data;
if (data.type === 'ready') {
resultsDiv.innerHTML += '<p class="log">Worker is ready. Starting test...</p>';
worker.postMessage('start-test');
} else if (data.type === 'log') {
resultsDiv.innerHTML += `<p class="log">[Worker] ${data.message}</p>`;
} else if (data.type === 'success') {
resultsDiv.innerHTML += `<p class="success">✅ ${data.message}</p>`;
} else if (data.type === 'error') {
resultsDiv.innerHTML += `<p class="error">❌ ${data.message}</p>`;
if (data.stack) {
resultsDiv.innerHTML += `<p class="error">Stack trace: ${data.stack}</p>`;
}
}
};
// Handle worker errors
worker.onerror = (error) => {
resultsDiv.innerHTML += `<p class="error">❌ Worker error: ${error.message}</p>`;
};
} catch (error) {
resultsDiv.innerHTML += `<p class="error">❌ Error creating worker: ${error.message}</p>`;
console.error('Error creating worker:', error);
}
});
</script>
</body>
</html>

256
scripts/download-model.js Normal file
View file

@ -0,0 +1,256 @@
/* eslint-env node */
/* eslint-disable no-console, no-undef */
// Script to download the Universal Sentence Encoder model locally
// This ensures the model is available in all environments without network dependencies
import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
import * as tf from '@tensorflow/tfjs'
import '@tensorflow/tfjs-backend-cpu'
import * as use from '@tensorflow-models/universal-sentence-encoder'
import { execSync } from 'child_process'
// Get the directory name in ESM
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
// Define model directories
const MODEL_DIR = path.join(__dirname, '..', 'models')
const USE_MODEL_DIR = path.join(MODEL_DIR, 'sentence-encoder')
// Create directories if they don't exist
if (!fs.existsSync(MODEL_DIR)) {
fs.mkdirSync(MODEL_DIR)
// eslint-disable-next-line no-console
console.log(`Created directory: ${MODEL_DIR}`)
}
if (!fs.existsSync(USE_MODEL_DIR)) {
fs.mkdirSync(USE_MODEL_DIR)
// eslint-disable-next-line no-console
console.log(`Created directory: ${USE_MODEL_DIR}`)
}
// eslint-disable-next-line no-console
console.log('Starting Universal Sentence Encoder model setup...')
// eslint-disable-next-line no-console
console.log(
'This script will create reference files that point to the TensorFlow Hub model.'
)
// eslint-disable-next-line no-console
console.log(
'NOTE: This does NOT download the full model locally. The full model (~25MB) will be downloaded'
)
// eslint-disable-next-line no-console
console.log(
'automatically when your application first uses it, and then cached for future use.'
)
async function downloadModel() {
try {
// Define modelMetadata at the top level so it's accessible throughout the function
let modelMetadata = {
name: 'universal-sentence-encoder',
version: '1.0.0',
description: 'Universal Sentence Encoder model for text embeddings',
dimensions: 512,
date: new Date().toISOString(),
source: 'tensorflow-models/universal-sentence-encoder',
savedLocally: true
}
// Load the model - this will download it from TF Hub
console.log('Loading Universal Sentence Encoder model...')
const model = await use.load()
console.log('Model loaded successfully!')
// Create a test sentence to ensure the model works
console.log('Testing model with a sample sentence...')
const singleEmbedding = await model.embed(['Hello world'])
const singleEmbeddingArray = await singleEmbedding.array()
console.log(`Test embedding dimensions: ${singleEmbeddingArray[0].length}`)
singleEmbedding.dispose()
// Test the model with a few sentences to verify it works
console.log('Testing model with sample sentences...')
const testSentences = [
'Hello world',
'How are you doing today?',
'Machine learning is fascinating'
]
// Get embeddings for test sentences
const batchEmbeddings = await model.embed(testSentences)
const batchEmbeddingArrays = await batchEmbeddings.array()
// Log dimensions of each embedding
for (let i = 0; i < testSentences.length; i++) {
console.log(
`Embedding ${i + 1} dimensions: ${batchEmbeddingArrays[i].length}`
)
}
// Clean up tensors
batchEmbeddings.dispose()
// Since we can't directly save the model in this environment,
// we'll download it from the TensorFlow Hub URL and save it manually
console.log('Downloading model files from TensorFlow Hub...')
// Create a model.json file that includes information about the model
// and points to the TensorFlow Hub URL
const modelJson = {
format: 'graph-model',
generatedBy: 'TensorFlow.js v4.22.0',
convertedBy: 'Brainy download-model script',
modelTopology: {
class_name: 'GraphModel',
config: {
name: 'universal-sentence-encoder'
}
},
userDefinedMetadata: {
signature: {
inputs: {
inputs: {
name: 'inputs',
dtype: 'string',
shape: [-1]
}
},
outputs: {
outputs: {
name: 'outputs',
dtype: 'float32',
shape: [-1, 512]
}
}
}
},
weightsManifest: [
{
paths: ['group1-shard1of1.bin'],
weights: [
{
name: 'embedding_matrix',
shape: [512, 512],
dtype: 'float32'
}
]
}
],
modelUrl:
'https://tfhub.dev/tensorflow/tfjs-model/universal-sentence-encoder/1/default/1'
}
// Write the model.json file
fs.writeFileSync(
path.join(USE_MODEL_DIR, 'model.json'),
JSON.stringify(modelJson, null, 2)
)
// Generate a sample embedding and save it as the weights file
// This will be a real embedding, not just zeros
console.log('Generating sample embedding for weights file...')
const sampleEmbedding = await model.embed([
'This is a sample sentence for the Universal Sentence Encoder model.'
])
const sampleEmbeddingArray = await sampleEmbedding.array()
// Create a Float32Array from the embedding
const embeddingData = new Float32Array(sampleEmbeddingArray[0])
// Write the embedding data to the weights file
fs.writeFileSync(
path.join(USE_MODEL_DIR, 'group1-shard1of1.bin'),
Buffer.from(embeddingData.buffer)
)
console.log('Sample embedding saved as weights file')
sampleEmbedding.dispose()
// Update metadata
modelMetadata.savedWith = 'manual-embedding'
modelMetadata.embeddingSize = embeddingData.length
console.log(`Model files created in ${USE_MODEL_DIR}`)
console.log(
`The model.json file points to the TensorFlow Hub URL for the actual model`
)
console.log(
`The weights file contains a real sample embedding of size ${embeddingData.length}`
)
// Add instructions for users
console.log(
'\nIMPORTANT: This setup uses the TensorFlow Hub URL for the model.'
)
console.log(
'The first time the model is used, it will download the full model from TensorFlow Hub.'
)
console.log('Subsequent uses will use the cached model.')
// Update metadata to indicate the approach used
modelMetadata.approach = 'tfhub-reference'
// Write metadata file
fs.writeFileSync(
path.join(USE_MODEL_DIR, 'metadata.json'),
JSON.stringify(modelMetadata, null, 2)
)
// eslint-disable-next-line no-console
console.log('✅ Model saved successfully!')
// eslint-disable-next-line no-console
console.log(`Model is now available at: ${USE_MODEL_DIR}`)
// Verify the model files exist
const modelJsonPath = path.join(USE_MODEL_DIR, 'model.json')
if (fs.existsSync(modelJsonPath)) {
// eslint-disable-next-line no-console
console.log('✅ model.json file verified')
// List the shard files
const modelFiles = fs.readdirSync(USE_MODEL_DIR)
const shardFiles = modelFiles.filter((file) => file.endsWith('.bin'))
// eslint-disable-next-line no-console
console.log(`Found ${shardFiles.length} model shard files:`)
// eslint-disable-next-line no-console
shardFiles.forEach((file) => console.log(` - ${file}`))
// eslint-disable-next-line no-console
console.log('\nModel reference files are ready!')
// eslint-disable-next-line no-console
console.log(
'IMPORTANT: These are NOT the full model files (~25MB), but reference files (~3KB total).'
)
// eslint-disable-next-line no-console
console.log(
'The full model will be downloaded automatically when your application first uses it.'
)
// eslint-disable-next-line no-console
console.log(
'After the first use, the model will be cached locally for future use.'
)
// eslint-disable-next-line no-console
console.log(
'These reference files should be checked into version control to ensure availability in all environments.'
)
} else {
// eslint-disable-next-line no-console
console.error('❌ model.json file not found after saving!')
// eslint-disable-next-line no-undef
process.exit(1)
}
} catch (error) {
// eslint-disable-next-line no-console
console.error('❌ Error downloading model:', error)
// eslint-disable-next-line no-undef
process.exit(1)
}
}
// eslint-disable-next-line no-console
downloadModel().catch(console.error)

190
scripts/download-models.cjs Executable file
View file

@ -0,0 +1,190 @@
#!/usr/bin/env node
/**
* Download and bundle models for offline usage
*/
const fs = require('fs').promises
const path = require('path')
const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'
const OUTPUT_DIR = './models'
async function downloadModels() {
// Use dynamic import for ES modules in CommonJS
const { pipeline, env } = await import('@huggingface/transformers')
// Configure transformers.js to use local cache
env.cacheDir = './models-cache'
env.allowRemoteModels = true
try {
console.log('🔄 Downloading all-MiniLM-L6-v2 model for offline bundling...')
console.log(` Model: ${MODEL_NAME}`)
console.log(` Cache: ${env.cacheDir}`)
// Create output directory
await fs.mkdir(OUTPUT_DIR, { recursive: true })
// Load the model to force download
console.log('📥 Loading model pipeline...')
const extractor = await pipeline('feature-extraction', MODEL_NAME)
// Test the model to make sure it works
console.log('🧪 Testing model...')
const testResult = await extractor(['Hello world!'], {
pooling: 'mean',
normalize: true
})
console.log(`✅ Model test successful! Embedding dimensions: ${testResult.data.length}`)
// Copy ALL model files from cache to our models directory
console.log('📋 Copying ALL model files to bundle directory...')
const cacheDir = path.resolve(env.cacheDir)
const outputDir = path.resolve(OUTPUT_DIR)
console.log(` From: ${cacheDir}`)
console.log(` To: ${outputDir}`)
// Copy the entire cache directory structure to ensure we get ALL files
// including tokenizer.json, config.json, and all ONNX model files
const modelCacheDir = path.join(cacheDir, 'Xenova', 'all-MiniLM-L6-v2')
if (await dirExists(modelCacheDir)) {
const targetModelDir = path.join(outputDir, 'Xenova', 'all-MiniLM-L6-v2')
console.log(` Copying complete model: Xenova/all-MiniLM-L6-v2`)
await copyDirectory(modelCacheDir, targetModelDir)
} else {
throw new Error(`Model cache directory not found: ${modelCacheDir}`)
}
console.log('✅ Model bundling complete!')
console.log(` Total size: ${await calculateDirectorySize(outputDir)} MB`)
console.log(` Location: ${outputDir}`)
// Create a marker file
await fs.writeFile(
path.join(outputDir, '.brainy-models-bundled'),
JSON.stringify({
model: MODEL_NAME,
bundledAt: new Date().toISOString(),
version: '1.0.0'
}, null, 2)
)
} catch (error) {
console.error('❌ Error downloading models:', error)
process.exit(1)
}
}
async function findModelDirectories(baseDir, modelName) {
const dirs = []
try {
// Convert model name to expected directory structure
const modelPath = modelName.replace('/', '--')
async function searchDirectory(currentDir) {
try {
const entries = await fs.readdir(currentDir, { withFileTypes: true })
for (const entry of entries) {
if (entry.isDirectory()) {
const fullPath = path.join(currentDir, entry.name)
// Check if this directory contains model files
if (entry.name.includes(modelPath) || entry.name === 'onnx') {
const hasModelFiles = await containsModelFiles(fullPath)
if (hasModelFiles) {
dirs.push(fullPath)
}
}
// Recursively search subdirectories
await searchDirectory(fullPath)
}
}
} catch (error) {
// Ignore access errors
}
}
await searchDirectory(baseDir)
} catch (error) {
console.warn('Warning: Error searching for model directories:', error)
}
return dirs
}
async function containsModelFiles(dir) {
try {
const files = await fs.readdir(dir)
return files.some(file =>
file.endsWith('.onnx') ||
file.endsWith('.json') ||
file === 'config.json' ||
file === 'tokenizer.json'
)
} catch (error) {
return false
}
}
async function dirExists(dir) {
try {
const stats = await fs.stat(dir)
return stats.isDirectory()
} catch (error) {
return false
}
}
async function copyDirectory(src, dest) {
await fs.mkdir(dest, { recursive: true })
const entries = await fs.readdir(src, { withFileTypes: true })
for (const entry of entries) {
const srcPath = path.join(src, entry.name)
const destPath = path.join(dest, entry.name)
if (entry.isDirectory()) {
await copyDirectory(srcPath, destPath)
} else {
await fs.copyFile(srcPath, destPath)
}
}
}
async function calculateDirectorySize(dir) {
let size = 0
async function calculateSize(currentDir) {
try {
const entries = await fs.readdir(currentDir, { withFileTypes: true })
for (const entry of entries) {
const fullPath = path.join(currentDir, entry.name)
if (entry.isDirectory()) {
await calculateSize(fullPath)
} else {
const stats = await fs.stat(fullPath)
size += stats.size
}
}
} catch (error) {
// Ignore access errors
}
}
await calculateSize(dir)
return Math.round(size / (1024 * 1024))
}
// Run the download
downloadModels().catch(error => {
console.error('Fatal error:', error)
process.exit(1)
})

47
scripts/encode-image.js Normal file
View file

@ -0,0 +1,47 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Path to the image file
const imagePath = path.join(__dirname, '..', 'brainy.png');
// Read the image file
const imageBuffer = fs.readFileSync(imagePath);
// Convert the image to base64
const base64Image = imageBuffer.toString('base64');
// Get the MIME type based on file extension
const getMimeType = (filePath) => {
const ext = path.extname(filePath).toLowerCase();
switch (ext) {
case '.png':
return 'image/png';
case '.jpg':
case '.jpeg':
return 'image/jpeg';
case '.gif':
return 'image/gif';
case '.svg':
return 'image/svg+xml';
default:
return 'application/octet-stream';
}
};
const mimeType = getMimeType(imagePath);
// Create the data URL
const dataUrl = `data:${mimeType};base64,${base64Image}`;
// Output the complete HTML img tag
const imgTag = `<img src="${dataUrl}" alt="Brainy Logo" width="200"/>`;
// Write to a file instead of console.log to avoid truncation
fs.writeFileSync(path.join(__dirname, '..', 'encoded-image.html'), imgTag);
console.log('Base64 encoded image has been saved to encoded-image.html');

201
scripts/extract-models.js Normal file
View file

@ -0,0 +1,201 @@
#!/usr/bin/env node
/**
* Extract Brainy Models Script
*
* Automatically extracts models from @soulcraft/brainy-models during Docker builds
* Works across all cloud providers (Google Cloud Run, AWS Lambda/ECS, Azure Container Instances, Cloudflare Workers)
*/
import { existsSync, mkdirSync, cpSync, readFileSync, writeFileSync } from 'fs'
import { join, dirname } from 'path'
import { fileURLToPath } from 'url'
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
function log(message) {
console.log(`[Brainy Model Extractor] ${message}`)
}
async function extractModels() {
try {
log('🔍 Checking for @soulcraft/brainy-models...')
// Get the project root (one level up from scripts/)
const projectRoot = join(__dirname, '..')
const modelsPackagePath = join(projectRoot, 'node_modules', '@soulcraft', 'brainy-models')
if (!existsSync(modelsPackagePath)) {
log('⚠️ @soulcraft/brainy-models not found - skipping model extraction')
log(' Models will be downloaded at runtime (slower startup)')
return false
}
log('✅ Found @soulcraft/brainy-models package')
// Create the models directory in the project root
const targetModelsDir = join(projectRoot, 'models')
if (existsSync(targetModelsDir)) {
log('📁 Models directory already exists - removing old version')
// Remove existing models directory to ensure clean extraction
try {
import('fs').then(fs => {
fs.rmSync(targetModelsDir, { recursive: true, force: true })
})
} catch (error) {
log(`⚠️ Could not remove existing models directory: ${error.message}`)
}
}
log('📦 Creating models directory...')
mkdirSync(targetModelsDir, { recursive: true })
// Look for models in the package
const possibleModelsPaths = [
join(modelsPackagePath, 'models'),
join(modelsPackagePath, 'dist', 'models'),
modelsPackagePath // Root of the package
]
let modelsSourcePath = null
for (const path of possibleModelsPaths) {
if (existsSync(path)) {
// Check if this directory contains model files
try {
const fs = await import('fs')
const files = fs.readdirSync(path)
if (files.length > 0) {
modelsSourcePath = path
break
}
} catch (error) {
continue
}
}
}
if (!modelsSourcePath) {
log('❌ Could not find models in @soulcraft/brainy-models package')
return false
}
log(`📋 Copying models from: ${modelsSourcePath}`)
log(`📋 Copying models to: ${targetModelsDir}`)
// Copy all models
try {
cpSync(modelsSourcePath, targetModelsDir, {
recursive: true,
force: true,
filter: (src, dest) => {
// Skip node_modules and other unnecessary files
const filename = src.split('/').pop() || ''
return !filename.startsWith('.') && filename !== 'node_modules'
}
})
log('✅ Models extracted successfully!')
// Create a marker file to indicate successful extraction
const markerFile = join(targetModelsDir, '.brainy-models-extracted')
writeFileSync(markerFile, JSON.stringify({
extractedAt: new Date().toISOString(),
sourcePackage: '@soulcraft/brainy-models',
extractorVersion: '1.0.0'
}, null, 2))
// List extracted models
try {
const fs = await import('fs')
const extractedItems = fs.readdirSync(targetModelsDir)
log(`📊 Extracted items: ${extractedItems.join(', ')}`)
} catch (error) {
log('📊 Model extraction completed (could not list contents)')
}
return true
} catch (error) {
log(`❌ Failed to copy models: ${error.message}`)
return false
}
} catch (error) {
log(`❌ Model extraction failed: ${error.message}`)
return false
}
}
// Auto-detect environment and provide helpful information
function detectEnvironment() {
const envs = []
// Docker detection
if (existsSync('/.dockerenv') || process.env.DOCKER_CONTAINER) {
envs.push('Docker')
}
// Cloud provider detection
if (process.env.GOOGLE_CLOUD_PROJECT || process.env.GAE_SERVICE) {
envs.push('Google Cloud')
}
if (process.env.AWS_EXECUTION_ENV || process.env.AWS_LAMBDA_FUNCTION_NAME) {
envs.push('AWS')
}
if (process.env.AZURE_CLIENT_ID || process.env.WEBSITE_SITE_NAME) {
envs.push('Azure')
}
if (process.env.CF_PAGES || process.env.CLOUDFLARE_ACCOUNT_ID) {
envs.push('Cloudflare')
}
if (process.env.VERCEL || process.env.VERCEL_ENV) {
envs.push('Vercel')
}
if (process.env.NETLIFY || process.env.NETLIFY_BUILD_BASE) {
envs.push('Netlify')
}
return envs
}
// Main execution
async function main() {
log('🚀 Starting Brainy model extraction...')
const detectedEnvs = detectEnvironment()
if (detectedEnvs.length > 0) {
log(`🌐 Detected environment(s): ${detectedEnvs.join(', ')}`)
}
const success = await extractModels()
if (success) {
log('🎉 Model extraction completed successfully!')
log('💡 Models are now embedded in your container/deployment')
log('💡 No runtime model downloads required!')
// Set environment variable hint for runtime
log('💡 Runtime will automatically detect extracted models')
} else {
log('⚠️ Model extraction failed or skipped')
log('💡 Application will fall back to runtime model downloads')
log('💡 Consider installing @soulcraft/brainy-models for better performance')
}
}
// Run if called directly
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch(error => {
console.error('Fatal error:', error)
process.exit(1)
})
}
export { extractModels, detectEnvironment }

View file

@ -0,0 +1,83 @@
// Script to check if there's any data in the database
import { BrainyData } from './dist/brainyData.js';
async function checkDatabase() {
try {
console.log('Initializing BrainyData...');
const db = new BrainyData();
await db.init();
console.log('Getting database status...');
const status = await db.status();
console.log('Database status:', JSON.stringify(status, null, 2));
console.log('Getting statistics...');
const stats = await db.getStatistics();
console.log('Statistics:', JSON.stringify(stats, null, 2));
console.log('Getting all nouns...');
const nouns = await db.getAllNouns();
console.log(`Found ${nouns.length} nouns in the database.`);
if (nouns.length > 0) {
console.log('Sample of nouns:');
for (let i = 0; i < Math.min(5, nouns.length); i++) {
console.log(`Noun ${i + 1}:`, JSON.stringify(nouns[i], null, 2));
}
}
console.log('Getting all verbs...');
const verbs = await db.getAllVerbs();
console.log(`Found ${verbs.length} verbs in the database.`);
if (verbs.length > 0) {
console.log('Sample of verbs:');
for (let i = 0; i < Math.min(5, verbs.length); i++) {
console.log(`Verb ${i + 1}:`, JSON.stringify(verbs[i], null, 2));
}
}
// Try a simple search to see if it returns any results
console.log('Trying a simple search...');
const searchResults = await db.searchText('test', 10);
console.log(`Search returned ${searchResults.length} results.`);
if (searchResults.length > 0) {
console.log('Sample of search results:');
for (let i = 0; i < Math.min(5, searchResults.length); i++) {
console.log(`Result ${i + 1}:`, JSON.stringify({
id: searchResults[i].id,
score: searchResults[i].score,
metadata: searchResults[i].metadata
}, null, 2));
}
}
// If no results, try adding a test item and searching again
if (searchResults.length === 0 && nouns.length === 0) {
console.log('No data found. Adding a test item...');
const id = await db.add('This is a test item for searching', { noun: 'Thing', category: 'test' });
console.log(`Added test item with ID: ${id}`);
console.log('Trying search again...');
const newSearchResults = await db.searchText('test', 10);
console.log(`Search returned ${newSearchResults.length} results.`);
if (newSearchResults.length > 0) {
console.log('Sample of search results:');
for (let i = 0; i < Math.min(5, newSearchResults.length); i++) {
console.log(`Result ${i + 1}:`, JSON.stringify({
id: newSearchResults[i].id,
score: newSearchResults[i].score,
metadata: newSearchResults[i].metadata
}, null, 2));
}
}
}
} catch (error) {
console.error('Error checking database:', error);
}
}
checkDatabase().catch(console.error);

View file

@ -0,0 +1,163 @@
// Script to fix dimension mismatch by re-embedding existing data
import { BrainyData } from './dist/brainyData.js';
import fs from 'fs';
import path from 'path';
async function fixDimensionMismatch() {
try {
console.log('Starting dimension mismatch fix...');
// Create a backup of the existing data
const backupDir = './brainy-data-backup-' + Date.now();
console.log(`Creating backup of existing data in ${backupDir}...`);
// Copy the entire brainy-data directory to the backup directory
await fs.promises.mkdir(backupDir, { recursive: true });
await copyDirectory('./brainy-data', backupDir);
console.log('Backup created successfully.');
// Initialize BrainyData with the current embedding function
console.log('Initializing BrainyData...');
const db = new BrainyData();
await db.init();
// Get database status to check if there's any data
const status = await db.status();
console.log('Database status:', JSON.stringify(status, null, 2));
// Read all noun files directly from the filesystem
console.log('Reading noun files directly from filesystem...');
const nounsDir = './brainy-data/nouns';
const files = await fs.promises.readdir(nounsDir);
// Process each noun file
const processedNouns = [];
for (const file of files) {
if (file.endsWith('.json')) {
const filePath = path.join(nounsDir, file);
const data = await fs.promises.readFile(filePath, 'utf-8');
const parsedNoun = JSON.parse(data);
// Get the metadata for this noun
const metadataPath = path.join('./brainy-data/metadata', `${parsedNoun.id}.json`);
let metadata = {};
try {
const metadataData = await fs.promises.readFile(metadataPath, 'utf-8');
metadata = JSON.parse(metadataData);
} catch (error) {
console.warn(`No metadata found for noun ${parsedNoun.id}`);
}
// Extract text from metadata if available
let text = '';
if (metadata.text) {
text = metadata.text;
} else if (metadata.description) {
text = metadata.description;
} else {
// If no text is available, use a placeholder
text = `Noun ${parsedNoun.id}`;
console.warn(`No text found for noun ${parsedNoun.id}, using placeholder`);
}
// Re-embed the text using the current embedding function
console.log(`Re-embedding noun ${parsedNoun.id}...`);
try {
// Delete the existing noun first
await db.delete(parsedNoun.id);
// Add the noun with the same ID but new vector
const newId = await db.add(text, metadata, { id: parsedNoun.id });
processedNouns.push({ id: newId, originalId: parsedNoun.id });
console.log(`Successfully re-embedded noun ${parsedNoun.id}`);
} catch (error) {
console.error(`Error re-embedding noun ${parsedNoun.id}:`, error);
}
}
}
console.log(`Processed ${processedNouns.length} nouns.`);
// Recreate verbs
console.log('Reading verb files directly from filesystem...');
const verbsDir = './brainy-data/verbs';
const verbFiles = await fs.promises.readdir(verbsDir);
// Process each verb file
const processedVerbs = [];
for (const file of verbFiles) {
if (file.endsWith('.json')) {
const filePath = path.join(verbsDir, file);
const data = await fs.promises.readFile(filePath, 'utf-8');
const parsedVerb = JSON.parse(data);
// Check if both source and target nouns exist
const sourceExists = processedNouns.some(n => n.originalId === parsedVerb.sourceId);
const targetExists = processedNouns.some(n => n.originalId === parsedVerb.targetId);
if (sourceExists && targetExists) {
console.log(`Re-creating verb ${parsedVerb.id} between ${parsedVerb.sourceId} and ${parsedVerb.targetId}...`);
try {
// Delete the existing verb first
await db.deleteVerb(parsedVerb.id);
// Add the verb with the same relationship
await db.addVerb(parsedVerb.sourceId, parsedVerb.targetId, {
verb: parsedVerb.type || 'RelatedTo',
...parsedVerb.metadata
});
processedVerbs.push(parsedVerb.id);
console.log(`Successfully re-created verb ${parsedVerb.id}`);
} catch (error) {
console.error(`Error re-creating verb ${parsedVerb.id}:`, error);
}
} else {
console.warn(`Skipping verb ${parsedVerb.id} because source or target noun doesn't exist`);
}
}
}
console.log(`Processed ${processedVerbs.length} verbs.`);
// Try a search to verify it works
console.log('Trying a search to verify it works...');
const searchResults = await db.searchText('test', 10);
console.log(`Search returned ${searchResults.length} results.`);
if (searchResults.length > 0) {
console.log('Sample of search results:');
for (let i = 0; i < Math.min(5, searchResults.length); i++) {
console.log(`Result ${i + 1}:`, JSON.stringify({
id: searchResults[i].id,
score: searchResults[i].score,
metadata: searchResults[i].metadata
}, null, 2));
}
}
console.log('Dimension mismatch fix completed successfully.');
} catch (error) {
console.error('Error fixing dimension mismatch:', error);
}
}
// Helper function to copy a directory recursively
async function copyDirectory(source, destination) {
const entries = await fs.promises.readdir(source, { withFileTypes: true });
await fs.promises.mkdir(destination, { recursive: true });
for (const entry of entries) {
const srcPath = path.join(source, entry.name);
const destPath = path.join(destination, entry.name);
if (entry.isDirectory()) {
await copyDirectory(srcPath, destPath);
} else {
await fs.promises.copyFile(srcPath, destPath);
}
}
}
fixDimensionMismatch().catch(console.error);

141
scripts/patch-textencoder.js Executable file
View file

@ -0,0 +1,141 @@
#!/usr/bin/env node
/**
* Simplified TextEncoder Patch
*
* This script patches the compiled unified.js file to fix the TextEncoder issue in Node.js
* by replacing references to this.util.TextEncoder with direct TextEncoder usage.
*/
import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
// Path to the compiled unified.js file
const unifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.js')
// Read the file
console.log(`Reading ${unifiedJsPath}...`)
let content = fs.readFileSync(unifiedJsPath, 'utf8')
// Simple replacement: replace all instances of new this.util.TextEncoder() with new TextEncoder()
const pattern = /new\s+this\.util\.TextEncoder\(\)/g
const replacement = 'new TextEncoder()'
// Apply the patch
const patchedContent = content.replace(pattern, replacement)
// Check if the patch was applied
if (patchedContent === content) {
console.warn(
'No instances of "new this.util.TextEncoder()" found in the file.'
)
} else {
// Write the patched file
console.log('Writing patched file...')
fs.writeFileSync(unifiedJsPath, patchedContent, 'utf8')
console.log('Patch applied successfully!')
}
// Also patch the minified version if it exists
const minifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.min.js')
if (fs.existsSync(minifiedJsPath)) {
console.log(`Reading ${minifiedJsPath}...`)
const minContent = fs.readFileSync(minifiedJsPath, 'utf8')
// Apply the same replacement to the minified file
const patchedMinContent = minContent.replace(pattern, replacement)
// Check if the patch was applied
if (patchedMinContent === minContent) {
console.warn(
'No instances of "new this.util.TextEncoder()" found in the minified file.'
)
} else {
// Write the patched file
console.log('Writing patched minified file...')
fs.writeFileSync(minifiedJsPath, patchedMinContent, 'utf8')
console.log('Minified file patched successfully!')
}
}
// Also patch the worker.js file if it exists
const workerJsPath = path.join(__dirname, '..', 'dist', 'worker.js')
if (fs.existsSync(workerJsPath)) {
console.log(`Reading ${workerJsPath}...`)
const workerContent = fs.readFileSync(workerJsPath, 'utf8')
// Apply the same replacement to the worker file
const patchedWorkerContent = workerContent.replace(pattern, replacement)
// Check if the patch was applied
if (patchedWorkerContent === workerContent) {
console.warn(
'No instances of "new this.util.TextEncoder()" found in the worker file.'
)
} else {
// Write the patched file
console.log('Writing patched worker file...')
fs.writeFileSync(workerJsPath, patchedWorkerContent, 'utf8')
console.log('Worker file patched successfully!')
}
}
// Also patch TextDecoder
console.log('Patching TextDecoder references...')
content = fs.readFileSync(unifiedJsPath, 'utf8')
const decoderPattern = /new\s+this\.util\.TextDecoder\(\)/g
const decoderReplacement = 'new TextDecoder()'
const patchedDecoderContent = content.replace(
decoderPattern,
decoderReplacement
)
if (patchedDecoderContent === content) {
console.warn(
'No instances of "new this.util.TextDecoder()" found in the file.'
)
} else {
fs.writeFileSync(unifiedJsPath, patchedDecoderContent, 'utf8')
console.log('TextDecoder patch applied successfully!')
}
// Patch the minified file for TextDecoder as well
if (fs.existsSync(minifiedJsPath)) {
const minContent = fs.readFileSync(minifiedJsPath, 'utf8')
const patchedMinDecoderContent = minContent.replace(
decoderPattern,
decoderReplacement
)
if (patchedMinDecoderContent === minContent) {
console.warn(
'No instances of "new this.util.TextDecoder()" found in the minified file.'
)
} else {
fs.writeFileSync(minifiedJsPath, patchedMinDecoderContent, 'utf8')
console.log('TextDecoder patch applied to minified file successfully!')
}
}
// Patch the worker.js file for TextDecoder as well
if (fs.existsSync(workerJsPath)) {
const workerContent = fs.readFileSync(workerJsPath, 'utf8')
const patchedWorkerDecoderContent = workerContent.replace(
decoderPattern,
decoderReplacement
)
if (patchedWorkerDecoderContent === workerContent) {
console.warn(
'No instances of "new this.util.TextDecoder()" found in the worker file.'
)
} else {
fs.writeFileSync(workerJsPath, patchedWorkerDecoderContent, 'utf8')
console.log('TextDecoder patch applied to worker file successfully!')
}
}

View file

@ -0,0 +1,148 @@
#!/usr/bin/env node
/**
* Comprehensive production readiness check for Brainy
* Tests all deployment scenarios to prevent emergency releases
*/
import { BrainyData, NounType, VerbType } from '../dist/index.js'
import fs from 'fs'
import { execSync } from 'child_process'
async function cleanEnvironment() {
// Clean up any existing models or cache
try {
if (fs.existsSync('./models')) {
console.log('🧹 Cleaning existing models directory...')
fs.rmSync('./models', { recursive: true, force: true })
}
if (fs.existsSync('./brainy-data')) {
console.log('🧹 Cleaning existing data directory...')
fs.rmSync('./brainy-data', { recursive: true, force: true })
}
} catch (error) {
console.log('Note: Some cleanup operations failed (this may be normal)')
}
}
async function testScenario(name, envVars, expectedResult) {
console.log(`\n🧪 TESTING: ${name}`)
console.log(`Environment: ${JSON.stringify(envVars)}`)
// Set environment variables
for (const [key, value] of Object.entries(envVars)) {
process.env[key] = value
}
try {
const brain = new BrainyData()
await brain.init()
// Test basic operations
const id = await brain.add("Test data for production scenario")
const results = await brain.search("test", 1)
if (expectedResult === 'success') {
console.log(`✅ SUCCESS: ${name} - Operations completed successfully`)
return true
} else {
console.log(`❌ UNEXPECTED SUCCESS: ${name} - Expected failure but got success`)
return false
}
} catch (error) {
if (expectedResult === 'failure') {
console.log(`✅ EXPECTED FAILURE: ${name} - ${error.message}`)
return true
} else {
console.log(`❌ UNEXPECTED FAILURE: ${name} - ${error.message}`)
return false
}
} finally {
// Clean environment variables
for (const key of Object.keys(envVars)) {
delete process.env[key]
}
}
}
async function runProductionReadinessCheck() {
console.log('🏭 BRAINY PRODUCTION READINESS CHECK')
console.log('====================================')
console.log('Testing all deployment scenarios to ensure no emergency releases')
await cleanEnvironment()
const testResults = []
// Test 1: Fresh install with default settings (NEW: should work with remote downloads)
testResults.push(await testScenario(
'Fresh Install - Default Settings',
{},
'success' // This should now work with the fix
))
// Test 2: Explicit remote models enabled
testResults.push(await testScenario(
'Remote Models Explicitly Enabled',
{ BRAINY_ALLOW_REMOTE_MODELS: 'true' },
'success'
))
// Test 3: Remote models explicitly disabled (air-gapped scenario)
testResults.push(await testScenario(
'Air-Gapped Deployment (Local Only)',
{ BRAINY_ALLOW_REMOTE_MODELS: 'false' },
'failure' // Should fail gracefully with helpful error
))
// Test 4: Development environment
testResults.push(await testScenario(
'Development Environment',
{ NODE_ENV: 'development' },
'success'
))
// Test 5: Production environment with explicit config
testResults.push(await testScenario(
'Production Environment',
{ NODE_ENV: 'production', BRAINY_ALLOW_REMOTE_MODELS: 'true' },
'success'
))
const successCount = testResults.filter(result => result).length
const totalTests = testResults.length
console.log('\n📊 PRODUCTION READINESS RESULTS')
console.log('===============================')
console.log(`Passed: ${successCount}/${totalTests}`)
if (successCount === totalTests) {
console.log('✅ ALL TESTS PASSED - PRODUCTION READY!')
console.log('\n🚀 DEPLOYMENT SCENARIOS VERIFIED:')
console.log(' ✅ Fresh npm install works out of the box')
console.log(' ✅ Remote model downloads work when enabled')
console.log(' ✅ Air-gapped deployments fail gracefully with clear guidance')
console.log(' ✅ Development environments work seamlessly')
console.log(' ✅ Production environments work with proper configuration')
return true
} else {
console.log('❌ PRODUCTION READINESS CHECK FAILED')
console.log('🚫 DO NOT RELEASE - Fix issues first')
return false
}
}
// Export for use in CI/CD
export { runProductionReadinessCheck }
// Run if called directly
if (import.meta.url === `file://${process.argv[1]}`) {
runProductionReadinessCheck()
.then(success => process.exit(success ? 0 : 1))
.catch(error => {
console.error('❌ Production readiness check crashed:', error)
process.exit(1)
})
}

12
scripts/publish-cli.js Executable file
View file

@ -0,0 +1,12 @@
#!/usr/bin/env node
/**
* DEPRECATED: Script to build and publish both the main package and CLI package
*
* This script is no longer functional as the CLI package has been removed.
* Use 'npm publish' directly to publish the main package only.
*/
console.error('This script is deprecated. The CLI package has been removed.')
console.error('To publish the main package only, use: npm publish')
process.exit(1)

164
scripts/release-workflow.js Executable file
View file

@ -0,0 +1,164 @@
#!/usr/bin/env node
/**
* Release Workflow Script
*
* This script provides a comprehensive workflow for releasing a new version:
* 1. Updates the version (major, minor, or patch)
* 2. Automatically updates the CHANGELOG.md with commit messages since the last release
* 3. Creates a GitHub release
* 4. Deploys to NPM
*
* Usage:
* node scripts/release-workflow.js [patch|minor|major]
*
* If no version type is specified, it defaults to "patch"
*/
/* global process, console */
import { execSync } from 'child_process'
import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
import readline from 'readline'
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
// Path to the root directory
const rootDir = path.join(__dirname, '..')
// Get the version type from command line arguments
const args = process.argv.slice(2)
const versionType = args[0] || 'patch'
// Validate version type
if (!['patch', 'minor', 'major'].includes(versionType)) {
// eslint-disable-next-line no-console
console.error('Error: Version type must be one of: patch, minor, major')
// eslint-disable-next-line no-process-exit
process.exit(1)
}
// Function to execute a command and log its output
function executeStep(command, description) {
// eslint-disable-next-line no-console
console.log(`\n🚀 ${description}...\n`)
try {
execSync(command, { stdio: 'inherit', cwd: rootDir })
// eslint-disable-next-line no-console
console.log(`${description} completed successfully!\n`)
return true
} catch (error) {
// eslint-disable-next-line no-console
console.error(
`❌ Error during ${description.toLowerCase()}: ${error.message}`
)
return false
}
}
// Main workflow
async function runReleaseWorkflow() {
// eslint-disable-next-line no-console
console.log(`\n=== Starting Release Workflow (${versionType}) ===\n`)
// Step 1: Build the project
if (!executeStep('npm run build', 'Building project')) {
// eslint-disable-next-line no-process-exit
process.exit(1)
}
// Step 2: Run tests to ensure everything is working
if (!executeStep('npm test', 'Running tests')) {
// eslint-disable-next-line no-console
console.warn(
'⚠️ Tests failed. This might indicate issues with the release.'
)
// Ask the user if they want to continue despite test failures
// eslint-disable-next-line no-console
console.log(
'\n⚠ Do you want to continue with the release process despite test failures? (y/N)'
)
const readline = require('readline').createInterface({
input: process.stdin,
output: process.stdout
})
const response = await new Promise((resolve) => {
readline.question('', (answer) => {
readline.close()
resolve(answer.toLowerCase())
})
})
if (response !== 'y' && response !== 'yes') {
// eslint-disable-next-line no-console
console.error('Release process aborted due to test failures.')
// eslint-disable-next-line no-process-exit
process.exit(1)
}
// eslint-disable-next-line no-console
console.log('Continuing with release process despite test failures...')
}
// Step 3: Update version and generate changelog
if (
!executeStep(
`npm run _release:${versionType}`,
`Updating version (${versionType}) and generating changelog`
)
) {
// eslint-disable-next-line no-process-exit
process.exit(1)
}
// Step 4: Create GitHub release
if (!executeStep('npm run _github-release', 'Creating GitHub release')) {
// eslint-disable-next-line no-console
console.log(
'Warning: GitHub release creation failed, but continuing with deployment...'
)
}
// Step 5: Publish to NPM
if (!executeStep('npm publish', 'Publishing to NPM')) {
// eslint-disable-next-line no-process-exit
process.exit(1)
}
// Get the new version from package.json
const packageJsonPath = path.join(rootDir, 'package.json')
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
const newVersion = packageJson.version
// eslint-disable-next-line no-console
console.log(`\n🎉 Release v${newVersion} completed successfully! 🎉\n`)
// eslint-disable-next-line no-console
console.log('Summary of actions:')
// eslint-disable-next-line no-console
console.log(`- Project built and tested`)
// eslint-disable-next-line no-console
console.log(`- Version bumped to v${newVersion} (${versionType})`)
// eslint-disable-next-line no-console
console.log(`- CHANGELOG.md updated with recent commits`)
// eslint-disable-next-line no-console
console.log(`- GitHub release created with auto-generated notes`)
// eslint-disable-next-line no-console
console.log(`- Package published to NPM`)
// eslint-disable-next-line no-console
console.log('\nThank you for using the release workflow!\n')
}
// Run the workflow
runReleaseWorkflow().catch((error) => {
// eslint-disable-next-line no-console
console.error('Unexpected error during release workflow:', error)
// eslint-disable-next-line no-process-exit
process.exit(1)
})

View file

@ -0,0 +1,83 @@
#!/usr/bin/env node
/**
* Brain Jar Broadcast Server
*
* Start this server to enable real-time communication between
* multiple Claude instances (Jarvis, Picasso, etc.)
*
* Usage:
* npm run broadcast:local # Start local server on port 8765
* npm run broadcast:cloud # Start cloud server on port 8080
*/
import { BrainyData } from '../src/brainyData.js'
import { BrainyMCPBroadcast } from '../src/mcp/brainyMCPBroadcast.js'
const isCloud = process.argv.includes('--cloud')
const port = isCloud ? 8080 : 8765
async function startServer() {
console.log('🧠🫙 Starting Brain Jar Broadcast Server...')
console.log('=====================================')
// Initialize Brainy for server-side memory
const brainy = new BrainyData({
storagePath: '.brain-jar/server',
dimensions: 384
})
await brainy.init()
console.log('✅ Brainy initialized for server memory')
// Create broadcast server
const broadcast = new BrainyMCPBroadcast(brainy, {
broadcastPort: port,
cloudUrl: isCloud ? process.env.CLOUD_URL : undefined
})
// Start the server
await broadcast.startBroadcastServer(port, isCloud)
console.log('')
console.log('🚀 Server Ready!')
console.log('=====================================')
console.log('Claude instances can now connect using:')
console.log('')
console.log('For Jarvis (Backend):')
console.log(` const client = new BrainyMCPClient({`)
console.log(` name: 'Jarvis',`)
console.log(` role: 'Backend Systems',`)
console.log(` serverUrl: 'ws://localhost:${port}'`)
console.log(` })`)
console.log(` await client.connect()`)
console.log('')
console.log('For Picasso (Frontend):')
console.log(` const client = new BrainyMCPClient({`)
console.log(` name: 'Picasso',`)
console.log(` role: 'Frontend Design',`)
console.log(` serverUrl: 'ws://localhost:${port}'`)
console.log(` })`)
console.log(` await client.connect()`)
console.log('')
console.log('=====================================')
console.log('Press Ctrl+C to stop the server')
// Handle graceful shutdown
process.on('SIGINT', async () => {
console.log('\n📛 Shutting down server...')
await broadcast.stopBroadcastServer()
process.exit(0)
})
process.on('SIGTERM', async () => {
await broadcast.stopBroadcastServer()
process.exit(0)
})
}
// Start the server
startServer().catch(error => {
console.error('Failed to start server:', error)
process.exit(1)
})

26
scripts/update-readme.js Normal file
View file

@ -0,0 +1,26 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Paths to the files
const readmePath = path.join(__dirname, '..', 'README.md');
const encodedImagePath = path.join(__dirname, '..', 'encoded-image.html');
// Read the files
const readmeContent = fs.readFileSync(readmePath, 'utf8');
const encodedImageTag = fs.readFileSync(encodedImagePath, 'utf8');
// Replace the image tag in the README
const updatedReadme = readmeContent.replace(
/<img src="brainy\.png" alt="Brainy Logo" width="200"\/>/,
encodedImageTag
);
// Write the updated README
fs.writeFileSync(readmePath, updatedReadme);
console.log('README.md has been updated with the base64-encoded image.');