Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
187
scripts/analyze-metadata-performance.js
Normal file
187
scripts/analyze-metadata-performance.js
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Metadata Performance Analysis Script
|
||||
* Quick performance analysis of metadata filtering system without full test suite
|
||||
*/
|
||||
|
||||
import { BrainyData } from '../dist/brainyData.js'
|
||||
|
||||
const measureTime = async (fn) => {
|
||||
const start = performance.now()
|
||||
const result = await fn()
|
||||
const end = performance.now()
|
||||
return { result, time: end - start }
|
||||
}
|
||||
|
||||
const generateTestData = (count) => {
|
||||
const departments = ['Engineering', 'Marketing', 'Sales', 'HR']
|
||||
const levels = ['junior', 'senior', 'staff', 'principal']
|
||||
const locations = ['SF', 'NYC', 'LA', 'Seattle']
|
||||
|
||||
return Array.from({ length: count }, (_, i) => ({
|
||||
text: `Profile ${i}: Professional with experience in software development`,
|
||||
metadata: {
|
||||
id: `profile-${i}`,
|
||||
department: departments[i % departments.length],
|
||||
level: levels[i % levels.length],
|
||||
location: locations[i % locations.length],
|
||||
salary: 50000 + (i % 10) * 10000,
|
||||
remote: i % 3 === 0,
|
||||
active: i % 5 !== 0
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async function analyzePerformance() {
|
||||
console.log('=== Metadata Performance Analysis ===\n')
|
||||
|
||||
// Test 1: Initialization with vs without metadata indexing
|
||||
console.log('1. INITIALIZATION COMPARISON')
|
||||
|
||||
const testData = generateTestData(100)
|
||||
|
||||
// Without indexing
|
||||
const withoutIndex = await measureTime(async () => {
|
||||
const brainy = new BrainyData({
|
||||
storage: { forceMemoryStorage: true },
|
||||
logging: { verbose: false }
|
||||
})
|
||||
await brainy.init()
|
||||
|
||||
for (const item of testData) {
|
||||
await brainy.add(item.text, item.metadata)
|
||||
}
|
||||
|
||||
return brainy
|
||||
})
|
||||
|
||||
console.log(`WITHOUT indexing: ${withoutIndex.time.toFixed(2)}ms for 100 items`)
|
||||
|
||||
// With indexing
|
||||
const withIndex = await measureTime(async () => {
|
||||
const brainy = new BrainyData({
|
||||
storage: { forceMemoryStorage: true },
|
||||
logging: { verbose: false },
|
||||
metadataIndex: { autoOptimize: true }
|
||||
})
|
||||
await brainy.init()
|
||||
|
||||
for (const item of testData) {
|
||||
await brainy.add(item.text, item.metadata)
|
||||
}
|
||||
|
||||
return brainy
|
||||
})
|
||||
|
||||
console.log(`WITH indexing: ${withIndex.time.toFixed(2)}ms for 100 items`)
|
||||
const overhead = ((withIndex.time - withoutIndex.time) / withoutIndex.time) * 100
|
||||
console.log(`Index overhead: ${overhead.toFixed(1)}%\n`)
|
||||
|
||||
// Test 2: Search Performance Comparison
|
||||
console.log('2. SEARCH PERFORMANCE COMPARISON')
|
||||
|
||||
const brainy = withIndex.result
|
||||
const searchQuery = 'Professional software development'
|
||||
const numSearches = 5
|
||||
|
||||
// No filtering
|
||||
let totalNoFilter = 0
|
||||
for (let i = 0; i < numSearches; i++) {
|
||||
const { time } = await measureTime(async () => {
|
||||
return await brainy.search(searchQuery, 10)
|
||||
})
|
||||
totalNoFilter += time
|
||||
}
|
||||
const avgNoFilter = totalNoFilter / numSearches
|
||||
console.log(`No filtering: ${avgNoFilter.toFixed(2)}ms average`)
|
||||
|
||||
// Simple filtering
|
||||
let totalSimpleFilter = 0
|
||||
for (let i = 0; i < numSearches; i++) {
|
||||
const { time } = await measureTime(async () => {
|
||||
return await brainy.search(searchQuery, 10, {
|
||||
metadata: { department: 'Engineering' }
|
||||
})
|
||||
})
|
||||
totalSimpleFilter += time
|
||||
}
|
||||
const avgSimpleFilter = totalSimpleFilter / numSearches
|
||||
console.log(`Simple filter: ${avgSimpleFilter.toFixed(2)}ms average`)
|
||||
|
||||
// Complex filtering
|
||||
let totalComplexFilter = 0
|
||||
for (let i = 0; i < numSearches; i++) {
|
||||
const { time } = await measureTime(async () => {
|
||||
return await brainy.search(searchQuery, 10, {
|
||||
metadata: {
|
||||
department: { $in: ['Engineering', 'Marketing'] },
|
||||
level: { $in: ['senior', 'staff'] },
|
||||
salary: { $gte: 80000 }
|
||||
}
|
||||
})
|
||||
})
|
||||
totalComplexFilter += time
|
||||
}
|
||||
const avgComplexFilter = totalComplexFilter / numSearches
|
||||
console.log(`Complex filter: ${avgComplexFilter.toFixed(2)}ms average`)
|
||||
|
||||
console.log('\nSearch Performance Impact:')
|
||||
console.log(`Simple filter overhead: ${((avgSimpleFilter / avgNoFilter - 1) * 100).toFixed(1)}%`)
|
||||
console.log(`Complex filter overhead: ${((avgComplexFilter / avgNoFilter - 1) * 100).toFixed(1)}%\n`)
|
||||
|
||||
// Test 3: Index Statistics
|
||||
console.log('3. INDEX STATISTICS')
|
||||
|
||||
if (brainy.metadataIndex) {
|
||||
const stats = await brainy.metadataIndex.getStats()
|
||||
console.log(`Total index entries: ${stats.totalEntries}`)
|
||||
console.log(`Total indexed IDs: ${stats.totalIds}`)
|
||||
console.log(`Fields indexed: ${stats.fieldsIndexed.join(', ')}`)
|
||||
console.log(`Estimated index size: ${stats.indexSize} bytes`)
|
||||
console.log(`Storage overhead per item: ${(stats.indexSize / 100).toFixed(2)} bytes\n`)
|
||||
}
|
||||
|
||||
// Test 4: Write Performance
|
||||
console.log('4. WRITE PERFORMANCE ANALYSIS')
|
||||
|
||||
const newTestData = generateTestData(50)
|
||||
|
||||
// Add performance
|
||||
const { time: addTime } = await measureTime(async () => {
|
||||
for (const item of newTestData) {
|
||||
await brainy.add(item.text, item.metadata)
|
||||
}
|
||||
})
|
||||
console.log(`ADD: 50 items in ${addTime.toFixed(2)}ms (${(addTime / 50).toFixed(2)}ms per item)`)
|
||||
|
||||
// Update performance
|
||||
const updateData = newTestData.slice(0, 20).map(item => ({
|
||||
...item,
|
||||
metadata: { ...item.metadata, level: 'updated', salary: item.metadata.salary + 10000 }
|
||||
}))
|
||||
|
||||
const { time: updateTime } = await measureTime(async () => {
|
||||
for (const item of updateData) {
|
||||
await brainy.updateMetadata(item.metadata.id, item.metadata)
|
||||
}
|
||||
})
|
||||
console.log(`UPDATE: 20 items in ${updateTime.toFixed(2)}ms (${(updateTime / 20).toFixed(2)}ms per item)`)
|
||||
|
||||
// Delete performance
|
||||
const idsToDelete = newTestData.slice(30, 40).map(item => item.metadata.id)
|
||||
const { time: deleteTime } = await measureTime(async () => {
|
||||
for (const id of idsToDelete) {
|
||||
await brainy.delete(id)
|
||||
}
|
||||
})
|
||||
console.log(`DELETE: 10 items in ${deleteTime.toFixed(2)}ms (${(deleteTime / 10).toFixed(2)}ms per item)\n`)
|
||||
|
||||
// Cleanup
|
||||
await withoutIndex.result.shutDown()
|
||||
await withIndex.result.shutDown()
|
||||
|
||||
console.log('Analysis complete!')
|
||||
}
|
||||
|
||||
analyzePerformance().catch(console.error)
|
||||
93
scripts/check-code-style.js
Executable file
93
scripts/check-code-style.js
Executable file
|
|
@ -0,0 +1,93 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Script to check code style and enforce no-semicolon rule
|
||||
* This script runs eslint and prettier checks on the codebase
|
||||
*/
|
||||
|
||||
const { execSync } = require('child_process')
|
||||
const path = require('path')
|
||||
const fs = require('fs')
|
||||
|
||||
// ANSI color codes for terminal output
|
||||
const colors = {
|
||||
reset: '\x1b[0m',
|
||||
red: '\x1b[31m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
blue: '\x1b[34m',
|
||||
magenta: '\x1b[35m',
|
||||
cyan: '\x1b[36m',
|
||||
white: '\x1b[37m'
|
||||
}
|
||||
|
||||
console.log(`${colors.cyan}Checking code style...${colors.reset}`)
|
||||
console.log(`${colors.cyan}====================${colors.reset}`)
|
||||
|
||||
// Run eslint
|
||||
try {
|
||||
console.log(`${colors.blue}Running ESLint...${colors.reset}`)
|
||||
execSync('npm run lint', { stdio: 'inherit' })
|
||||
console.log(`${colors.green}ESLint check passed!${colors.reset}`)
|
||||
} catch (error) {
|
||||
console.error(`${colors.red}ESLint check failed!${colors.reset}`)
|
||||
console.log(`${colors.yellow}Run 'npm run lint:fix' to automatically fix some issues.${colors.reset}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Run prettier check
|
||||
try {
|
||||
console.log(`${colors.blue}Running Prettier check...${colors.reset}`)
|
||||
execSync('npm run check-format', { stdio: 'inherit' })
|
||||
console.log(`${colors.green}Prettier check passed!${colors.reset}`)
|
||||
} catch (error) {
|
||||
console.error(`${colors.red}Prettier check failed!${colors.reset}`)
|
||||
console.log(`${colors.yellow}Run 'npm run format' to automatically format your code.${colors.reset}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Specific check for semicolons
|
||||
console.log(`${colors.blue}Checking for semicolons in code...${colors.reset}`)
|
||||
try {
|
||||
// Find all .ts and .js files in src directory
|
||||
const findCommand = "find src -type f -name '*.ts' -o -name '*.js'"
|
||||
const files = execSync(findCommand, { encoding: 'utf8' }).trim().split('\n')
|
||||
|
||||
let semicolonFound = false
|
||||
|
||||
for (const file of files) {
|
||||
if (!file) continue
|
||||
|
||||
const content = fs.readFileSync(file, 'utf8')
|
||||
const lines = content.split('\n')
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]
|
||||
// Skip comments and strings
|
||||
if (line.trim().startsWith('//') || line.trim().startsWith('/*') ||
|
||||
line.trim().startsWith('*') || line.trim().startsWith('*/')) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check for semicolons at the end of lines (excluding in string literals and comments)
|
||||
if (line.trim().endsWith(';') && !line.includes('//') && !line.includes('/*')) {
|
||||
console.error(`${colors.red}Semicolon found in ${file}:${i+1}${colors.reset}`)
|
||||
console.error(`${colors.yellow}${line}${colors.reset}`)
|
||||
semicolonFound = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (semicolonFound) {
|
||||
console.error(`${colors.red}Semicolons found in code! Please remove them.${colors.reset}`)
|
||||
process.exit(1)
|
||||
} else {
|
||||
console.log(`${colors.green}No semicolons found in code!${colors.reset}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`${colors.red}Error checking for semicolons: ${error}${colors.reset}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log(`${colors.green}All code style checks passed!${colors.reset}`)
|
||||
console.log(`${colors.cyan}Remember: No semicolons in code!${colors.reset}`)
|
||||
15
scripts/claude-commit.sh
Executable file
15
scripts/claude-commit.sh
Executable file
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script now calls the global claude-commit command
|
||||
# The global version is located at ~/.local/bin/claude-commit
|
||||
# Or can be installed from docs/tools/claude-commit/setup.sh
|
||||
|
||||
if command -v claude-commit >/dev/null 2>&1; then
|
||||
exec claude-commit "$@"
|
||||
elif [ -x "$HOME/.local/bin/claude-commit" ]; then
|
||||
exec "$HOME/.local/bin/claude-commit" "$@"
|
||||
else
|
||||
echo "claude-commit not found. Please run:"
|
||||
echo " ./docs/tools/claude-commit/setup.sh"
|
||||
exit 1
|
||||
fi
|
||||
20
scripts/create-favicon.js
Normal file
20
scripts/create-favicon.js
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
// Create a simple favicon.ico file
|
||||
import { writeFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Get the directory name of the current module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// This is a base64-encoded 16x16 transparent favicon
|
||||
const faviconBase64 = 'AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAABILAAASCwAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAA==';
|
||||
|
||||
// Path to save the favicon
|
||||
const faviconPath = join(__dirname, '..', 'favicon.ico');
|
||||
|
||||
// Convert base64 to binary and save
|
||||
const faviconBuffer = Buffer.from(faviconBase64, 'base64');
|
||||
writeFileSync(faviconPath, faviconBuffer);
|
||||
|
||||
console.log(`Favicon created at ${faviconPath}`);
|
||||
98
scripts/create-github-release.js
Normal file
98
scripts/create-github-release.js
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Create GitHub Release Script
|
||||
*
|
||||
* This script creates a GitHub release with auto-generated release notes
|
||||
* for the current version of the package.
|
||||
*
|
||||
* It uses the GitHub CLI (gh) to create the release, so the gh CLI must be installed
|
||||
* and authenticated with appropriate permissions.
|
||||
*
|
||||
* The script:
|
||||
* 1. Gets the current version from package.json
|
||||
* 2. Creates a GitHub release for that version
|
||||
* 3. Auto-generates release notes based on commits since the last release
|
||||
*
|
||||
* This ensures that each npm release has a corresponding GitHub release with notes.
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
// Path to the root directory
|
||||
const rootDir = path.join(__dirname, '..')
|
||||
|
||||
// Path to package.json
|
||||
const packageJsonPath = path.join(rootDir, 'package.json')
|
||||
|
||||
// Read package.json
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
|
||||
const version = packageJson.version
|
||||
|
||||
// Check if GitHub CLI is installed
|
||||
try {
|
||||
execSync('gh --version', { stdio: 'ignore' })
|
||||
} catch (error) {
|
||||
console.error('Error: GitHub CLI (gh) is not installed or not in PATH')
|
||||
console.error('Please install it from https://cli.github.com/ and authenticate with `gh auth login`')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Check if the tag exists locally
|
||||
let tagExistsLocally = false
|
||||
try {
|
||||
execSync(`git tag -l v${version}`, { stdio: 'pipe', cwd: rootDir }).toString().trim() === `v${version}` ? tagExistsLocally = true : tagExistsLocally = false
|
||||
} catch (error) {
|
||||
console.log(`Error checking if tag exists: ${error.message}`)
|
||||
tagExistsLocally = false
|
||||
}
|
||||
|
||||
// Push the tag to remote if it exists locally
|
||||
if (tagExistsLocally) {
|
||||
try {
|
||||
console.log(`Pushing tag v${version} to remote...`)
|
||||
execSync(`git push origin v${version}`, { stdio: 'inherit', cwd: rootDir })
|
||||
console.log(`Successfully pushed tag v${version} to remote`)
|
||||
} catch (error) {
|
||||
console.error(`Error pushing tag to remote: ${error.message}`)
|
||||
// Continue with release creation even if tag push fails
|
||||
}
|
||||
} else {
|
||||
console.log(`Tag v${version} does not exist locally, skipping tag push`)
|
||||
}
|
||||
|
||||
// Create the GitHub release
|
||||
try {
|
||||
console.log(`Creating GitHub release for v${version}...`)
|
||||
|
||||
// Create a release with auto-generated notes
|
||||
// The --generate-notes flag automatically generates release notes based on PRs and commits
|
||||
execSync(
|
||||
`gh release create v${version} --title "v${version}" --generate-notes`,
|
||||
{ stdio: 'inherit', cwd: rootDir }
|
||||
)
|
||||
|
||||
console.log(`GitHub release v${version} created successfully!`)
|
||||
|
||||
// GitHub will automatically handle the changelog
|
||||
console.log('GitHub release created with auto-generated notes')
|
||||
} catch (error) {
|
||||
// If the release already exists, this is not a fatal error
|
||||
if (error.message.includes('already exists')) {
|
||||
console.log(`GitHub release v${version} already exists, skipping creation.`)
|
||||
|
||||
// GitHub will automatically handle the changelog
|
||||
console.log('GitHub release already exists with auto-generated notes')
|
||||
} else {
|
||||
console.error('Error creating GitHub release:', error.message)
|
||||
// Don't exit with error to allow the npm publish to continue
|
||||
// process.exit(1)
|
||||
}
|
||||
}
|
||||
85
scripts/development/cli-wrapper.js
Executable file
85
scripts/development/cli-wrapper.js
Executable file
|
|
@ -0,0 +1,85 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* CLI Wrapper Script
|
||||
*
|
||||
* This script serves as a wrapper for the Brainy CLI, ensuring that command-line arguments
|
||||
* are properly passed to the CLI when invoked through npm scripts.
|
||||
*/
|
||||
|
||||
import { spawn, execSync } from 'child_process'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { dirname, join } from 'path'
|
||||
import fs from 'fs'
|
||||
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
// Path to the actual CLI script
|
||||
const cliPath = join(__dirname, 'dist', 'cli.js')
|
||||
|
||||
// Check if the CLI script exists
|
||||
if (!fs.existsSync(cliPath)) {
|
||||
// Check if we're running in a global installation context
|
||||
const isGlobalInstall = __dirname.includes('node_modules') && !__dirname.includes('node_modules/.')
|
||||
|
||||
if (isGlobalInstall) {
|
||||
console.error(`Error: CLI script not found at ${cliPath}`)
|
||||
console.error('This is likely because the CLI was not built during package installation.')
|
||||
console.error('Please reinstall the package with:')
|
||||
console.error('npm uninstall -g @soulcraft/brainy')
|
||||
console.error('npm install -g @soulcraft/brainy --legacy-peer-deps')
|
||||
process.exit(1)
|
||||
} else {
|
||||
// In a local development context, try to build the CLI
|
||||
console.log(`CLI script not found at ${cliPath}. Building CLI...`)
|
||||
|
||||
try {
|
||||
// Run the build:cli script
|
||||
execSync('npm run build:cli', { stdio: 'inherit' })
|
||||
|
||||
// Check again if the CLI script exists after building
|
||||
if (!fs.existsSync(cliPath)) {
|
||||
console.error(`Error: Failed to build CLI script at ${cliPath}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log('CLI built successfully.')
|
||||
} catch (error) {
|
||||
console.error(`Error building CLI: ${error.message}`)
|
||||
console.error('Make sure you have the necessary dependencies installed.')
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Special handling for version flags
|
||||
if (process.argv.includes('--version') || process.argv.includes('-V')) {
|
||||
// Read version directly from package.json to ensure it's always correct
|
||||
try {
|
||||
const packageJsonPath = join(__dirname, 'package.json')
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
|
||||
console.log(packageJson.version)
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error('Error loading version information:', error.message)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Forward all arguments to the CLI script
|
||||
const args = process.argv.slice(2)
|
||||
|
||||
// Check if npm is passing --force flag
|
||||
// When npm runs with --force, it sets the npm_config_force environment variable
|
||||
if (process.env.npm_config_force === 'true' && args.includes('clear') && !args.includes('--force') && !args.includes('-f')) {
|
||||
args.push('--force')
|
||||
}
|
||||
|
||||
const cli = spawn('node', [cliPath, ...args], { stdio: 'inherit' })
|
||||
|
||||
cli.on('close', (code) => {
|
||||
process.exit(code)
|
||||
})
|
||||
1
scripts/development/encoded-image.html
Normal file
1
scripts/development/encoded-image.html
Normal file
File diff suppressed because one or more lines are too long
17
scripts/development/index.html
Normal file
17
scripts/development/index.html
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Brainy Interactive Demo - Redirecting...</title>
|
||||
<link rel="icon" href="brainy.png" type="image/png">
|
||||
<meta http-equiv="refresh" content="0;url=demo/index.html">
|
||||
<script>
|
||||
window.location.href = 'demo/index.html'
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<p>Redirecting to <a href="demo/index.html">Brainy Interactive Demo</a>...</p>
|
||||
<p>If you are not redirected automatically, please click the link above.</p>
|
||||
</body>
|
||||
</html>
|
||||
78
scripts/development/test-browser-cache-detection.html
Normal file
78
scripts/development/test-browser-cache-detection.html
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Brainy Cache Detection Browser Test</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
#results {
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 5px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
.success {
|
||||
color: green;
|
||||
font-weight: bold;
|
||||
}
|
||||
.error {
|
||||
color: red;
|
||||
font-weight: bold;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Brainy Cache Detection Browser Test</h1>
|
||||
<p>This page tests if Brainy's cache detection works properly in browser environments.</p>
|
||||
|
||||
<button id="runTest">Run Test</button>
|
||||
|
||||
<div id="results">
|
||||
<p>Test results will appear here...</p>
|
||||
</div>
|
||||
|
||||
<script type="module">
|
||||
import { BrainyData } from './dist/unified.js';
|
||||
|
||||
document.getElementById('runTest').addEventListener('click', async () => {
|
||||
const resultsDiv = document.getElementById('results');
|
||||
resultsDiv.innerHTML = '<p>Running test...</p>';
|
||||
|
||||
try {
|
||||
console.log('Creating BrainyData instance...');
|
||||
resultsDiv.innerHTML += '<p>Creating BrainyData instance...</p>';
|
||||
|
||||
const brainy = new BrainyData();
|
||||
|
||||
console.log('BrainyData instance created successfully!');
|
||||
resultsDiv.innerHTML += '<p class="success">BrainyData instance created successfully!</p>';
|
||||
|
||||
// Initialize to make sure all components are created
|
||||
await brainy.init();
|
||||
|
||||
console.log('BrainyData initialized successfully!');
|
||||
resultsDiv.innerHTML += '<p class="success">BrainyData initialized successfully!</p>';
|
||||
|
||||
// Add a simple item to verify everything works
|
||||
const id = await brainy.add("Test item for browser cache detection");
|
||||
|
||||
console.log('Added test item with ID:', id);
|
||||
resultsDiv.innerHTML += `<p class="success">Added test item with ID: ${id}</p>`;
|
||||
|
||||
resultsDiv.innerHTML += '<p class="success">✅ Test completed successfully! Cache detection works in browser environment.</p>';
|
||||
} catch (error) {
|
||||
console.error('Error during cache detection test:', error);
|
||||
resultsDiv.innerHTML += `<p class="error">❌ Error during test: ${error.message}</p>`;
|
||||
resultsDiv.innerHTML += `<p class="error">Stack trace: ${error.stack}</p>`;
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
130
scripts/development/test-worker-cache-detection.html
Normal file
130
scripts/development/test-worker-cache-detection.html
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Brainy Cache Detection Worker Test</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
#results {
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 5px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
.success {
|
||||
color: green;
|
||||
font-weight: bold;
|
||||
}
|
||||
.error {
|
||||
color: red;
|
||||
font-weight: bold;
|
||||
}
|
||||
.log {
|
||||
color: #333;
|
||||
margin: 5px 0;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Brainy Cache Detection Worker Test</h1>
|
||||
<p>This page tests if Brainy's cache detection works properly in Web Worker environments.</p>
|
||||
|
||||
<button id="runTest">Run Test</button>
|
||||
|
||||
<div id="results">
|
||||
<p>Test results will appear here...</p>
|
||||
</div>
|
||||
|
||||
<script type="module">
|
||||
document.getElementById('runTest').addEventListener('click', () => {
|
||||
const resultsDiv = document.getElementById('results');
|
||||
resultsDiv.innerHTML = '<p>Starting worker test...</p>';
|
||||
|
||||
try {
|
||||
// Create a blob URL for the worker script
|
||||
const workerScript = `
|
||||
// Worker script for testing cache detection
|
||||
import { BrainyData } from './dist/unified.js';
|
||||
|
||||
// Listen for messages from the main thread
|
||||
self.onmessage = async (event) => {
|
||||
if (event.data === 'start-test') {
|
||||
try {
|
||||
self.postMessage({ type: 'log', message: 'Creating BrainyData instance...' });
|
||||
|
||||
const brainy = new BrainyData();
|
||||
|
||||
self.postMessage({ type: 'log', message: 'BrainyData instance created successfully!' });
|
||||
|
||||
// Initialize to make sure all components are created
|
||||
await brainy.init();
|
||||
|
||||
self.postMessage({ type: 'log', message: 'BrainyData initialized successfully!' });
|
||||
|
||||
// Add a simple item to verify everything works
|
||||
const id = await brainy.add("Test item for worker cache detection");
|
||||
|
||||
self.postMessage({ type: 'log', message: \`Added test item with ID: \${id}\` });
|
||||
|
||||
self.postMessage({
|
||||
type: 'success',
|
||||
message: 'Test completed successfully! Cache detection works in worker environment.'
|
||||
});
|
||||
} catch (error) {
|
||||
self.postMessage({
|
||||
type: 'error',
|
||||
message: \`Error during test: \${error.message}\`,
|
||||
stack: error.stack
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Notify that the worker is ready
|
||||
self.postMessage({ type: 'ready' });
|
||||
`;
|
||||
|
||||
const blob = new Blob([workerScript], { type: 'application/javascript' });
|
||||
const workerUrl = URL.createObjectURL(blob);
|
||||
|
||||
// Create the worker
|
||||
const worker = new Worker(workerUrl, { type: 'module' });
|
||||
|
||||
// Handle messages from the worker
|
||||
worker.onmessage = (event) => {
|
||||
const data = event.data;
|
||||
|
||||
if (data.type === 'ready') {
|
||||
resultsDiv.innerHTML += '<p class="log">Worker is ready. Starting test...</p>';
|
||||
worker.postMessage('start-test');
|
||||
} else if (data.type === 'log') {
|
||||
resultsDiv.innerHTML += `<p class="log">[Worker] ${data.message}</p>`;
|
||||
} else if (data.type === 'success') {
|
||||
resultsDiv.innerHTML += `<p class="success">✅ ${data.message}</p>`;
|
||||
} else if (data.type === 'error') {
|
||||
resultsDiv.innerHTML += `<p class="error">❌ ${data.message}</p>`;
|
||||
if (data.stack) {
|
||||
resultsDiv.innerHTML += `<p class="error">Stack trace: ${data.stack}</p>`;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Handle worker errors
|
||||
worker.onerror = (error) => {
|
||||
resultsDiv.innerHTML += `<p class="error">❌ Worker error: ${error.message}</p>`;
|
||||
};
|
||||
} catch (error) {
|
||||
resultsDiv.innerHTML += `<p class="error">❌ Error creating worker: ${error.message}</p>`;
|
||||
console.error('Error creating worker:', error);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
256
scripts/download-model.js
Normal file
256
scripts/download-model.js
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
/* eslint-env node */
|
||||
/* eslint-disable no-console, no-undef */
|
||||
|
||||
// Script to download the Universal Sentence Encoder model locally
|
||||
// This ensures the model is available in all environments without network dependencies
|
||||
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as tf from '@tensorflow/tfjs'
|
||||
import '@tensorflow/tfjs-backend-cpu'
|
||||
import * as use from '@tensorflow-models/universal-sentence-encoder'
|
||||
import { execSync } from 'child_process'
|
||||
|
||||
// Get the directory name in ESM
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
// Define model directories
|
||||
const MODEL_DIR = path.join(__dirname, '..', 'models')
|
||||
const USE_MODEL_DIR = path.join(MODEL_DIR, 'sentence-encoder')
|
||||
|
||||
// Create directories if they don't exist
|
||||
if (!fs.existsSync(MODEL_DIR)) {
|
||||
fs.mkdirSync(MODEL_DIR)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Created directory: ${MODEL_DIR}`)
|
||||
}
|
||||
|
||||
if (!fs.existsSync(USE_MODEL_DIR)) {
|
||||
fs.mkdirSync(USE_MODEL_DIR)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Created directory: ${USE_MODEL_DIR}`)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Starting Universal Sentence Encoder model setup...')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'This script will create reference files that point to the TensorFlow Hub model.'
|
||||
)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'NOTE: This does NOT download the full model locally. The full model (~25MB) will be downloaded'
|
||||
)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'automatically when your application first uses it, and then cached for future use.'
|
||||
)
|
||||
|
||||
async function downloadModel() {
|
||||
try {
|
||||
// Define modelMetadata at the top level so it's accessible throughout the function
|
||||
let modelMetadata = {
|
||||
name: 'universal-sentence-encoder',
|
||||
version: '1.0.0',
|
||||
description: 'Universal Sentence Encoder model for text embeddings',
|
||||
dimensions: 512,
|
||||
date: new Date().toISOString(),
|
||||
source: 'tensorflow-models/universal-sentence-encoder',
|
||||
savedLocally: true
|
||||
}
|
||||
|
||||
// Load the model - this will download it from TF Hub
|
||||
console.log('Loading Universal Sentence Encoder model...')
|
||||
const model = await use.load()
|
||||
console.log('Model loaded successfully!')
|
||||
|
||||
// Create a test sentence to ensure the model works
|
||||
console.log('Testing model with a sample sentence...')
|
||||
const singleEmbedding = await model.embed(['Hello world'])
|
||||
const singleEmbeddingArray = await singleEmbedding.array()
|
||||
console.log(`Test embedding dimensions: ${singleEmbeddingArray[0].length}`)
|
||||
singleEmbedding.dispose()
|
||||
|
||||
// Test the model with a few sentences to verify it works
|
||||
console.log('Testing model with sample sentences...')
|
||||
const testSentences = [
|
||||
'Hello world',
|
||||
'How are you doing today?',
|
||||
'Machine learning is fascinating'
|
||||
]
|
||||
|
||||
// Get embeddings for test sentences
|
||||
const batchEmbeddings = await model.embed(testSentences)
|
||||
const batchEmbeddingArrays = await batchEmbeddings.array()
|
||||
|
||||
// Log dimensions of each embedding
|
||||
for (let i = 0; i < testSentences.length; i++) {
|
||||
console.log(
|
||||
`Embedding ${i + 1} dimensions: ${batchEmbeddingArrays[i].length}`
|
||||
)
|
||||
}
|
||||
|
||||
// Clean up tensors
|
||||
batchEmbeddings.dispose()
|
||||
|
||||
// Since we can't directly save the model in this environment,
|
||||
// we'll download it from the TensorFlow Hub URL and save it manually
|
||||
console.log('Downloading model files from TensorFlow Hub...')
|
||||
|
||||
// Create a model.json file that includes information about the model
|
||||
// and points to the TensorFlow Hub URL
|
||||
const modelJson = {
|
||||
format: 'graph-model',
|
||||
generatedBy: 'TensorFlow.js v4.22.0',
|
||||
convertedBy: 'Brainy download-model script',
|
||||
modelTopology: {
|
||||
class_name: 'GraphModel',
|
||||
config: {
|
||||
name: 'universal-sentence-encoder'
|
||||
}
|
||||
},
|
||||
userDefinedMetadata: {
|
||||
signature: {
|
||||
inputs: {
|
||||
inputs: {
|
||||
name: 'inputs',
|
||||
dtype: 'string',
|
||||
shape: [-1]
|
||||
}
|
||||
},
|
||||
outputs: {
|
||||
outputs: {
|
||||
name: 'outputs',
|
||||
dtype: 'float32',
|
||||
shape: [-1, 512]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
weightsManifest: [
|
||||
{
|
||||
paths: ['group1-shard1of1.bin'],
|
||||
weights: [
|
||||
{
|
||||
name: 'embedding_matrix',
|
||||
shape: [512, 512],
|
||||
dtype: 'float32'
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
modelUrl:
|
||||
'https://tfhub.dev/tensorflow/tfjs-model/universal-sentence-encoder/1/default/1'
|
||||
}
|
||||
|
||||
// Write the model.json file
|
||||
fs.writeFileSync(
|
||||
path.join(USE_MODEL_DIR, 'model.json'),
|
||||
JSON.stringify(modelJson, null, 2)
|
||||
)
|
||||
|
||||
// Generate a sample embedding and save it as the weights file
|
||||
// This will be a real embedding, not just zeros
|
||||
console.log('Generating sample embedding for weights file...')
|
||||
const sampleEmbedding = await model.embed([
|
||||
'This is a sample sentence for the Universal Sentence Encoder model.'
|
||||
])
|
||||
const sampleEmbeddingArray = await sampleEmbedding.array()
|
||||
|
||||
// Create a Float32Array from the embedding
|
||||
const embeddingData = new Float32Array(sampleEmbeddingArray[0])
|
||||
|
||||
// Write the embedding data to the weights file
|
||||
fs.writeFileSync(
|
||||
path.join(USE_MODEL_DIR, 'group1-shard1of1.bin'),
|
||||
Buffer.from(embeddingData.buffer)
|
||||
)
|
||||
|
||||
console.log('Sample embedding saved as weights file')
|
||||
sampleEmbedding.dispose()
|
||||
|
||||
// Update metadata
|
||||
modelMetadata.savedWith = 'manual-embedding'
|
||||
modelMetadata.embeddingSize = embeddingData.length
|
||||
|
||||
console.log(`Model files created in ${USE_MODEL_DIR}`)
|
||||
console.log(
|
||||
`The model.json file points to the TensorFlow Hub URL for the actual model`
|
||||
)
|
||||
console.log(
|
||||
`The weights file contains a real sample embedding of size ${embeddingData.length}`
|
||||
)
|
||||
|
||||
// Add instructions for users
|
||||
console.log(
|
||||
'\nIMPORTANT: This setup uses the TensorFlow Hub URL for the model.'
|
||||
)
|
||||
console.log(
|
||||
'The first time the model is used, it will download the full model from TensorFlow Hub.'
|
||||
)
|
||||
console.log('Subsequent uses will use the cached model.')
|
||||
|
||||
// Update metadata to indicate the approach used
|
||||
modelMetadata.approach = 'tfhub-reference'
|
||||
|
||||
// Write metadata file
|
||||
fs.writeFileSync(
|
||||
path.join(USE_MODEL_DIR, 'metadata.json'),
|
||||
JSON.stringify(modelMetadata, null, 2)
|
||||
)
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('✅ Model saved successfully!')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Model is now available at: ${USE_MODEL_DIR}`)
|
||||
|
||||
// Verify the model files exist
|
||||
const modelJsonPath = path.join(USE_MODEL_DIR, 'model.json')
|
||||
if (fs.existsSync(modelJsonPath)) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('✅ model.json file verified')
|
||||
|
||||
// List the shard files
|
||||
const modelFiles = fs.readdirSync(USE_MODEL_DIR)
|
||||
const shardFiles = modelFiles.filter((file) => file.endsWith('.bin'))
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Found ${shardFiles.length} model shard files:`)
|
||||
// eslint-disable-next-line no-console
|
||||
shardFiles.forEach((file) => console.log(` - ${file}`))
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\nModel reference files are ready!')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'IMPORTANT: These are NOT the full model files (~25MB), but reference files (~3KB total).'
|
||||
)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'The full model will be downloaded automatically when your application first uses it.'
|
||||
)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'After the first use, the model will be cached locally for future use.'
|
||||
)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'These reference files should be checked into version control to ensure availability in all environments.'
|
||||
)
|
||||
} else {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error('❌ model.json file not found after saving!')
|
||||
// eslint-disable-next-line no-undef
|
||||
process.exit(1)
|
||||
}
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error('❌ Error downloading model:', error)
|
||||
// eslint-disable-next-line no-undef
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
downloadModel().catch(console.error)
|
||||
190
scripts/download-models.cjs
Executable file
190
scripts/download-models.cjs
Executable file
|
|
@ -0,0 +1,190 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Download and bundle models for offline usage
|
||||
*/
|
||||
|
||||
const fs = require('fs').promises
|
||||
const path = require('path')
|
||||
|
||||
const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'
|
||||
const OUTPUT_DIR = './models'
|
||||
|
||||
async function downloadModels() {
|
||||
// Use dynamic import for ES modules in CommonJS
|
||||
const { pipeline, env } = await import('@huggingface/transformers')
|
||||
|
||||
// Configure transformers.js to use local cache
|
||||
env.cacheDir = './models-cache'
|
||||
env.allowRemoteModels = true
|
||||
try {
|
||||
console.log('🔄 Downloading all-MiniLM-L6-v2 model for offline bundling...')
|
||||
console.log(` Model: ${MODEL_NAME}`)
|
||||
console.log(` Cache: ${env.cacheDir}`)
|
||||
|
||||
// Create output directory
|
||||
await fs.mkdir(OUTPUT_DIR, { recursive: true })
|
||||
|
||||
// Load the model to force download
|
||||
console.log('📥 Loading model pipeline...')
|
||||
const extractor = await pipeline('feature-extraction', MODEL_NAME)
|
||||
|
||||
// Test the model to make sure it works
|
||||
console.log('🧪 Testing model...')
|
||||
const testResult = await extractor(['Hello world!'], {
|
||||
pooling: 'mean',
|
||||
normalize: true
|
||||
})
|
||||
|
||||
console.log(`✅ Model test successful! Embedding dimensions: ${testResult.data.length}`)
|
||||
|
||||
// Copy ALL model files from cache to our models directory
|
||||
console.log('📋 Copying ALL model files to bundle directory...')
|
||||
|
||||
const cacheDir = path.resolve(env.cacheDir)
|
||||
const outputDir = path.resolve(OUTPUT_DIR)
|
||||
|
||||
console.log(` From: ${cacheDir}`)
|
||||
console.log(` To: ${outputDir}`)
|
||||
|
||||
// Copy the entire cache directory structure to ensure we get ALL files
|
||||
// including tokenizer.json, config.json, and all ONNX model files
|
||||
const modelCacheDir = path.join(cacheDir, 'Xenova', 'all-MiniLM-L6-v2')
|
||||
|
||||
if (await dirExists(modelCacheDir)) {
|
||||
const targetModelDir = path.join(outputDir, 'Xenova', 'all-MiniLM-L6-v2')
|
||||
console.log(` Copying complete model: Xenova/all-MiniLM-L6-v2`)
|
||||
await copyDirectory(modelCacheDir, targetModelDir)
|
||||
} else {
|
||||
throw new Error(`Model cache directory not found: ${modelCacheDir}`)
|
||||
}
|
||||
|
||||
console.log('✅ Model bundling complete!')
|
||||
console.log(` Total size: ${await calculateDirectorySize(outputDir)} MB`)
|
||||
console.log(` Location: ${outputDir}`)
|
||||
|
||||
// Create a marker file
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, '.brainy-models-bundled'),
|
||||
JSON.stringify({
|
||||
model: MODEL_NAME,
|
||||
bundledAt: new Date().toISOString(),
|
||||
version: '1.0.0'
|
||||
}, null, 2)
|
||||
)
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error downloading models:', error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
async function findModelDirectories(baseDir, modelName) {
|
||||
const dirs = []
|
||||
|
||||
try {
|
||||
// Convert model name to expected directory structure
|
||||
const modelPath = modelName.replace('/', '--')
|
||||
|
||||
async function searchDirectory(currentDir) {
|
||||
try {
|
||||
const entries = await fs.readdir(currentDir, { withFileTypes: true })
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const fullPath = path.join(currentDir, entry.name)
|
||||
|
||||
// Check if this directory contains model files
|
||||
if (entry.name.includes(modelPath) || entry.name === 'onnx') {
|
||||
const hasModelFiles = await containsModelFiles(fullPath)
|
||||
if (hasModelFiles) {
|
||||
dirs.push(fullPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Recursively search subdirectories
|
||||
await searchDirectory(fullPath)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore access errors
|
||||
}
|
||||
}
|
||||
|
||||
await searchDirectory(baseDir)
|
||||
} catch (error) {
|
||||
console.warn('Warning: Error searching for model directories:', error)
|
||||
}
|
||||
|
||||
return dirs
|
||||
}
|
||||
|
||||
async function containsModelFiles(dir) {
|
||||
try {
|
||||
const files = await fs.readdir(dir)
|
||||
return files.some(file =>
|
||||
file.endsWith('.onnx') ||
|
||||
file.endsWith('.json') ||
|
||||
file === 'config.json' ||
|
||||
file === 'tokenizer.json'
|
||||
)
|
||||
} catch (error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function dirExists(dir) {
|
||||
try {
|
||||
const stats = await fs.stat(dir)
|
||||
return stats.isDirectory()
|
||||
} catch (error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function copyDirectory(src, dest) {
|
||||
await fs.mkdir(dest, { recursive: true })
|
||||
const entries = await fs.readdir(src, { withFileTypes: true })
|
||||
|
||||
for (const entry of entries) {
|
||||
const srcPath = path.join(src, entry.name)
|
||||
const destPath = path.join(dest, entry.name)
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await copyDirectory(srcPath, destPath)
|
||||
} else {
|
||||
await fs.copyFile(srcPath, destPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function calculateDirectorySize(dir) {
|
||||
let size = 0
|
||||
|
||||
async function calculateSize(currentDir) {
|
||||
try {
|
||||
const entries = await fs.readdir(currentDir, { withFileTypes: true })
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(currentDir, entry.name)
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await calculateSize(fullPath)
|
||||
} else {
|
||||
const stats = await fs.stat(fullPath)
|
||||
size += stats.size
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore access errors
|
||||
}
|
||||
}
|
||||
|
||||
await calculateSize(dir)
|
||||
return Math.round(size / (1024 * 1024))
|
||||
}
|
||||
|
||||
// Run the download
|
||||
downloadModels().catch(error => {
|
||||
console.error('Fatal error:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
47
scripts/encode-image.js
Normal file
47
scripts/encode-image.js
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Path to the image file
|
||||
const imagePath = path.join(__dirname, '..', 'brainy.png');
|
||||
|
||||
// Read the image file
|
||||
const imageBuffer = fs.readFileSync(imagePath);
|
||||
|
||||
// Convert the image to base64
|
||||
const base64Image = imageBuffer.toString('base64');
|
||||
|
||||
// Get the MIME type based on file extension
|
||||
const getMimeType = (filePath) => {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
switch (ext) {
|
||||
case '.png':
|
||||
return 'image/png';
|
||||
case '.jpg':
|
||||
case '.jpeg':
|
||||
return 'image/jpeg';
|
||||
case '.gif':
|
||||
return 'image/gif';
|
||||
case '.svg':
|
||||
return 'image/svg+xml';
|
||||
default:
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
};
|
||||
|
||||
const mimeType = getMimeType(imagePath);
|
||||
|
||||
// Create the data URL
|
||||
const dataUrl = `data:${mimeType};base64,${base64Image}`;
|
||||
|
||||
// Output the complete HTML img tag
|
||||
const imgTag = `<img src="${dataUrl}" alt="Brainy Logo" width="200"/>`;
|
||||
|
||||
// Write to a file instead of console.log to avoid truncation
|
||||
fs.writeFileSync(path.join(__dirname, '..', 'encoded-image.html'), imgTag);
|
||||
|
||||
console.log('Base64 encoded image has been saved to encoded-image.html');
|
||||
201
scripts/extract-models.js
Normal file
201
scripts/extract-models.js
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Extract Brainy Models Script
|
||||
*
|
||||
* Automatically extracts models from @soulcraft/brainy-models during Docker builds
|
||||
* Works across all cloud providers (Google Cloud Run, AWS Lambda/ECS, Azure Container Instances, Cloudflare Workers)
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, cpSync, readFileSync, writeFileSync } from 'fs'
|
||||
import { join, dirname } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
function log(message) {
|
||||
console.log(`[Brainy Model Extractor] ${message}`)
|
||||
}
|
||||
|
||||
async function extractModels() {
|
||||
try {
|
||||
log('🔍 Checking for @soulcraft/brainy-models...')
|
||||
|
||||
// Get the project root (one level up from scripts/)
|
||||
const projectRoot = join(__dirname, '..')
|
||||
const modelsPackagePath = join(projectRoot, 'node_modules', '@soulcraft', 'brainy-models')
|
||||
|
||||
if (!existsSync(modelsPackagePath)) {
|
||||
log('⚠️ @soulcraft/brainy-models not found - skipping model extraction')
|
||||
log(' Models will be downloaded at runtime (slower startup)')
|
||||
return false
|
||||
}
|
||||
|
||||
log('✅ Found @soulcraft/brainy-models package')
|
||||
|
||||
// Create the models directory in the project root
|
||||
const targetModelsDir = join(projectRoot, 'models')
|
||||
|
||||
if (existsSync(targetModelsDir)) {
|
||||
log('📁 Models directory already exists - removing old version')
|
||||
// Remove existing models directory to ensure clean extraction
|
||||
try {
|
||||
import('fs').then(fs => {
|
||||
fs.rmSync(targetModelsDir, { recursive: true, force: true })
|
||||
})
|
||||
} catch (error) {
|
||||
log(`⚠️ Could not remove existing models directory: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
log('📦 Creating models directory...')
|
||||
mkdirSync(targetModelsDir, { recursive: true })
|
||||
|
||||
// Look for models in the package
|
||||
const possibleModelsPaths = [
|
||||
join(modelsPackagePath, 'models'),
|
||||
join(modelsPackagePath, 'dist', 'models'),
|
||||
modelsPackagePath // Root of the package
|
||||
]
|
||||
|
||||
let modelsSourcePath = null
|
||||
for (const path of possibleModelsPaths) {
|
||||
if (existsSync(path)) {
|
||||
// Check if this directory contains model files
|
||||
try {
|
||||
const fs = await import('fs')
|
||||
const files = fs.readdirSync(path)
|
||||
if (files.length > 0) {
|
||||
modelsSourcePath = path
|
||||
break
|
||||
}
|
||||
} catch (error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!modelsSourcePath) {
|
||||
log('❌ Could not find models in @soulcraft/brainy-models package')
|
||||
return false
|
||||
}
|
||||
|
||||
log(`📋 Copying models from: ${modelsSourcePath}`)
|
||||
log(`📋 Copying models to: ${targetModelsDir}`)
|
||||
|
||||
// Copy all models
|
||||
try {
|
||||
cpSync(modelsSourcePath, targetModelsDir, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
filter: (src, dest) => {
|
||||
// Skip node_modules and other unnecessary files
|
||||
const filename = src.split('/').pop() || ''
|
||||
return !filename.startsWith('.') && filename !== 'node_modules'
|
||||
}
|
||||
})
|
||||
|
||||
log('✅ Models extracted successfully!')
|
||||
|
||||
// Create a marker file to indicate successful extraction
|
||||
const markerFile = join(targetModelsDir, '.brainy-models-extracted')
|
||||
writeFileSync(markerFile, JSON.stringify({
|
||||
extractedAt: new Date().toISOString(),
|
||||
sourcePackage: '@soulcraft/brainy-models',
|
||||
extractorVersion: '1.0.0'
|
||||
}, null, 2))
|
||||
|
||||
// List extracted models
|
||||
try {
|
||||
const fs = await import('fs')
|
||||
const extractedItems = fs.readdirSync(targetModelsDir)
|
||||
log(`📊 Extracted items: ${extractedItems.join(', ')}`)
|
||||
} catch (error) {
|
||||
log('📊 Model extraction completed (could not list contents)')
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Failed to copy models: ${error.message}`)
|
||||
return false
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Model extraction failed: ${error.message}`)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-detect environment and provide helpful information
|
||||
function detectEnvironment() {
|
||||
const envs = []
|
||||
|
||||
// Docker detection
|
||||
if (existsSync('/.dockerenv') || process.env.DOCKER_CONTAINER) {
|
||||
envs.push('Docker')
|
||||
}
|
||||
|
||||
// Cloud provider detection
|
||||
if (process.env.GOOGLE_CLOUD_PROJECT || process.env.GAE_SERVICE) {
|
||||
envs.push('Google Cloud')
|
||||
}
|
||||
|
||||
if (process.env.AWS_EXECUTION_ENV || process.env.AWS_LAMBDA_FUNCTION_NAME) {
|
||||
envs.push('AWS')
|
||||
}
|
||||
|
||||
if (process.env.AZURE_CLIENT_ID || process.env.WEBSITE_SITE_NAME) {
|
||||
envs.push('Azure')
|
||||
}
|
||||
|
||||
if (process.env.CF_PAGES || process.env.CLOUDFLARE_ACCOUNT_ID) {
|
||||
envs.push('Cloudflare')
|
||||
}
|
||||
|
||||
if (process.env.VERCEL || process.env.VERCEL_ENV) {
|
||||
envs.push('Vercel')
|
||||
}
|
||||
|
||||
if (process.env.NETLIFY || process.env.NETLIFY_BUILD_BASE) {
|
||||
envs.push('Netlify')
|
||||
}
|
||||
|
||||
return envs
|
||||
}
|
||||
|
||||
// Main execution
|
||||
async function main() {
|
||||
log('🚀 Starting Brainy model extraction...')
|
||||
|
||||
const detectedEnvs = detectEnvironment()
|
||||
if (detectedEnvs.length > 0) {
|
||||
log(`🌐 Detected environment(s): ${detectedEnvs.join(', ')}`)
|
||||
}
|
||||
|
||||
const success = await extractModels()
|
||||
|
||||
if (success) {
|
||||
log('🎉 Model extraction completed successfully!')
|
||||
log('💡 Models are now embedded in your container/deployment')
|
||||
log('💡 No runtime model downloads required!')
|
||||
|
||||
// Set environment variable hint for runtime
|
||||
log('💡 Runtime will automatically detect extracted models')
|
||||
} else {
|
||||
log('⚠️ Model extraction failed or skipped')
|
||||
log('💡 Application will fall back to runtime model downloads')
|
||||
log('💡 Consider installing @soulcraft/brainy-models for better performance')
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(error => {
|
||||
console.error('Fatal error:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
|
||||
export { extractModels, detectEnvironment }
|
||||
83
scripts/maintenance/check-database.js
Normal file
83
scripts/maintenance/check-database.js
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// Script to check if there's any data in the database
|
||||
import { BrainyData } from './dist/brainyData.js';
|
||||
|
||||
async function checkDatabase() {
|
||||
try {
|
||||
console.log('Initializing BrainyData...');
|
||||
const db = new BrainyData();
|
||||
await db.init();
|
||||
|
||||
console.log('Getting database status...');
|
||||
const status = await db.status();
|
||||
console.log('Database status:', JSON.stringify(status, null, 2));
|
||||
|
||||
console.log('Getting statistics...');
|
||||
const stats = await db.getStatistics();
|
||||
console.log('Statistics:', JSON.stringify(stats, null, 2));
|
||||
|
||||
console.log('Getting all nouns...');
|
||||
const nouns = await db.getAllNouns();
|
||||
console.log(`Found ${nouns.length} nouns in the database.`);
|
||||
|
||||
if (nouns.length > 0) {
|
||||
console.log('Sample of nouns:');
|
||||
for (let i = 0; i < Math.min(5, nouns.length); i++) {
|
||||
console.log(`Noun ${i + 1}:`, JSON.stringify(nouns[i], null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Getting all verbs...');
|
||||
const verbs = await db.getAllVerbs();
|
||||
console.log(`Found ${verbs.length} verbs in the database.`);
|
||||
|
||||
if (verbs.length > 0) {
|
||||
console.log('Sample of verbs:');
|
||||
for (let i = 0; i < Math.min(5, verbs.length); i++) {
|
||||
console.log(`Verb ${i + 1}:`, JSON.stringify(verbs[i], null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// Try a simple search to see if it returns any results
|
||||
console.log('Trying a simple search...');
|
||||
const searchResults = await db.searchText('test', 10);
|
||||
console.log(`Search returned ${searchResults.length} results.`);
|
||||
|
||||
if (searchResults.length > 0) {
|
||||
console.log('Sample of search results:');
|
||||
for (let i = 0; i < Math.min(5, searchResults.length); i++) {
|
||||
console.log(`Result ${i + 1}:`, JSON.stringify({
|
||||
id: searchResults[i].id,
|
||||
score: searchResults[i].score,
|
||||
metadata: searchResults[i].metadata
|
||||
}, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// If no results, try adding a test item and searching again
|
||||
if (searchResults.length === 0 && nouns.length === 0) {
|
||||
console.log('No data found. Adding a test item...');
|
||||
const id = await db.add('This is a test item for searching', { noun: 'Thing', category: 'test' });
|
||||
console.log(`Added test item with ID: ${id}`);
|
||||
|
||||
console.log('Trying search again...');
|
||||
const newSearchResults = await db.searchText('test', 10);
|
||||
console.log(`Search returned ${newSearchResults.length} results.`);
|
||||
|
||||
if (newSearchResults.length > 0) {
|
||||
console.log('Sample of search results:');
|
||||
for (let i = 0; i < Math.min(5, newSearchResults.length); i++) {
|
||||
console.log(`Result ${i + 1}:`, JSON.stringify({
|
||||
id: newSearchResults[i].id,
|
||||
score: newSearchResults[i].score,
|
||||
metadata: newSearchResults[i].metadata
|
||||
}, null, 2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error checking database:', error);
|
||||
}
|
||||
}
|
||||
|
||||
checkDatabase().catch(console.error);
|
||||
163
scripts/maintenance/fix-dimension-mismatch.js
Normal file
163
scripts/maintenance/fix-dimension-mismatch.js
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
// Script to fix dimension mismatch by re-embedding existing data
|
||||
import { BrainyData } from './dist/brainyData.js';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
async function fixDimensionMismatch() {
|
||||
try {
|
||||
console.log('Starting dimension mismatch fix...');
|
||||
|
||||
// Create a backup of the existing data
|
||||
const backupDir = './brainy-data-backup-' + Date.now();
|
||||
console.log(`Creating backup of existing data in ${backupDir}...`);
|
||||
|
||||
// Copy the entire brainy-data directory to the backup directory
|
||||
await fs.promises.mkdir(backupDir, { recursive: true });
|
||||
await copyDirectory('./brainy-data', backupDir);
|
||||
console.log('Backup created successfully.');
|
||||
|
||||
// Initialize BrainyData with the current embedding function
|
||||
console.log('Initializing BrainyData...');
|
||||
const db = new BrainyData();
|
||||
await db.init();
|
||||
|
||||
// Get database status to check if there's any data
|
||||
const status = await db.status();
|
||||
console.log('Database status:', JSON.stringify(status, null, 2));
|
||||
|
||||
// Read all noun files directly from the filesystem
|
||||
console.log('Reading noun files directly from filesystem...');
|
||||
const nounsDir = './brainy-data/nouns';
|
||||
const files = await fs.promises.readdir(nounsDir);
|
||||
|
||||
// Process each noun file
|
||||
const processedNouns = [];
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(nounsDir, file);
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8');
|
||||
const parsedNoun = JSON.parse(data);
|
||||
|
||||
// Get the metadata for this noun
|
||||
const metadataPath = path.join('./brainy-data/metadata', `${parsedNoun.id}.json`);
|
||||
let metadata = {};
|
||||
try {
|
||||
const metadataData = await fs.promises.readFile(metadataPath, 'utf-8');
|
||||
metadata = JSON.parse(metadataData);
|
||||
} catch (error) {
|
||||
console.warn(`No metadata found for noun ${parsedNoun.id}`);
|
||||
}
|
||||
|
||||
// Extract text from metadata if available
|
||||
let text = '';
|
||||
if (metadata.text) {
|
||||
text = metadata.text;
|
||||
} else if (metadata.description) {
|
||||
text = metadata.description;
|
||||
} else {
|
||||
// If no text is available, use a placeholder
|
||||
text = `Noun ${parsedNoun.id}`;
|
||||
console.warn(`No text found for noun ${parsedNoun.id}, using placeholder`);
|
||||
}
|
||||
|
||||
// Re-embed the text using the current embedding function
|
||||
console.log(`Re-embedding noun ${parsedNoun.id}...`);
|
||||
try {
|
||||
// Delete the existing noun first
|
||||
await db.delete(parsedNoun.id);
|
||||
|
||||
// Add the noun with the same ID but new vector
|
||||
const newId = await db.add(text, metadata, { id: parsedNoun.id });
|
||||
processedNouns.push({ id: newId, originalId: parsedNoun.id });
|
||||
console.log(`Successfully re-embedded noun ${parsedNoun.id}`);
|
||||
} catch (error) {
|
||||
console.error(`Error re-embedding noun ${parsedNoun.id}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Processed ${processedNouns.length} nouns.`);
|
||||
|
||||
// Recreate verbs
|
||||
console.log('Reading verb files directly from filesystem...');
|
||||
const verbsDir = './brainy-data/verbs';
|
||||
const verbFiles = await fs.promises.readdir(verbsDir);
|
||||
|
||||
// Process each verb file
|
||||
const processedVerbs = [];
|
||||
for (const file of verbFiles) {
|
||||
if (file.endsWith('.json')) {
|
||||
const filePath = path.join(verbsDir, file);
|
||||
const data = await fs.promises.readFile(filePath, 'utf-8');
|
||||
const parsedVerb = JSON.parse(data);
|
||||
|
||||
// Check if both source and target nouns exist
|
||||
const sourceExists = processedNouns.some(n => n.originalId === parsedVerb.sourceId);
|
||||
const targetExists = processedNouns.some(n => n.originalId === parsedVerb.targetId);
|
||||
|
||||
if (sourceExists && targetExists) {
|
||||
console.log(`Re-creating verb ${parsedVerb.id} between ${parsedVerb.sourceId} and ${parsedVerb.targetId}...`);
|
||||
try {
|
||||
// Delete the existing verb first
|
||||
await db.deleteVerb(parsedVerb.id);
|
||||
|
||||
// Add the verb with the same relationship
|
||||
await db.addVerb(parsedVerb.sourceId, parsedVerb.targetId, {
|
||||
verb: parsedVerb.type || 'RelatedTo',
|
||||
...parsedVerb.metadata
|
||||
});
|
||||
|
||||
processedVerbs.push(parsedVerb.id);
|
||||
console.log(`Successfully re-created verb ${parsedVerb.id}`);
|
||||
} catch (error) {
|
||||
console.error(`Error re-creating verb ${parsedVerb.id}:`, error);
|
||||
}
|
||||
} else {
|
||||
console.warn(`Skipping verb ${parsedVerb.id} because source or target noun doesn't exist`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Processed ${processedVerbs.length} verbs.`);
|
||||
|
||||
// Try a search to verify it works
|
||||
console.log('Trying a search to verify it works...');
|
||||
const searchResults = await db.searchText('test', 10);
|
||||
console.log(`Search returned ${searchResults.length} results.`);
|
||||
|
||||
if (searchResults.length > 0) {
|
||||
console.log('Sample of search results:');
|
||||
for (let i = 0; i < Math.min(5, searchResults.length); i++) {
|
||||
console.log(`Result ${i + 1}:`, JSON.stringify({
|
||||
id: searchResults[i].id,
|
||||
score: searchResults[i].score,
|
||||
metadata: searchResults[i].metadata
|
||||
}, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Dimension mismatch fix completed successfully.');
|
||||
} catch (error) {
|
||||
console.error('Error fixing dimension mismatch:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to copy a directory recursively
|
||||
async function copyDirectory(source, destination) {
|
||||
const entries = await fs.promises.readdir(source, { withFileTypes: true });
|
||||
|
||||
await fs.promises.mkdir(destination, { recursive: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const srcPath = path.join(source, entry.name);
|
||||
const destPath = path.join(destination, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await copyDirectory(srcPath, destPath);
|
||||
} else {
|
||||
await fs.promises.copyFile(srcPath, destPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fixDimensionMismatch().catch(console.error);
|
||||
141
scripts/patch-textencoder.js
Executable file
141
scripts/patch-textencoder.js
Executable file
|
|
@ -0,0 +1,141 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Simplified TextEncoder Patch
|
||||
*
|
||||
* This script patches the compiled unified.js file to fix the TextEncoder issue in Node.js
|
||||
* by replacing references to this.util.TextEncoder with direct TextEncoder usage.
|
||||
*/
|
||||
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
// Path to the compiled unified.js file
|
||||
const unifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.js')
|
||||
|
||||
// Read the file
|
||||
console.log(`Reading ${unifiedJsPath}...`)
|
||||
let content = fs.readFileSync(unifiedJsPath, 'utf8')
|
||||
|
||||
// Simple replacement: replace all instances of new this.util.TextEncoder() with new TextEncoder()
|
||||
const pattern = /new\s+this\.util\.TextEncoder\(\)/g
|
||||
const replacement = 'new TextEncoder()'
|
||||
|
||||
// Apply the patch
|
||||
const patchedContent = content.replace(pattern, replacement)
|
||||
|
||||
// Check if the patch was applied
|
||||
if (patchedContent === content) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextEncoder()" found in the file.'
|
||||
)
|
||||
} else {
|
||||
// Write the patched file
|
||||
console.log('Writing patched file...')
|
||||
fs.writeFileSync(unifiedJsPath, patchedContent, 'utf8')
|
||||
console.log('Patch applied successfully!')
|
||||
}
|
||||
|
||||
// Also patch the minified version if it exists
|
||||
const minifiedJsPath = path.join(__dirname, '..', 'dist', 'unified.min.js')
|
||||
if (fs.existsSync(minifiedJsPath)) {
|
||||
console.log(`Reading ${minifiedJsPath}...`)
|
||||
const minContent = fs.readFileSync(minifiedJsPath, 'utf8')
|
||||
|
||||
// Apply the same replacement to the minified file
|
||||
const patchedMinContent = minContent.replace(pattern, replacement)
|
||||
|
||||
// Check if the patch was applied
|
||||
if (patchedMinContent === minContent) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextEncoder()" found in the minified file.'
|
||||
)
|
||||
} else {
|
||||
// Write the patched file
|
||||
console.log('Writing patched minified file...')
|
||||
fs.writeFileSync(minifiedJsPath, patchedMinContent, 'utf8')
|
||||
console.log('Minified file patched successfully!')
|
||||
}
|
||||
}
|
||||
|
||||
// Also patch the worker.js file if it exists
|
||||
const workerJsPath = path.join(__dirname, '..', 'dist', 'worker.js')
|
||||
if (fs.existsSync(workerJsPath)) {
|
||||
console.log(`Reading ${workerJsPath}...`)
|
||||
const workerContent = fs.readFileSync(workerJsPath, 'utf8')
|
||||
|
||||
// Apply the same replacement to the worker file
|
||||
const patchedWorkerContent = workerContent.replace(pattern, replacement)
|
||||
|
||||
// Check if the patch was applied
|
||||
if (patchedWorkerContent === workerContent) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextEncoder()" found in the worker file.'
|
||||
)
|
||||
} else {
|
||||
// Write the patched file
|
||||
console.log('Writing patched worker file...')
|
||||
fs.writeFileSync(workerJsPath, patchedWorkerContent, 'utf8')
|
||||
console.log('Worker file patched successfully!')
|
||||
}
|
||||
}
|
||||
|
||||
// Also patch TextDecoder
|
||||
console.log('Patching TextDecoder references...')
|
||||
content = fs.readFileSync(unifiedJsPath, 'utf8')
|
||||
const decoderPattern = /new\s+this\.util\.TextDecoder\(\)/g
|
||||
const decoderReplacement = 'new TextDecoder()'
|
||||
const patchedDecoderContent = content.replace(
|
||||
decoderPattern,
|
||||
decoderReplacement
|
||||
)
|
||||
|
||||
if (patchedDecoderContent === content) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextDecoder()" found in the file.'
|
||||
)
|
||||
} else {
|
||||
fs.writeFileSync(unifiedJsPath, patchedDecoderContent, 'utf8')
|
||||
console.log('TextDecoder patch applied successfully!')
|
||||
}
|
||||
|
||||
// Patch the minified file for TextDecoder as well
|
||||
if (fs.existsSync(minifiedJsPath)) {
|
||||
const minContent = fs.readFileSync(minifiedJsPath, 'utf8')
|
||||
const patchedMinDecoderContent = minContent.replace(
|
||||
decoderPattern,
|
||||
decoderReplacement
|
||||
)
|
||||
|
||||
if (patchedMinDecoderContent === minContent) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextDecoder()" found in the minified file.'
|
||||
)
|
||||
} else {
|
||||
fs.writeFileSync(minifiedJsPath, patchedMinDecoderContent, 'utf8')
|
||||
console.log('TextDecoder patch applied to minified file successfully!')
|
||||
}
|
||||
}
|
||||
|
||||
// Patch the worker.js file for TextDecoder as well
|
||||
if (fs.existsSync(workerJsPath)) {
|
||||
const workerContent = fs.readFileSync(workerJsPath, 'utf8')
|
||||
const patchedWorkerDecoderContent = workerContent.replace(
|
||||
decoderPattern,
|
||||
decoderReplacement
|
||||
)
|
||||
|
||||
if (patchedWorkerDecoderContent === workerContent) {
|
||||
console.warn(
|
||||
'No instances of "new this.util.TextDecoder()" found in the worker file.'
|
||||
)
|
||||
} else {
|
||||
fs.writeFileSync(workerJsPath, patchedWorkerDecoderContent, 'utf8')
|
||||
console.log('TextDecoder patch applied to worker file successfully!')
|
||||
}
|
||||
}
|
||||
148
scripts/production-readiness-check.js
Normal file
148
scripts/production-readiness-check.js
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Comprehensive production readiness check for Brainy
|
||||
* Tests all deployment scenarios to prevent emergency releases
|
||||
*/
|
||||
|
||||
import { BrainyData, NounType, VerbType } from '../dist/index.js'
|
||||
import fs from 'fs'
|
||||
import { execSync } from 'child_process'
|
||||
|
||||
async function cleanEnvironment() {
|
||||
// Clean up any existing models or cache
|
||||
try {
|
||||
if (fs.existsSync('./models')) {
|
||||
console.log('🧹 Cleaning existing models directory...')
|
||||
fs.rmSync('./models', { recursive: true, force: true })
|
||||
}
|
||||
if (fs.existsSync('./brainy-data')) {
|
||||
console.log('🧹 Cleaning existing data directory...')
|
||||
fs.rmSync('./brainy-data', { recursive: true, force: true })
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Note: Some cleanup operations failed (this may be normal)')
|
||||
}
|
||||
}
|
||||
|
||||
async function testScenario(name, envVars, expectedResult) {
|
||||
console.log(`\n🧪 TESTING: ${name}`)
|
||||
console.log(`Environment: ${JSON.stringify(envVars)}`)
|
||||
|
||||
// Set environment variables
|
||||
for (const [key, value] of Object.entries(envVars)) {
|
||||
process.env[key] = value
|
||||
}
|
||||
|
||||
try {
|
||||
const brain = new BrainyData()
|
||||
await brain.init()
|
||||
|
||||
// Test basic operations
|
||||
const id = await brain.add("Test data for production scenario")
|
||||
const results = await brain.search("test", 1)
|
||||
|
||||
if (expectedResult === 'success') {
|
||||
console.log(`✅ SUCCESS: ${name} - Operations completed successfully`)
|
||||
return true
|
||||
} else {
|
||||
console.log(`❌ UNEXPECTED SUCCESS: ${name} - Expected failure but got success`)
|
||||
return false
|
||||
}
|
||||
} catch (error) {
|
||||
if (expectedResult === 'failure') {
|
||||
console.log(`✅ EXPECTED FAILURE: ${name} - ${error.message}`)
|
||||
return true
|
||||
} else {
|
||||
console.log(`❌ UNEXPECTED FAILURE: ${name} - ${error.message}`)
|
||||
return false
|
||||
}
|
||||
} finally {
|
||||
// Clean environment variables
|
||||
for (const key of Object.keys(envVars)) {
|
||||
delete process.env[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runProductionReadinessCheck() {
|
||||
console.log('🏭 BRAINY PRODUCTION READINESS CHECK')
|
||||
console.log('====================================')
|
||||
console.log('Testing all deployment scenarios to ensure no emergency releases')
|
||||
|
||||
await cleanEnvironment()
|
||||
|
||||
const testResults = []
|
||||
|
||||
// Test 1: Fresh install with default settings (NEW: should work with remote downloads)
|
||||
testResults.push(await testScenario(
|
||||
'Fresh Install - Default Settings',
|
||||
{},
|
||||
'success' // This should now work with the fix
|
||||
))
|
||||
|
||||
// Test 2: Explicit remote models enabled
|
||||
testResults.push(await testScenario(
|
||||
'Remote Models Explicitly Enabled',
|
||||
{ BRAINY_ALLOW_REMOTE_MODELS: 'true' },
|
||||
'success'
|
||||
))
|
||||
|
||||
// Test 3: Remote models explicitly disabled (air-gapped scenario)
|
||||
testResults.push(await testScenario(
|
||||
'Air-Gapped Deployment (Local Only)',
|
||||
{ BRAINY_ALLOW_REMOTE_MODELS: 'false' },
|
||||
'failure' // Should fail gracefully with helpful error
|
||||
))
|
||||
|
||||
// Test 4: Development environment
|
||||
testResults.push(await testScenario(
|
||||
'Development Environment',
|
||||
{ NODE_ENV: 'development' },
|
||||
'success'
|
||||
))
|
||||
|
||||
// Test 5: Production environment with explicit config
|
||||
testResults.push(await testScenario(
|
||||
'Production Environment',
|
||||
{ NODE_ENV: 'production', BRAINY_ALLOW_REMOTE_MODELS: 'true' },
|
||||
'success'
|
||||
))
|
||||
|
||||
const successCount = testResults.filter(result => result).length
|
||||
const totalTests = testResults.length
|
||||
|
||||
console.log('\n📊 PRODUCTION READINESS RESULTS')
|
||||
console.log('===============================')
|
||||
console.log(`Passed: ${successCount}/${totalTests}`)
|
||||
|
||||
if (successCount === totalTests) {
|
||||
console.log('✅ ALL TESTS PASSED - PRODUCTION READY!')
|
||||
console.log('\n🚀 DEPLOYMENT SCENARIOS VERIFIED:')
|
||||
console.log(' ✅ Fresh npm install works out of the box')
|
||||
console.log(' ✅ Remote model downloads work when enabled')
|
||||
console.log(' ✅ Air-gapped deployments fail gracefully with clear guidance')
|
||||
console.log(' ✅ Development environments work seamlessly')
|
||||
console.log(' ✅ Production environments work with proper configuration')
|
||||
|
||||
return true
|
||||
} else {
|
||||
console.log('❌ PRODUCTION READINESS CHECK FAILED')
|
||||
console.log('🚫 DO NOT RELEASE - Fix issues first')
|
||||
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Export for use in CI/CD
|
||||
export { runProductionReadinessCheck }
|
||||
|
||||
// Run if called directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runProductionReadinessCheck()
|
||||
.then(success => process.exit(success ? 0 : 1))
|
||||
.catch(error => {
|
||||
console.error('❌ Production readiness check crashed:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
12
scripts/publish-cli.js
Executable file
12
scripts/publish-cli.js
Executable file
|
|
@ -0,0 +1,12 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* DEPRECATED: Script to build and publish both the main package and CLI package
|
||||
*
|
||||
* This script is no longer functional as the CLI package has been removed.
|
||||
* Use 'npm publish' directly to publish the main package only.
|
||||
*/
|
||||
|
||||
console.error('This script is deprecated. The CLI package has been removed.')
|
||||
console.error('To publish the main package only, use: npm publish')
|
||||
process.exit(1)
|
||||
164
scripts/release-workflow.js
Executable file
164
scripts/release-workflow.js
Executable file
|
|
@ -0,0 +1,164 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Release Workflow Script
|
||||
*
|
||||
* This script provides a comprehensive workflow for releasing a new version:
|
||||
* 1. Updates the version (major, minor, or patch)
|
||||
* 2. Automatically updates the CHANGELOG.md with commit messages since the last release
|
||||
* 3. Creates a GitHub release
|
||||
* 4. Deploys to NPM
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/release-workflow.js [patch|minor|major]
|
||||
*
|
||||
* If no version type is specified, it defaults to "patch"
|
||||
*/
|
||||
|
||||
/* global process, console */
|
||||
|
||||
import { execSync } from 'child_process'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import readline from 'readline'
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
// Path to the root directory
|
||||
const rootDir = path.join(__dirname, '..')
|
||||
|
||||
// Get the version type from command line arguments
|
||||
const args = process.argv.slice(2)
|
||||
const versionType = args[0] || 'patch'
|
||||
|
||||
// Validate version type
|
||||
if (!['patch', 'minor', 'major'].includes(versionType)) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error('Error: Version type must be one of: patch, minor, major')
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Function to execute a command and log its output
|
||||
function executeStep(command, description) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`\n🚀 ${description}...\n`)
|
||||
try {
|
||||
execSync(command, { stdio: 'inherit', cwd: rootDir })
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`✅ ${description} completed successfully!\n`)
|
||||
return true
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
`❌ Error during ${description.toLowerCase()}: ${error.message}`
|
||||
)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Main workflow
|
||||
async function runReleaseWorkflow() {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`\n=== Starting Release Workflow (${versionType}) ===\n`)
|
||||
|
||||
// Step 1: Build the project
|
||||
if (!executeStep('npm run build', 'Building project')) {
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Step 2: Run tests to ensure everything is working
|
||||
if (!executeStep('npm test', 'Running tests')) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
'⚠️ Tests failed. This might indicate issues with the release.'
|
||||
)
|
||||
|
||||
// Ask the user if they want to continue despite test failures
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'\n⚠️ Do you want to continue with the release process despite test failures? (y/N)'
|
||||
)
|
||||
|
||||
const readline = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
})
|
||||
|
||||
const response = await new Promise((resolve) => {
|
||||
readline.question('', (answer) => {
|
||||
readline.close()
|
||||
resolve(answer.toLowerCase())
|
||||
})
|
||||
})
|
||||
|
||||
if (response !== 'y' && response !== 'yes') {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error('Release process aborted due to test failures.')
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Continuing with release process despite test failures...')
|
||||
}
|
||||
|
||||
// Step 3: Update version and generate changelog
|
||||
if (
|
||||
!executeStep(
|
||||
`npm run _release:${versionType}`,
|
||||
`Updating version (${versionType}) and generating changelog`
|
||||
)
|
||||
) {
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Step 4: Create GitHub release
|
||||
if (!executeStep('npm run _github-release', 'Creating GitHub release')) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'Warning: GitHub release creation failed, but continuing with deployment...'
|
||||
)
|
||||
}
|
||||
|
||||
// Step 5: Publish to NPM
|
||||
if (!executeStep('npm publish', 'Publishing to NPM')) {
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Get the new version from package.json
|
||||
const packageJsonPath = path.join(rootDir, 'package.json')
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
|
||||
const newVersion = packageJson.version
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`\n🎉 Release v${newVersion} completed successfully! 🎉\n`)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Summary of actions:')
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`- Project built and tested`)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`- Version bumped to v${newVersion} (${versionType})`)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`- CHANGELOG.md updated with recent commits`)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`- GitHub release created with auto-generated notes`)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`- Package published to NPM`)
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('\nThank you for using the release workflow!\n')
|
||||
}
|
||||
|
||||
// Run the workflow
|
||||
runReleaseWorkflow().catch((error) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error('Unexpected error during release workflow:', error)
|
||||
// eslint-disable-next-line no-process-exit
|
||||
process.exit(1)
|
||||
})
|
||||
83
scripts/start-broadcast-server.ts
Normal file
83
scripts/start-broadcast-server.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Brain Jar Broadcast Server
|
||||
*
|
||||
* Start this server to enable real-time communication between
|
||||
* multiple Claude instances (Jarvis, Picasso, etc.)
|
||||
*
|
||||
* Usage:
|
||||
* npm run broadcast:local # Start local server on port 8765
|
||||
* npm run broadcast:cloud # Start cloud server on port 8080
|
||||
*/
|
||||
|
||||
import { BrainyData } from '../src/brainyData.js'
|
||||
import { BrainyMCPBroadcast } from '../src/mcp/brainyMCPBroadcast.js'
|
||||
|
||||
const isCloud = process.argv.includes('--cloud')
|
||||
const port = isCloud ? 8080 : 8765
|
||||
|
||||
async function startServer() {
|
||||
console.log('🧠🫙 Starting Brain Jar Broadcast Server...')
|
||||
console.log('=====================================')
|
||||
|
||||
// Initialize Brainy for server-side memory
|
||||
const brainy = new BrainyData({
|
||||
storagePath: '.brain-jar/server',
|
||||
dimensions: 384
|
||||
})
|
||||
|
||||
await brainy.init()
|
||||
console.log('✅ Brainy initialized for server memory')
|
||||
|
||||
// Create broadcast server
|
||||
const broadcast = new BrainyMCPBroadcast(brainy, {
|
||||
broadcastPort: port,
|
||||
cloudUrl: isCloud ? process.env.CLOUD_URL : undefined
|
||||
})
|
||||
|
||||
// Start the server
|
||||
await broadcast.startBroadcastServer(port, isCloud)
|
||||
|
||||
console.log('')
|
||||
console.log('🚀 Server Ready!')
|
||||
console.log('=====================================')
|
||||
console.log('Claude instances can now connect using:')
|
||||
console.log('')
|
||||
console.log('For Jarvis (Backend):')
|
||||
console.log(` const client = new BrainyMCPClient({`)
|
||||
console.log(` name: 'Jarvis',`)
|
||||
console.log(` role: 'Backend Systems',`)
|
||||
console.log(` serverUrl: 'ws://localhost:${port}'`)
|
||||
console.log(` })`)
|
||||
console.log(` await client.connect()`)
|
||||
console.log('')
|
||||
console.log('For Picasso (Frontend):')
|
||||
console.log(` const client = new BrainyMCPClient({`)
|
||||
console.log(` name: 'Picasso',`)
|
||||
console.log(` role: 'Frontend Design',`)
|
||||
console.log(` serverUrl: 'ws://localhost:${port}'`)
|
||||
console.log(` })`)
|
||||
console.log(` await client.connect()`)
|
||||
console.log('')
|
||||
console.log('=====================================')
|
||||
console.log('Press Ctrl+C to stop the server')
|
||||
|
||||
// Handle graceful shutdown
|
||||
process.on('SIGINT', async () => {
|
||||
console.log('\n📛 Shutting down server...')
|
||||
await broadcast.stopBroadcastServer()
|
||||
process.exit(0)
|
||||
})
|
||||
|
||||
process.on('SIGTERM', async () => {
|
||||
await broadcast.stopBroadcastServer()
|
||||
process.exit(0)
|
||||
})
|
||||
}
|
||||
|
||||
// Start the server
|
||||
startServer().catch(error => {
|
||||
console.error('Failed to start server:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
26
scripts/update-readme.js
Normal file
26
scripts/update-readme.js
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Get the directory of the current module
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Paths to the files
|
||||
const readmePath = path.join(__dirname, '..', 'README.md');
|
||||
const encodedImagePath = path.join(__dirname, '..', 'encoded-image.html');
|
||||
|
||||
// Read the files
|
||||
const readmeContent = fs.readFileSync(readmePath, 'utf8');
|
||||
const encodedImageTag = fs.readFileSync(encodedImagePath, 'utf8');
|
||||
|
||||
// Replace the image tag in the README
|
||||
const updatedReadme = readmeContent.replace(
|
||||
/<img src="brainy\.png" alt="Brainy Logo" width="200"\/>/,
|
||||
encodedImageTag
|
||||
);
|
||||
|
||||
// Write the updated README
|
||||
fs.writeFileSync(readmePath, updatedReadme);
|
||||
|
||||
console.log('README.md has been updated with the base64-encoded image.');
|
||||
Loading…
Add table
Add a link
Reference in a new issue