chore(8.0): final pre-RC1 sweep — API consistency, named errors, orphans, zero-cast codebase
This commit is contained in:
parent
970e08c466
commit
1f7e365a4e
237 changed files with 1951 additions and 49413 deletions
|
|
@ -1,60 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Create GitHub release with unarchived model files for direct serving
|
||||
# This allows transformers.js to download individual files directly
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Creating GitHub release with unarchived model files..."
|
||||
|
||||
# Create temporary directory
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
cd "$TEMP_DIR"
|
||||
|
||||
echo "📥 Downloading existing model archive..."
|
||||
curl -sL https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz -o models.tar.gz
|
||||
|
||||
echo "📦 Extracting models..."
|
||||
tar -xzf models.tar.gz
|
||||
|
||||
echo "🔧 Preparing individual model files for release..."
|
||||
|
||||
# Create properly structured directory
|
||||
mkdir -p release-files/Xenova/all-MiniLM-L6-v2/onnx
|
||||
|
||||
# Copy files with correct naming for direct serving
|
||||
cp all-MiniLM-L6-v2/config.json release-files/Xenova/all-MiniLM-L6-v2/
|
||||
cp all-MiniLM-L6-v2/tokenizer.json release-files/Xenova/all-MiniLM-L6-v2/
|
||||
cp all-MiniLM-L6-v2/tokenizer_config.json release-files/Xenova/all-MiniLM-L6-v2/
|
||||
cp all-MiniLM-L6-v2/onnx/model.onnx release-files/Xenova/all-MiniLM-L6-v2/onnx/
|
||||
|
||||
# Also keep the tar.gz for backward compatibility
|
||||
cp models.tar.gz release-files/
|
||||
|
||||
echo "📋 Files to upload:"
|
||||
find release-files -type f -exec ls -lh {} \;
|
||||
|
||||
echo ""
|
||||
echo "✅ Files ready for release!"
|
||||
echo ""
|
||||
echo "To create the release with individual files:"
|
||||
echo ""
|
||||
echo "1. Go to: https://github.com/soulcraftlabs/brainy/releases/new"
|
||||
echo "2. Tag: models-v2"
|
||||
echo "3. Title: Model Files v2 - Unarchived"
|
||||
echo "4. Upload these files from $TEMP_DIR/release-files/:"
|
||||
echo " - Xenova/all-MiniLM-L6-v2/config.json"
|
||||
echo " - Xenova/all-MiniLM-L6-v2/tokenizer.json"
|
||||
echo " - Xenova/all-MiniLM-L6-v2/tokenizer_config.json"
|
||||
echo " - Xenova/all-MiniLM-L6-v2/onnx/model.onnx"
|
||||
echo " - all-MiniLM-L6-v2.tar.gz (for compatibility)"
|
||||
echo ""
|
||||
echo "Or use GitHub CLI:"
|
||||
echo " cd $TEMP_DIR/release-files"
|
||||
echo " gh release create models-v2 --repo soulcraftlabs/brainy \\"
|
||||
echo " --title 'Model Files v2 - Direct Serving' \\"
|
||||
echo " --notes 'Unarchived model files for direct HTTP serving' \\"
|
||||
echo " Xenova/all-MiniLM-L6-v2/config.json \\"
|
||||
echo " Xenova/all-MiniLM-L6-v2/tokenizer.json \\"
|
||||
echo " Xenova/all-MiniLM-L6-v2/tokenizer_config.json \\"
|
||||
echo " Xenova/all-MiniLM-L6-v2/onnx/model.onnx \\"
|
||||
echo " models.tar.gz"
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
/**
|
||||
* Diagnostic script for Workshop type filtering issue
|
||||
*
|
||||
* This script mimics Workshop's exact import pattern to see what's happening
|
||||
*/
|
||||
|
||||
import { Brainy, NounType } from '../src/index.js'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
|
||||
async function diagnose() {
|
||||
console.log('\n🔬 Workshop Type Filtering Diagnostic\n')
|
||||
console.log('='.repeat(70))
|
||||
|
||||
// Use temporary directory
|
||||
const testDir = './test-workshop-data'
|
||||
if (fs.existsSync(testDir)) {
|
||||
fs.rmSync(testDir, { recursive: true })
|
||||
}
|
||||
|
||||
const brain = new Brainy({
|
||||
storage: {
|
||||
type: 'filesystem',
|
||||
path: testDir
|
||||
}
|
||||
})
|
||||
await brain.init()
|
||||
|
||||
console.log('\n1️⃣ Testing WITHOUT VFS (control group)...\n')
|
||||
|
||||
// Add entities without VFS
|
||||
console.log('Adding 3 person entities directly...')
|
||||
await brain.add({ data: 'Person 1', type: NounType.Person, metadata: { name: 'Person 1' } })
|
||||
await brain.add({ data: 'Person 2', type: NounType.Person, metadata: { name: 'Person 2' } })
|
||||
await brain.add({ data: 'Person 3', type: NounType.Person, metadata: { name: 'Person 3' } })
|
||||
|
||||
console.log('Adding 2 location entities directly...')
|
||||
await brain.add({ data: 'Location 1', type: NounType.Location, metadata: { name: 'Location 1' } })
|
||||
await brain.add({ data: 'Location 2', type: NounType.Location, metadata: { name: 'Location 2' } })
|
||||
|
||||
console.log('\n📊 Testing type filtering on direct entities...\n')
|
||||
|
||||
const allDirect = await brain.find({ limit: 100 })
|
||||
console.log(` Total entities: ${allDirect.length}`)
|
||||
|
||||
const peopleDirect = await brain.find({ type: NounType.Person, limit: 100 })
|
||||
console.log(` Person filter: ${peopleDirect.length} (expected: 3)`)
|
||||
|
||||
const locationsDirect = await brain.find({ type: NounType.Location, limit: 100 })
|
||||
console.log(` Location filter: ${locationsDirect.length} (expected: 2)`)
|
||||
|
||||
if (peopleDirect.length === 3 && locationsDirect.length === 2) {
|
||||
console.log('\n ✅ Type filtering works on direct entities!')
|
||||
} else {
|
||||
console.log('\n ❌ Type filtering BROKEN on direct entities!')
|
||||
return
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(70))
|
||||
console.log('\n2️⃣ Testing WITH VFS (Workshop pattern)...\n')
|
||||
|
||||
// Simulate Workshop's pattern: create VFS file entities
|
||||
console.log('Creating VFS file wrappers (like import does)...')
|
||||
|
||||
const vfs = brain.vfs()
|
||||
await vfs.init()
|
||||
|
||||
// Create VFS directory
|
||||
await vfs.mkdir('/imports/test', { recursive: true })
|
||||
|
||||
// Create VFS files with embedded entity data (mimics import)
|
||||
console.log('Creating VFS file for Person entity...')
|
||||
const personEntityData = {
|
||||
id: 'ent_person_test',
|
||||
name: 'John Smith',
|
||||
type: 'person',
|
||||
metadata: {
|
||||
source: 'excel',
|
||||
originalData: {
|
||||
Name: 'John Smith',
|
||||
_sheet: 'Characters'
|
||||
}
|
||||
}
|
||||
}
|
||||
await vfs.writeFile(
|
||||
'/imports/test/john_smith.json',
|
||||
Buffer.from(JSON.stringify(personEntityData, null, 2))
|
||||
)
|
||||
|
||||
console.log('Creating VFS file for Location entity...')
|
||||
const locationEntityData = {
|
||||
id: 'ent_location_test',
|
||||
name: 'New York',
|
||||
type: 'location',
|
||||
metadata: {
|
||||
source: 'excel',
|
||||
originalData: {
|
||||
Name: 'New York',
|
||||
_sheet: 'Places'
|
||||
}
|
||||
}
|
||||
}
|
||||
await vfs.writeFile(
|
||||
'/imports/test/new_york.json',
|
||||
Buffer.from(JSON.stringify(locationEntityData, null, 2))
|
||||
)
|
||||
|
||||
console.log('\n📊 Checking what entities exist now...\n')
|
||||
|
||||
const allWithVfs = await brain.find({ limit: 100 })
|
||||
console.log(` Total entities: ${allWithVfs.length}`)
|
||||
|
||||
// Analyze entity types
|
||||
const typeCounts: Record<string, number> = {}
|
||||
for (const result of allWithVfs) {
|
||||
const type = result.type || 'unknown'
|
||||
typeCounts[type] = (typeCounts[type] || 0) + 1
|
||||
}
|
||||
|
||||
console.log('\n Entity types breakdown:')
|
||||
for (const [type, count] of Object.entries(typeCounts)) {
|
||||
console.log(` - ${type}: ${count}`)
|
||||
}
|
||||
|
||||
console.log('\n📊 Testing type filtering with VFS entities...\n')
|
||||
|
||||
const peopleWithVfs = await brain.find({ type: NounType.Person, limit: 100 })
|
||||
console.log(` Person filter: ${peopleWithVfs.length} (expected: 3 direct + maybe VFS?)`)
|
||||
|
||||
const locationsWithVfs = await brain.find({ type: NounType.Location, limit: 100 })
|
||||
console.log(` Location filter: ${locationsWithVfs.length} (expected: 2 direct + maybe VFS?)`)
|
||||
|
||||
const documents = await brain.find({ type: NounType.Document, limit: 100 })
|
||||
console.log(` Document filter: ${documents.length} (VFS wrappers?)`)
|
||||
|
||||
console.log('\n' + '='.repeat(70))
|
||||
console.log('\n3️⃣ Analyzing VFS wrapper structure...\n')
|
||||
|
||||
// Get a VFS wrapper entity
|
||||
const vfsWrapper = allWithVfs.find(e => e.metadata?.vfsType === 'file')
|
||||
if (vfsWrapper) {
|
||||
console.log('Found VFS wrapper entity:')
|
||||
console.log(` - ID: ${vfsWrapper.id}`)
|
||||
console.log(` - Type: ${vfsWrapper.type}`)
|
||||
console.log(` - VFS Type: ${vfsWrapper.metadata?.vfsType}`)
|
||||
console.log(` - Has rawData: ${!!vfsWrapper.metadata?.rawData}`)
|
||||
console.log(` - Path: ${vfsWrapper.metadata?.path}`)
|
||||
|
||||
if (vfsWrapper.metadata?.rawData) {
|
||||
console.log('\n Decoding rawData...')
|
||||
try {
|
||||
const decoded = Buffer.from(vfsWrapper.metadata.rawData, 'base64').toString()
|
||||
const entity = JSON.parse(decoded)
|
||||
console.log(` - Embedded entity name: ${entity.name}`)
|
||||
console.log(` - Embedded entity type: ${entity.type}`)
|
||||
console.log(` - Wrapper type: ${vfsWrapper.type}`)
|
||||
console.log('\n 🔍 KEY INSIGHT:')
|
||||
console.log(` Wrapper has type="${vfsWrapper.type}"`)
|
||||
console.log(` But embedded entity has type="${entity.type}"`)
|
||||
console.log(' When you filter by person, you get the WRAPPER type, not embedded type!')
|
||||
} catch (err) {
|
||||
console.log(' ❌ Failed to decode rawData')
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log('No VFS wrapper entities found')
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(70))
|
||||
console.log('\n4️⃣ DIAGNOSIS\n')
|
||||
|
||||
if (documents.length > 0 && peopleWithVfs.length === 3) {
|
||||
console.log('❌ FOUND THE BUG!')
|
||||
console.log('')
|
||||
console.log('VFS creates document wrappers with type="document".')
|
||||
console.log('The actual entity data is stored as base64 in metadata.rawData.')
|
||||
console.log('When you filter by type="person", you\'re filtering the WRAPPER type.')
|
||||
console.log('Since wrappers are type="document", you get 0 results.')
|
||||
console.log('')
|
||||
console.log('This is a DESIGN ISSUE in how VFS import works!')
|
||||
} else if (peopleWithVfs.length > 3) {
|
||||
console.log('✅ Type filtering works correctly!')
|
||||
console.log('VFS entities are being created with proper types.')
|
||||
} else {
|
||||
console.log('🤔 Unclear - need more investigation')
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(70) + '\n')
|
||||
|
||||
// Cleanup
|
||||
fs.rmSync(testDir, { recursive: true })
|
||||
}
|
||||
|
||||
diagnose().catch(console.error)
|
||||
|
|
@ -1,323 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Migration script to replace console.log statements with structured logger
|
||||
* Usage: npm run migrate:logger [--dry-run] [--file=path/to/file.ts]
|
||||
*/
|
||||
|
||||
import * as fs from 'fs/promises'
|
||||
import * as path from 'path'
|
||||
import { glob } from 'glob'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
interface MigrationOptions {
|
||||
dryRun: boolean
|
||||
targetFile?: string
|
||||
verbose: boolean
|
||||
}
|
||||
|
||||
interface MigrationResult {
|
||||
file: string
|
||||
changes: number
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
class LoggerMigrator {
|
||||
private results: MigrationResult[] = []
|
||||
|
||||
constructor(private options: MigrationOptions) {}
|
||||
|
||||
async migrate(): Promise<void> {
|
||||
console.log('🔄 Starting logger migration...')
|
||||
|
||||
const files = await this.getFilesToMigrate()
|
||||
console.log(`Found ${files.length} TypeScript files to process`)
|
||||
|
||||
for (const file of files) {
|
||||
await this.migrateFile(file)
|
||||
}
|
||||
|
||||
this.printSummary()
|
||||
}
|
||||
|
||||
private async getFilesToMigrate(): Promise<string[]> {
|
||||
if (this.options.targetFile) {
|
||||
return [this.options.targetFile]
|
||||
}
|
||||
|
||||
// Get all TypeScript files excluding node_modules, dist, and test files
|
||||
const pattern = 'src/**/*.ts'
|
||||
const ignore = [
|
||||
'**/node_modules/**',
|
||||
'**/dist/**',
|
||||
'**/*.test.ts',
|
||||
'**/*.spec.ts',
|
||||
'**/logger.ts',
|
||||
'**/structuredLogger.ts'
|
||||
]
|
||||
|
||||
return glob(pattern, { ignore })
|
||||
}
|
||||
|
||||
private async migrateFile(filePath: string): Promise<void> {
|
||||
const result: MigrationResult = {
|
||||
file: filePath,
|
||||
changes: 0,
|
||||
errors: []
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await fs.readFile(filePath, 'utf-8')
|
||||
const moduleName = this.extractModuleName(filePath)
|
||||
|
||||
let modified = content
|
||||
let changesMade = false
|
||||
|
||||
// Check if file already imports logger
|
||||
const hasLoggerImport = /import.*(?:createModuleLogger|structuredLogger).*from/.test(content)
|
||||
const hasConsoleUsage = /console\.(log|warn|error|info|debug)/.test(content)
|
||||
|
||||
if (!hasConsoleUsage) {
|
||||
if (this.options.verbose) {
|
||||
console.log(` ⏭️ ${filePath} - No console statements found`)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Add import if needed
|
||||
if (!hasLoggerImport && hasConsoleUsage) {
|
||||
modified = this.addLoggerImport(modified)
|
||||
changesMade = true
|
||||
}
|
||||
|
||||
// Replace console statements
|
||||
const patterns = [
|
||||
{
|
||||
// console.log with string literal
|
||||
pattern: /console\.log\s*\(\s*(['"`])([^'"`]*)\1\s*(?:,\s*(.+?))?\s*\)/g,
|
||||
replacement: (match: string, quote: string, message: string, args?: string) => {
|
||||
result.changes++
|
||||
return args
|
||||
? `logger.info('${message}', ${args})`
|
||||
: `logger.info('${message}')`
|
||||
}
|
||||
},
|
||||
{
|
||||
// console.error with string literal
|
||||
pattern: /console\.error\s*\(\s*(['"`])([^'"`]*)\1\s*(?:,\s*(.+?))?\s*\)/g,
|
||||
replacement: (match: string, quote: string, message: string, args?: string) => {
|
||||
result.changes++
|
||||
return args
|
||||
? `logger.error('${message}', ${args})`
|
||||
: `logger.error('${message}')`
|
||||
}
|
||||
},
|
||||
{
|
||||
// console.warn with string literal
|
||||
pattern: /console\.warn\s*\(\s*(['"`])([^'"`]*)\1\s*(?:,\s*(.+?))?\s*\)/g,
|
||||
replacement: (match: string, quote: string, message: string, args?: string) => {
|
||||
result.changes++
|
||||
return args
|
||||
? `logger.warn('${message}', ${args})`
|
||||
: `logger.warn('${message}')`
|
||||
}
|
||||
},
|
||||
{
|
||||
// console.info with string literal
|
||||
pattern: /console\.info\s*\(\s*(['"`])([^'"`]*)\1\s*(?:,\s*(.+?))?\s*\)/g,
|
||||
replacement: (match: string, quote: string, message: string, args?: string) => {
|
||||
result.changes++
|
||||
return args
|
||||
? `logger.info('${message}', ${args})`
|
||||
: `logger.info('${message}')`
|
||||
}
|
||||
},
|
||||
{
|
||||
// console.debug with string literal
|
||||
pattern: /console\.debug\s*\(\s*(['"`])([^'"`]*)\1\s*(?:,\s*(.+?))?\s*\)/g,
|
||||
replacement: (match: string, quote: string, message: string, args?: string) => {
|
||||
result.changes++
|
||||
return args
|
||||
? `logger.debug('${message}', ${args})`
|
||||
: `logger.debug('${message}')`
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
// Apply replacements
|
||||
for (const { pattern, replacement } of patterns) {
|
||||
const before = modified
|
||||
modified = modified.replace(pattern, replacement as any)
|
||||
if (before !== modified) {
|
||||
changesMade = true
|
||||
}
|
||||
}
|
||||
|
||||
// Handle complex console statements that need manual review
|
||||
const complexPatterns = [
|
||||
/console\.(log|warn|error|info|debug)\s*\([^'"`]/g
|
||||
]
|
||||
|
||||
for (const pattern of complexPatterns) {
|
||||
const matches = modified.match(pattern)
|
||||
if (matches) {
|
||||
for (const match of matches) {
|
||||
result.errors.push(`Complex console statement needs manual review: ${match.substring(0, 50)}...`)
|
||||
|
||||
// Add a TODO comment for manual review
|
||||
modified = modified.replace(
|
||||
match,
|
||||
`// TODO: Migrate to structured logger\n ${match}`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add logger declaration after imports
|
||||
if (changesMade && !hasLoggerImport) {
|
||||
const importEndMatch = modified.match(/^((?:import.*\n)+)/m)
|
||||
if (importEndMatch) {
|
||||
const afterImports = importEndMatch.index! + importEndMatch[0].length
|
||||
modified =
|
||||
modified.slice(0, afterImports) +
|
||||
`\nconst logger = createModuleLogger('${moduleName}')\n` +
|
||||
modified.slice(afterImports)
|
||||
}
|
||||
}
|
||||
|
||||
// Write changes
|
||||
if (changesMade && !this.options.dryRun) {
|
||||
await fs.writeFile(filePath, modified, 'utf-8')
|
||||
console.log(` ✅ ${filePath} - ${result.changes} changes`)
|
||||
} else if (changesMade) {
|
||||
console.log(` 🔍 ${filePath} - ${result.changes} changes (dry run)`)
|
||||
}
|
||||
|
||||
this.results.push(result)
|
||||
|
||||
} catch (error) {
|
||||
result.errors.push(`Failed to process file: ${error}`)
|
||||
this.results.push(result)
|
||||
}
|
||||
}
|
||||
|
||||
private addLoggerImport(content: string): string {
|
||||
// Find the last import statement
|
||||
const importMatches = [...content.matchAll(/^import.*$/gm)]
|
||||
|
||||
if (importMatches.length > 0) {
|
||||
const lastImport = importMatches[importMatches.length - 1]
|
||||
const insertPos = lastImport.index! + lastImport[0].length
|
||||
|
||||
const relativeImportPath = this.getRelativeImportPath()
|
||||
const importStatement = `\nimport { createModuleLogger } from '${relativeImportPath}'`
|
||||
|
||||
return content.slice(0, insertPos) + importStatement + content.slice(insertPos)
|
||||
}
|
||||
|
||||
// No imports found, add at the beginning
|
||||
const relativeImportPath = this.getRelativeImportPath()
|
||||
return `import { createModuleLogger } from '${relativeImportPath}'\n\n${content}`
|
||||
}
|
||||
|
||||
private getRelativeImportPath(): string {
|
||||
// This will be calculated based on the file being processed
|
||||
// For now, return a placeholder
|
||||
return '../utils/structuredLogger.js'
|
||||
}
|
||||
|
||||
private extractModuleName(filePath: string): string {
|
||||
// Extract module name from file path
|
||||
const relativePath = path.relative('src', filePath)
|
||||
const moduleName = relativePath
|
||||
.replace(/\.ts$/, '')
|
||||
.replace(/\//g, ':')
|
||||
.replace(/\\+/g, ':')
|
||||
|
||||
return moduleName
|
||||
}
|
||||
|
||||
private printSummary(): void {
|
||||
console.log('\n📊 Migration Summary:')
|
||||
console.log('=' .repeat(50))
|
||||
|
||||
let totalChanges = 0
|
||||
let totalErrors = 0
|
||||
let filesWithChanges = 0
|
||||
let filesWithErrors = 0
|
||||
|
||||
for (const result of this.results) {
|
||||
if (result.changes > 0) {
|
||||
filesWithChanges++
|
||||
totalChanges += result.changes
|
||||
}
|
||||
if (result.errors.length > 0) {
|
||||
filesWithErrors++
|
||||
totalErrors += result.errors.length
|
||||
|
||||
console.log(`\n⚠️ ${result.file}:`)
|
||||
for (const error of result.errors) {
|
||||
console.log(` - ${error}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n📈 Statistics:')
|
||||
console.log(` Files processed: ${this.results.length}`)
|
||||
console.log(` Files modified: ${filesWithChanges}`)
|
||||
console.log(` Total changes: ${totalChanges}`)
|
||||
console.log(` Files with errors: ${filesWithErrors}`)
|
||||
console.log(` Total errors: ${totalErrors}`)
|
||||
|
||||
if (this.options.dryRun) {
|
||||
console.log('\n⚠️ This was a dry run. No files were modified.')
|
||||
console.log('Run without --dry-run to apply changes.')
|
||||
}
|
||||
|
||||
if (totalErrors > 0) {
|
||||
console.log('\n⚠️ Some console statements need manual review.')
|
||||
console.log('Search for "TODO: Migrate to structured logger" in the code.')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse command line arguments
|
||||
function parseArgs(): MigrationOptions {
|
||||
const args = process.argv.slice(2)
|
||||
const options: MigrationOptions = {
|
||||
dryRun: false,
|
||||
verbose: false
|
||||
}
|
||||
|
||||
for (const arg of args) {
|
||||
if (arg === '--dry-run') {
|
||||
options.dryRun = true
|
||||
} else if (arg === '--verbose' || arg === '-v') {
|
||||
options.verbose = true
|
||||
} else if (arg.startsWith('--file=')) {
|
||||
options.targetFile = arg.split('=')[1]
|
||||
}
|
||||
}
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
// Main execution
|
||||
async function main() {
|
||||
const options = parseArgs()
|
||||
const migrator = new LoggerMigrator(options)
|
||||
|
||||
try {
|
||||
await migrator.migrate()
|
||||
} catch (error) {
|
||||
console.error('Migration failed:', error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(console.error)
|
||||
}
|
||||
|
||||
export { LoggerMigrator, MigrationOptions }
|
||||
|
|
@ -1,269 +0,0 @@
|
|||
/**
|
||||
* VFS Storage Migration Script: v5.1.x → v5.2.0
|
||||
*
|
||||
* Converts VFS files from 3-tier storage (inline/reference/chunked)
|
||||
* to unified BlobStorage (content-addressable with deduplication)
|
||||
*
|
||||
* Usage:
|
||||
* import { migrateVFSStorage } from './scripts/migrate-vfs-storage'
|
||||
* await migrateVFSStorage(brain)
|
||||
*/
|
||||
|
||||
import { Brainy } from '../src/brainy.js'
|
||||
import { VFSMetadata } from '../src/vfs/types.js'
|
||||
|
||||
export interface MigrationStats {
|
||||
filesProcessed: number
|
||||
inlineMigrated: number
|
||||
referenceMigrated: number
|
||||
chunkedMigrated: number
|
||||
alreadyMigrated: number
|
||||
errors: number
|
||||
deduplicated: number
|
||||
totalSizeBefore: number
|
||||
totalSizeAfter: number
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
export async function migrateVFSStorage(
|
||||
brain: Brainy,
|
||||
options: {
|
||||
dryRun?: boolean
|
||||
verbose?: boolean
|
||||
batchSize?: number
|
||||
} = {}
|
||||
): Promise<MigrationStats> {
|
||||
const { dryRun = false, verbose = false, batchSize = 100 } = options
|
||||
|
||||
const stats: MigrationStats = {
|
||||
filesProcessed: 0,
|
||||
inlineMigrated: 0,
|
||||
referenceMigrated: 0,
|
||||
chunkedMigrated: 0,
|
||||
alreadyMigrated: 0,
|
||||
errors: 0,
|
||||
deduplicated: 0,
|
||||
totalSizeBefore: 0,
|
||||
totalSizeAfter: 0,
|
||||
durationMs: 0
|
||||
}
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
if (verbose) {
|
||||
console.log('🔄 Starting VFS storage migration (v5.1.x → v5.2.0)')
|
||||
if (dryRun) console.log(' DRY RUN: No changes will be made')
|
||||
}
|
||||
|
||||
try {
|
||||
// Find all VFS file entities
|
||||
const files = await brain.find({
|
||||
where: { 'metadata.vfsType': 'file' },
|
||||
limit: 100000 // Large limit to get all files
|
||||
})
|
||||
|
||||
if (verbose) {
|
||||
console.log(`📁 Found ${files.length} VFS files to process`)
|
||||
}
|
||||
|
||||
// Process in batches
|
||||
for (let i = 0; i < files.length; i += batchSize) {
|
||||
const batch = files.slice(i, i + batchSize)
|
||||
|
||||
await Promise.all(batch.map(async (file) => {
|
||||
try {
|
||||
stats.filesProcessed++
|
||||
|
||||
const metadata = file.metadata as VFSMetadata
|
||||
const storage = metadata.storage
|
||||
|
||||
// Already migrated?
|
||||
if (storage?.type === 'blob') {
|
||||
stats.alreadyMigrated++
|
||||
if (verbose && stats.filesProcessed % 100 === 0) {
|
||||
console.log(` ✓ ${metadata.path} (already migrated)`)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
let buffer: Buffer | null = null
|
||||
let sizeBefore = 0
|
||||
|
||||
// Migrate based on old storage type
|
||||
if (!storage || storage.type === 'inline') {
|
||||
// Inline storage: content in metadata.rawData
|
||||
if (metadata.rawData) {
|
||||
buffer = Buffer.from(metadata.rawData, 'base64')
|
||||
sizeBefore = buffer.length
|
||||
stats.inlineMigrated++
|
||||
|
||||
if (verbose && stats.filesProcessed % 100 === 0) {
|
||||
console.log(` → ${metadata.path} (inline, ${sizeBefore} bytes)`)
|
||||
}
|
||||
}
|
||||
} else if (storage.type === 'reference') {
|
||||
// Reference storage: content stored as separate entity
|
||||
if (storage.key) {
|
||||
const contentEntity = await brain.get(storage.key)
|
||||
if (contentEntity && contentEntity.data) {
|
||||
buffer = Buffer.isBuffer(contentEntity.data)
|
||||
? contentEntity.data
|
||||
: Buffer.from(contentEntity.data as string)
|
||||
sizeBefore = buffer.length
|
||||
stats.referenceMigrated++
|
||||
|
||||
if (verbose && stats.filesProcessed % 100 === 0) {
|
||||
console.log(` → ${metadata.path} (reference, ${sizeBefore} bytes)`)
|
||||
}
|
||||
|
||||
// Delete old reference entity (unless dry run)
|
||||
if (!dryRun) {
|
||||
await brain.delete(storage.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (storage.type === 'chunked') {
|
||||
// Chunked storage: content split across multiple entities
|
||||
if (storage.chunks && storage.chunks.length > 0) {
|
||||
const chunkBuffers = await Promise.all(
|
||||
storage.chunks.map(async (chunkId) => {
|
||||
const chunkEntity = await brain.get(chunkId)
|
||||
if (chunkEntity && chunkEntity.data) {
|
||||
return Buffer.isBuffer(chunkEntity.data)
|
||||
? chunkEntity.data
|
||||
: Buffer.from(chunkEntity.data as string)
|
||||
}
|
||||
return Buffer.alloc(0)
|
||||
})
|
||||
)
|
||||
|
||||
buffer = Buffer.concat(chunkBuffers)
|
||||
sizeBefore = buffer.length
|
||||
stats.chunkedMigrated++
|
||||
|
||||
if (verbose && stats.filesProcessed % 100 === 0) {
|
||||
console.log(` → ${metadata.path} (chunked, ${storage.chunks.length} chunks, ${sizeBefore} bytes)`)
|
||||
}
|
||||
|
||||
// Delete old chunk entities (unless dry run)
|
||||
if (!dryRun) {
|
||||
await Promise.all(storage.chunks.map(id => brain.delete(id)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store in BlobStorage
|
||||
if (buffer) {
|
||||
stats.totalSizeBefore += sizeBefore
|
||||
|
||||
if (!dryRun) {
|
||||
// Write to BlobStorage (content-addressable)
|
||||
const blobHash = await brain.vfs['blobStorage'].write(buffer)
|
||||
|
||||
// Check if deduplicated
|
||||
const blobMeta = await brain.vfs['blobStorage'].getMetadata(blobHash)
|
||||
if (blobMeta && blobMeta.refCount > 1) {
|
||||
stats.deduplicated++
|
||||
}
|
||||
|
||||
// Update VFS metadata
|
||||
await brain.update(file.id, {
|
||||
metadata: {
|
||||
...metadata,
|
||||
storage: {
|
||||
type: 'blob',
|
||||
hash: blobHash,
|
||||
size: buffer.length,
|
||||
compressed: blobMeta?.compressed
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
stats.totalSizeAfter += blobMeta?.compressedSize || buffer.length
|
||||
} else {
|
||||
// Dry run: just count sizes
|
||||
stats.totalSizeAfter += buffer.length
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
stats.errors++
|
||||
if (verbose) {
|
||||
console.error(` ✗ Error migrating ${file.metadata?.path}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
if (verbose && i + batchSize < files.length) {
|
||||
const progress = Math.round(((i + batchSize) / files.length) * 100)
|
||||
console.log(` Progress: ${progress}% (${i + batchSize}/${files.length})`)
|
||||
}
|
||||
}
|
||||
|
||||
stats.durationMs = Date.now() - startTime
|
||||
|
||||
if (verbose) {
|
||||
console.log('\n✅ Migration complete!\n')
|
||||
console.log('📊 Statistics:')
|
||||
console.log(` Files processed: ${stats.filesProcessed}`)
|
||||
console.log(` Inline migrated: ${stats.inlineMigrated}`)
|
||||
console.log(` Reference migrated: ${stats.referenceMigrated}`)
|
||||
console.log(` Chunked migrated: ${stats.chunkedMigrated}`)
|
||||
console.log(` Already migrated: ${stats.alreadyMigrated}`)
|
||||
console.log(` Deduplicated: ${stats.deduplicated}`)
|
||||
console.log(` Errors: ${stats.errors}`)
|
||||
console.log(` Size before: ${formatBytes(stats.totalSizeBefore)}`)
|
||||
console.log(` Size after: ${formatBytes(stats.totalSizeAfter)}`)
|
||||
console.log(` Space saved: ${formatBytes(stats.totalSizeBefore - stats.totalSizeAfter)}`)
|
||||
console.log(` Duration: ${stats.durationMs}ms`)
|
||||
|
||||
if (dryRun) {
|
||||
console.log('\n⚠️ DRY RUN: No changes were made')
|
||||
}
|
||||
}
|
||||
|
||||
return stats
|
||||
} catch (error) {
|
||||
if (verbose) {
|
||||
console.error('❌ Migration failed:', error)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 Bytes'
|
||||
|
||||
const k = 1024
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB']
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k))
|
||||
|
||||
return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i]
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect and migrate if needed
|
||||
* Called automatically on brain.init() if old format detected
|
||||
*/
|
||||
export async function autoMigrateIfNeeded(brain: Brainy): Promise<boolean> {
|
||||
try {
|
||||
// Check for old format files
|
||||
const oldFormatFiles = await brain.find({
|
||||
where: {
|
||||
'metadata.vfsType': 'file',
|
||||
'metadata.storage.type': { $in: ['inline', 'reference', 'chunked'] }
|
||||
},
|
||||
limit: 1
|
||||
})
|
||||
|
||||
if (oldFormatFiles.length > 0) {
|
||||
console.log('🔄 Detected v5.1.x VFS storage format. Auto-migrating to v5.2.0...')
|
||||
await migrateVFSStorage(brain, { verbose: true })
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
} catch (error) {
|
||||
console.error('❌ Auto-migration failed:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Setup GitHub branch with extracted model files for direct serving
|
||||
# This creates a 'models' branch with uncompressed files that transformers.js can use directly
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Setting up GitHub models branch for reliable model serving..."
|
||||
|
||||
# Create a temporary directory for our work
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
cd "$TEMP_DIR"
|
||||
|
||||
echo "📥 Downloading model tar.gz from release..."
|
||||
curl -sL https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz -o models.tar.gz
|
||||
|
||||
echo "📦 Extracting models..."
|
||||
tar -xzf models.tar.gz
|
||||
|
||||
echo "🔧 Setting up models branch..."
|
||||
git clone https://github.com/soulcraftlabs/brainy.git --depth 1 -b main repo
|
||||
cd repo
|
||||
|
||||
# Create orphan branch for models (no history needed)
|
||||
git checkout --orphan models
|
||||
git rm -rf . 2>/dev/null || true
|
||||
|
||||
# Create the proper directory structure for transformers.js
|
||||
mkdir -p Xenova/all-MiniLM-L6-v2/onnx
|
||||
|
||||
# Copy model files with correct structure
|
||||
cp ../all-MiniLM-L6-v2/config.json Xenova/all-MiniLM-L6-v2/
|
||||
cp ../all-MiniLM-L6-v2/tokenizer.json Xenova/all-MiniLM-L6-v2/
|
||||
cp ../all-MiniLM-L6-v2/tokenizer_config.json Xenova/all-MiniLM-L6-v2/
|
||||
cp ../all-MiniLM-L6-v2/onnx/model.onnx Xenova/all-MiniLM-L6-v2/onnx/
|
||||
cp ../all-MiniLM-L6-v2/onnx/model_quantized.onnx Xenova/all-MiniLM-L6-v2/onnx/ 2>/dev/null || true
|
||||
|
||||
# Add README explaining this branch
|
||||
cat > README.md << 'EOF'
|
||||
# Brainy Model Files
|
||||
|
||||
This branch contains extracted model files for direct serving via raw.githubusercontent.com
|
||||
|
||||
## Structure
|
||||
```
|
||||
Xenova/
|
||||
all-MiniLM-L6-v2/
|
||||
config.json
|
||||
tokenizer.json
|
||||
tokenizer_config.json
|
||||
onnx/
|
||||
model.onnx
|
||||
model_quantized.onnx
|
||||
```
|
||||
|
||||
## Usage
|
||||
These files are automatically loaded by Brainy when models are needed.
|
||||
The URL pattern is:
|
||||
`https://raw.githubusercontent.com/soulcraftlabs/brainy/models/Xenova/all-MiniLM-L6-v2/{file}`
|
||||
|
||||
## DO NOT EDIT
|
||||
This branch is managed automatically. Do not edit files directly.
|
||||
EOF
|
||||
|
||||
# Create .gitattributes for LFS if needed (for large model files)
|
||||
cat > .gitattributes << 'EOF'
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
EOF
|
||||
|
||||
echo "📝 Creating commit..."
|
||||
git add .
|
||||
git commit -m "feat: extracted model files for direct GitHub serving
|
||||
|
||||
- Xenova/all-MiniLM-L6-v2 model files
|
||||
- Proper directory structure for transformers.js
|
||||
- Direct serving via raw.githubusercontent.com"
|
||||
|
||||
echo "✅ Models branch ready!"
|
||||
echo ""
|
||||
echo "To push this branch to GitHub, run:"
|
||||
echo " cd $TEMP_DIR/repo"
|
||||
echo " git push origin models"
|
||||
echo ""
|
||||
echo "Once pushed, models will be available at:"
|
||||
echo " https://raw.githubusercontent.com/soulcraftlabs/brainy/models/Xenova/all-MiniLM-L6-v2/config.json"
|
||||
echo " https://raw.githubusercontent.com/soulcraftlabs/brainy/models/Xenova/all-MiniLM-L6-v2/tokenizer.json"
|
||||
echo " etc..."
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Run tests with adequate memory for transformer models
|
||||
echo "🧠 Running Brainy tests with 8GB heap allocation"
|
||||
echo "This is required for the transformer model (ONNX runtime)"
|
||||
echo "================================================"
|
||||
|
||||
# Set memory allocation
|
||||
export NODE_OPTIONS='--max-old-space-size=8192'
|
||||
|
||||
# Run tests based on argument
|
||||
if [ "$1" = "single" ]; then
|
||||
echo "Running tests sequentially (memory-safe)..."
|
||||
npx vitest run --pool=forks --poolOptions.forks.maxForks=1 --reporter=dot
|
||||
elif [ "$1" = "quick" ]; then
|
||||
echo "Running quick test..."
|
||||
node test-quick.js
|
||||
elif [ "$1" = "core" ]; then
|
||||
echo "Running core tests only..."
|
||||
npx vitest run tests/core.test.ts --reporter=verbose
|
||||
else
|
||||
echo "Running full test suite..."
|
||||
echo "Note: This requires 8GB+ RAM available"
|
||||
npm test
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Test complete. Memory was allocated at 8GB for ONNX runtime."
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Script to update all augmentations with metadata declarations
|
||||
|
||||
echo "📝 Updating augmentations with metadata declarations..."
|
||||
|
||||
# Category 1: No metadata access ('none')
|
||||
echo "🚫 Updating 'none' metadata augmentations..."
|
||||
|
||||
# RequestDeduplicatorAugmentation
|
||||
sed -i "/export class RequestDeduplicatorAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'none' as const // Doesn't access metadata" \
|
||||
src/augmentations/requestDeduplicatorAugmentation.ts
|
||||
|
||||
# ConnectionPoolAugmentation
|
||||
sed -i "/export class ConnectionPoolAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'none' as const // Doesn't access metadata" \
|
||||
src/augmentations/connectionPoolAugmentation.ts
|
||||
|
||||
# StorageAugmentation (abstract class)
|
||||
sed -i "/export abstract class StorageAugmentation extends BaseAugmentation/a\\
|
||||
readonly metadata = 'none' as const // Storage doesn't directly access metadata" \
|
||||
src/augmentations/storageAugmentation.ts
|
||||
|
||||
# Category 2: Read-only access ('readonly')
|
||||
echo "👁 Updating 'readonly' metadata augmentations..."
|
||||
|
||||
# WALAugmentation
|
||||
sed -i "/export class WALAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata for logging" \
|
||||
src/augmentations/walAugmentation.ts
|
||||
|
||||
# IndexAugmentation
|
||||
sed -i "/export class IndexAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata to build indexes" \
|
||||
src/augmentations/indexAugmentation.ts
|
||||
|
||||
# MonitoringAugmentation
|
||||
sed -i "/export class MonitoringAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata for monitoring" \
|
||||
src/augmentations/monitoringAugmentation.ts
|
||||
|
||||
# MetricsAugmentation
|
||||
sed -i "/export class MetricsAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata for metrics" \
|
||||
src/augmentations/metricsAugmentation.ts
|
||||
|
||||
# BatchProcessingAugmentation
|
||||
sed -i "/export class BatchProcessingAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata for batching decisions" \
|
||||
src/augmentations/batchProcessingAugmentation.ts
|
||||
|
||||
# EntityRegistryAugmentation
|
||||
sed -i "/export class EntityRegistryAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata to register entities" \
|
||||
src/augmentations/entityRegistryAugmentation.ts
|
||||
|
||||
# AutoRegisterEntitiesAugmentation
|
||||
sed -i "/export class AutoRegisterEntitiesAugmentation extends BaseAugmentation {/a\\
|
||||
readonly metadata = 'readonly' as const // Reads metadata for auto-registration" \
|
||||
src/augmentations/entityRegistryAugmentation.ts
|
||||
|
||||
echo "✅ Done updating augmentations!"
|
||||
Loading…
Add table
Add a link
Reference in a new issue