chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
19
.recovery-workspace/dist-backup-20250910-141917/scripts/precomputePatternEmbeddings.d.ts
vendored
Normal file
19
.recovery-workspace/dist-backup-20250910-141917/scripts/precomputePatternEmbeddings.d.ts
vendored
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* 🧠 Pre-compute Pattern Embeddings Script
|
||||
*
|
||||
* This script pre-computes embeddings for all patterns and saves them to disk.
|
||||
* Run this once after adding new patterns to avoid runtime embedding costs.
|
||||
*
|
||||
* How it works:
|
||||
* 1. Load all patterns from library.json
|
||||
* 2. Use Brainy's embedding model to encode each pattern's examples
|
||||
* 3. Average the example embeddings to get a robust pattern representation
|
||||
* 4. Save embeddings to patterns/embeddings.bin for instant loading
|
||||
*
|
||||
* Benefits:
|
||||
* - Pattern matching becomes pure math (cosine similarity)
|
||||
* - No embedding model calls during query processing
|
||||
* - Patterns load instantly with pre-computed vectors
|
||||
*/
|
||||
export {};
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* 🧠 Pre-compute Pattern Embeddings Script
|
||||
*
|
||||
* This script pre-computes embeddings for all patterns and saves them to disk.
|
||||
* Run this once after adding new patterns to avoid runtime embedding costs.
|
||||
*
|
||||
* How it works:
|
||||
* 1. Load all patterns from library.json
|
||||
* 2. Use Brainy's embedding model to encode each pattern's examples
|
||||
* 3. Average the example embeddings to get a robust pattern representation
|
||||
* 4. Save embeddings to patterns/embeddings.bin for instant loading
|
||||
*
|
||||
* Benefits:
|
||||
* - Pattern matching becomes pure math (cosine similarity)
|
||||
* - No embedding model calls during query processing
|
||||
* - Patterns load instantly with pre-computed vectors
|
||||
*/
|
||||
import { BrainyData } from '../brainyData.js';
|
||||
import patternData from '../patterns/library.json' assert { type: 'json' };
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
async function precomputeEmbeddings() {
|
||||
console.log('🧠 Pre-computing pattern embeddings...');
|
||||
// Initialize Brainy with minimal config
|
||||
const brain = new BrainyData({
|
||||
storage: { forceMemoryStorage: true },
|
||||
logging: { verbose: false }
|
||||
});
|
||||
await brain.init();
|
||||
console.log('✅ Brainy initialized');
|
||||
const embeddings = {};
|
||||
let processedCount = 0;
|
||||
const totalPatterns = patternData.patterns.length;
|
||||
for (const pattern of patternData.patterns) {
|
||||
console.log(`\n📝 Processing pattern: ${pattern.id} (${++processedCount}/${totalPatterns})`);
|
||||
console.log(` Category: ${pattern.category}`);
|
||||
console.log(` Examples: ${pattern.examples.length}`);
|
||||
// Embed all examples
|
||||
const exampleEmbeddings = [];
|
||||
for (const example of pattern.examples) {
|
||||
try {
|
||||
const embedding = await brain.embed(example);
|
||||
exampleEmbeddings.push(embedding);
|
||||
console.log(` ✓ Embedded: "${example.substring(0, 50)}..."`);
|
||||
}
|
||||
catch (error) {
|
||||
console.error(` ✗ Failed to embed: "${example}"`, error);
|
||||
}
|
||||
}
|
||||
if (exampleEmbeddings.length === 0) {
|
||||
console.warn(` ⚠️ No embeddings generated for pattern ${pattern.id}`);
|
||||
continue;
|
||||
}
|
||||
// Average the embeddings for a robust representation
|
||||
const avgEmbedding = averageVectors(exampleEmbeddings);
|
||||
embeddings[pattern.id] = {
|
||||
patternId: pattern.id,
|
||||
embedding: avgEmbedding,
|
||||
examples: pattern.examples,
|
||||
averageMethod: 'arithmetic_mean'
|
||||
};
|
||||
console.log(` ✅ Generated ${avgEmbedding.length}-dimensional embedding`);
|
||||
}
|
||||
// Save embeddings to file
|
||||
const outputPath = path.join(process.cwd(), 'src', 'patterns', 'embeddings.json');
|
||||
await fs.writeFile(outputPath, JSON.stringify(embeddings, null, 2));
|
||||
console.log(`\n✅ Saved ${Object.keys(embeddings).length} pattern embeddings to ${outputPath}`);
|
||||
// Calculate storage size
|
||||
const stats = await fs.stat(outputPath);
|
||||
console.log(`📊 File size: ${(stats.size / 1024).toFixed(2)} KB`);
|
||||
// Print statistics
|
||||
console.log('\n📈 Embedding Statistics:');
|
||||
console.log(` Total patterns: ${totalPatterns}`);
|
||||
console.log(` Successfully embedded: ${Object.keys(embeddings).length}`);
|
||||
console.log(` Failed: ${totalPatterns - Object.keys(embeddings).length}`);
|
||||
console.log(` Embedding dimensions: ${Object.values(embeddings)[0]?.embedding.length || 0}`);
|
||||
await brain.close();
|
||||
console.log('\n✅ Complete!');
|
||||
}
|
||||
function averageVectors(vectors) {
|
||||
if (vectors.length === 0)
|
||||
return [];
|
||||
const dim = vectors[0].length;
|
||||
const avg = new Array(dim).fill(0);
|
||||
// Sum all vectors
|
||||
for (const vec of vectors) {
|
||||
for (let i = 0; i < dim; i++) {
|
||||
avg[i] += vec[i];
|
||||
}
|
||||
}
|
||||
// Divide by count to get average
|
||||
for (let i = 0; i < dim; i++) {
|
||||
avg[i] /= vectors.length;
|
||||
}
|
||||
return avg;
|
||||
}
|
||||
// Run the script
|
||||
precomputeEmbeddings().catch(console.error);
|
||||
//# sourceMappingURL=precomputePatternEmbeddings.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"precomputePatternEmbeddings.js","sourceRoot":"","sources":["../../src/scripts/precomputePatternEmbeddings.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,WAAW,MAAM,0BAA0B,CAAC,SAAS,IAAI,EAAE,MAAM,EAAE,CAAA;AAC1E,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAE5B,KAAK,UAAU,oBAAoB;IACjC,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAA;IAErD,wCAAwC;IACxC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QAC3B,OAAO,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE;QACrC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;KAC5B,CAAC,CAAA;IAEF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;IAClB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IAEnC,MAAM,UAAU,GAKX,EAAE,CAAA;IAEP,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,MAAM,aAAa,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAA;IAEjD,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,CAAC,EAAE,KAAK,EAAE,cAAc,IAAI,aAAa,GAAG,CAAC,CAAA;QAC5F,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC/C,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QAEtD,qBAAqB;QACrB,MAAM,iBAAiB,GAAe,EAAE,CAAA;QAExC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;gBAC5C,iBAAiB,CAAC,IAAI,CAAC,SAAqB,CAAC,CAAA;gBAC7C,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAA;YAChE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,GAAG,EAAE,KAAK,CAAC,CAAA;YAC5D,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC,6CAA6C,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;YACvE,SAAQ;QACV,CAAC;QAED,qDAAqD;QACrD,MAAM,YAAY,GAAG,cAAc,CAAC,iBAAiB,CAAC,CAAA;QAEtD,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;YACvB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,SAAS,EAAE,YAAY;YACvB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,aAAa,EAAE,iBAAiB;SACjC,CAAA;QAED,OAAO,CAAC,GAAG,CAAC,kBAAkB,YAAY,CAAC,MAAM,wBAAwB,CAAC,CAAA;IAC5E,CAAC;IAED,0BAA0B;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,CAAC,CAAA;IACjF,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;IAEnE,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,0BAA0B,UAAU,EAAE,CAAC,CAAA;IAE9F,yBAAyB;IACzB,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAEjE,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAA;IACzC,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,EAAE,CAAC,CAAA;IAClD,OAAO,CAAC,GAAG,CAAC,6BAA6B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;IAC1E,OAAO,CAAC,GAAG,CAAC,cAAc,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3E,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,CAAA;IAE9F,MAAM,KAAK,CAAC,KAAK,EAAE,CAAA;IACnB,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;AAC9B,CAAC;AAED,SAAS,cAAc,CAAC,OAAmB;IACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAEnC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAC7B,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAElC,kBAAkB;IAClB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,iBAAiB;AACjB,oBAAoB,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA"}
|
||||
Loading…
Add table
Add a link
Reference in a new issue