David Snelling
8ff382ca3b
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
316 lines
No EOL
14 KiB
JavaScript
316 lines
No EOL
14 KiB
JavaScript
/**
|
|
* Neural Entity Extractor using Brainy's NounTypes
|
|
* Uses embeddings and similarity matching for accurate type detection
|
|
*/
|
|
import { NounType } from '../types/graphTypes.js';
|
|
export class NeuralEntityExtractor {
|
|
constructor(brain) {
|
|
// Type embeddings for similarity matching
|
|
this.typeEmbeddings = new Map();
|
|
this.initialized = false;
|
|
this.brain = brain;
|
|
}
|
|
/**
|
|
* Initialize type embeddings for neural matching
|
|
*/
|
|
async initializeTypeEmbeddings() {
|
|
if (this.initialized)
|
|
return;
|
|
// Create representative embeddings for each NounType
|
|
const typeExamples = {
|
|
[NounType.Person]: ['John Smith', 'Jane Doe', 'person', 'individual', 'human'],
|
|
[NounType.Organization]: ['Microsoft Corporation', 'company', 'organization', 'business', 'enterprise'],
|
|
[NounType.Location]: ['New York City', 'location', 'place', 'address', 'geography'],
|
|
[NounType.Document]: ['document', 'file', 'report', 'paper', 'text'],
|
|
[NounType.Event]: ['conference', 'meeting', 'event', 'occurrence', 'happening'],
|
|
[NounType.Product]: ['iPhone', 'product', 'item', 'merchandise', 'goods'],
|
|
[NounType.Service]: ['consulting', 'service', 'offering', 'provision'],
|
|
[NounType.Concept]: ['idea', 'concept', 'theory', 'principle', 'notion'],
|
|
[NounType.Media]: ['image', 'video', 'audio', 'media', 'content'],
|
|
[NounType.Message]: ['email', 'message', 'communication', 'note'],
|
|
[NounType.Task]: ['task', 'todo', 'assignment', 'job', 'work'],
|
|
[NounType.Project]: ['project', 'initiative', 'program', 'endeavor'],
|
|
[NounType.Process]: ['workflow', 'process', 'procedure', 'method'],
|
|
[NounType.User]: ['user', 'account', 'profile', 'member'],
|
|
[NounType.Role]: ['manager', 'role', 'position', 'title', 'responsibility'],
|
|
[NounType.Topic]: ['subject', 'topic', 'theme', 'matter'],
|
|
[NounType.Language]: ['English', 'language', 'tongue', 'dialect'],
|
|
[NounType.Currency]: ['dollar', 'currency', 'money', 'USD', 'EUR'],
|
|
[NounType.Measurement]: ['meter', 'measurement', 'unit', 'quantity'],
|
|
[NounType.Contract]: ['agreement', 'contract', 'deal', 'treaty'],
|
|
[NounType.Regulation]: ['law', 'regulation', 'rule', 'policy'],
|
|
[NounType.Resource]: ['resource', 'asset', 'material', 'supply'],
|
|
[NounType.Dataset]: ['database', 'dataset', 'data', 'records'],
|
|
[NounType.Interface]: ['API', 'interface', 'endpoint', 'connection'],
|
|
[NounType.Thing]: ['thing', 'object', 'item', 'entity'],
|
|
[NounType.Content]: ['content', 'material', 'information'],
|
|
[NounType.Collection]: ['collection', 'group', 'set', 'list'],
|
|
[NounType.File]: ['file', 'document', 'archive'],
|
|
[NounType.State]: ['state', 'status', 'condition'],
|
|
[NounType.Hypothesis]: ['hypothesis', 'theory', 'assumption'],
|
|
[NounType.Experiment]: ['experiment', 'test', 'trial', 'study']
|
|
};
|
|
// Generate embeddings for each type
|
|
for (const [type, examples] of Object.entries(typeExamples)) {
|
|
const combinedText = examples.join(' ');
|
|
const embedding = await this.getEmbedding(combinedText);
|
|
this.typeEmbeddings.set(type, embedding);
|
|
}
|
|
this.initialized = true;
|
|
}
|
|
/**
|
|
* Extract entities from text using neural matching
|
|
*/
|
|
async extract(text, options) {
|
|
await this.initializeTypeEmbeddings();
|
|
const entities = [];
|
|
const minConfidence = options?.confidence || 0.6;
|
|
const targetTypes = options?.types || Object.values(NounType);
|
|
const useNeuralMatching = options?.neuralMatching !== false; // Default true
|
|
// Step 1: Extract potential entities using patterns
|
|
const candidates = await this.extractCandidates(text);
|
|
// Step 2: Classify each candidate using neural matching
|
|
for (const candidate of candidates) {
|
|
let bestType = NounType.Thing;
|
|
let bestConfidence = 0;
|
|
if (useNeuralMatching) {
|
|
// Get embedding for the candidate
|
|
const candidateVector = await this.getEmbedding(candidate.text);
|
|
// Find best matching NounType
|
|
for (const type of targetTypes) {
|
|
const typeVector = this.typeEmbeddings.get(type);
|
|
if (!typeVector)
|
|
continue;
|
|
const similarity = this.cosineSimilarity(candidateVector, typeVector);
|
|
// Apply context-based boosting
|
|
const contextBoost = this.getContextBoost(candidate.text, candidate.context, type);
|
|
const adjustedConfidence = similarity * (1 + contextBoost);
|
|
if (adjustedConfidence > bestConfidence) {
|
|
bestConfidence = adjustedConfidence;
|
|
bestType = type;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
// Fallback to rule-based classification
|
|
const classification = this.classifyByRules(candidate);
|
|
bestType = classification.type;
|
|
bestConfidence = classification.confidence;
|
|
}
|
|
if (bestConfidence >= minConfidence) {
|
|
const entity = {
|
|
text: candidate.text,
|
|
type: bestType,
|
|
position: candidate.position,
|
|
confidence: bestConfidence
|
|
};
|
|
if (options?.includeVectors) {
|
|
entity.vector = await this.getEmbedding(candidate.text);
|
|
}
|
|
entities.push(entity);
|
|
}
|
|
}
|
|
// Remove duplicates and overlaps
|
|
return this.deduplicateEntities(entities);
|
|
}
|
|
/**
|
|
* Extract candidate entities using patterns
|
|
*/
|
|
async extractCandidates(text) {
|
|
const candidates = [];
|
|
// Enhanced patterns for entity detection
|
|
const patterns = [
|
|
// Capitalized words (potential names, places, organizations)
|
|
/\b([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*)\b/g,
|
|
// Email addresses
|
|
/\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b/g,
|
|
// URLs
|
|
/\b(https?:\/\/[^\s]+|www\.[^\s]+)\b/g,
|
|
// Phone numbers
|
|
/\b(\+?\d{1,3}?[- .]?\(?\d{1,4}\)?[- .]?\d{1,4}[- .]?\d{1,4})\b/g,
|
|
// Dates
|
|
/\b(\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}|\d{4}[\/\-]\d{1,2}[\/\-]\d{1,2})\b/g,
|
|
// Money amounts
|
|
/\b(\$[\d,]+(?:\.\d{2})?|[\d,]+(?:\.\d{2})?\s*(?:USD|EUR|GBP|JPY|CNY))\b/gi,
|
|
// Percentages
|
|
/\b(\d+(?:\.\d+)?%)\b/g,
|
|
// Hashtags and mentions
|
|
/([#@][a-zA-Z0-9_]+)/g,
|
|
// Product versions
|
|
/\b([A-Z][a-zA-Z0-9]+\s+v?\d+(?:\.\d+)*)\b/g,
|
|
// Quoted strings (potential names, titles)
|
|
/"([^"]+)"/g,
|
|
/'([^']+)'/g
|
|
];
|
|
for (const pattern of patterns) {
|
|
let match;
|
|
while ((match = pattern.exec(text)) !== null) {
|
|
const extractedText = match[1] || match[0];
|
|
// Skip too short or too long
|
|
if (extractedText.length < 2 || extractedText.length > 100)
|
|
continue;
|
|
// Get context (surrounding text)
|
|
const contextStart = Math.max(0, match.index - 30);
|
|
const contextEnd = Math.min(text.length, match.index + match[0].length + 30);
|
|
const context = text.substring(contextStart, contextEnd);
|
|
candidates.push({
|
|
text: extractedText,
|
|
position: {
|
|
start: match.index,
|
|
end: match.index + match[0].length
|
|
},
|
|
context
|
|
});
|
|
}
|
|
}
|
|
return candidates;
|
|
}
|
|
/**
|
|
* Get context-based confidence boost for type matching
|
|
*/
|
|
getContextBoost(text, context, type) {
|
|
const contextLower = context.toLowerCase();
|
|
let boost = 0;
|
|
// Context clues for each type
|
|
const contextClues = {
|
|
[NounType.Person]: ['mr', 'ms', 'mrs', 'dr', 'prof', 'said', 'told', 'wrote'],
|
|
[NounType.Organization]: ['inc', 'corp', 'llc', 'ltd', 'company', 'announced'],
|
|
[NounType.Location]: ['in', 'at', 'from', 'to', 'near', 'located', 'city', 'country'],
|
|
[NounType.Document]: ['file', 'document', 'report', 'paper', 'pdf', 'doc'],
|
|
[NounType.Event]: ['event', 'conference', 'meeting', 'summit', 'on', 'at'],
|
|
[NounType.Product]: ['product', 'version', 'release', 'model', 'buy', 'sell'],
|
|
[NounType.Currency]: ['$', '€', '£', '¥', 'usd', 'eur', 'price', 'cost'],
|
|
[NounType.Message]: ['email', 'message', 'sent', 'received', 'wrote', 'reply'],
|
|
// Add more context clues as needed
|
|
};
|
|
const clues = contextClues[type] || [];
|
|
for (const clue of clues) {
|
|
if (contextLower.includes(clue)) {
|
|
boost += 0.1;
|
|
}
|
|
}
|
|
return Math.min(boost, 0.3); // Cap boost at 0.3
|
|
}
|
|
/**
|
|
* Rule-based classification fallback
|
|
*/
|
|
classifyByRules(candidate) {
|
|
const text = candidate.text;
|
|
// Email
|
|
if (text.includes('@')) {
|
|
return { type: NounType.Message, confidence: 0.9 };
|
|
}
|
|
// URL
|
|
if (text.startsWith('http') || text.startsWith('www.')) {
|
|
return { type: NounType.Resource, confidence: 0.9 };
|
|
}
|
|
// Money
|
|
if (text.startsWith('$') || /\d+\.\d{2}/.test(text)) {
|
|
return { type: NounType.Currency, confidence: 0.85 };
|
|
}
|
|
// Percentage
|
|
if (text.endsWith('%')) {
|
|
return { type: NounType.Measurement, confidence: 0.85 };
|
|
}
|
|
// Date pattern
|
|
if (/\d{1,2}[\/\-]\d{1,2}/.test(text)) {
|
|
return { type: NounType.Event, confidence: 0.7 };
|
|
}
|
|
// Hashtag
|
|
if (text.startsWith('#')) {
|
|
return { type: NounType.Topic, confidence: 0.8 };
|
|
}
|
|
// Mention
|
|
if (text.startsWith('@')) {
|
|
return { type: NounType.User, confidence: 0.8 };
|
|
}
|
|
// Capitalized words (likely proper nouns)
|
|
if (/^[A-Z]/.test(text)) {
|
|
// Multiple words - likely organization or person
|
|
const words = text.split(/\s+/);
|
|
if (words.length > 1) {
|
|
// Check for organization suffixes
|
|
if (/\b(Inc|Corp|LLC|Ltd|Co|Group|Foundation|University)\b/i.test(text)) {
|
|
return { type: NounType.Organization, confidence: 0.75 };
|
|
}
|
|
// Likely a person's name
|
|
return { type: NounType.Person, confidence: 0.65 };
|
|
}
|
|
// Single capitalized word - could be location
|
|
return { type: NounType.Location, confidence: 0.5 };
|
|
}
|
|
// Default to Thing with low confidence
|
|
return { type: NounType.Thing, confidence: 0.3 };
|
|
}
|
|
/**
|
|
* Get embedding for text
|
|
*/
|
|
async getEmbedding(text) {
|
|
if ('embed' in this.brain && typeof this.brain.embed === 'function') {
|
|
return await this.brain.embed(text);
|
|
}
|
|
else {
|
|
// Fallback - create simple hash-based vector
|
|
const vector = new Array(384).fill(0);
|
|
for (let i = 0; i < text.length; i++) {
|
|
vector[i % 384] += text.charCodeAt(i) / 255;
|
|
}
|
|
return vector.map(v => v / text.length);
|
|
}
|
|
}
|
|
/**
|
|
* Calculate cosine similarity between vectors
|
|
*/
|
|
cosineSimilarity(a, b) {
|
|
let dotProduct = 0;
|
|
let normA = 0;
|
|
let normB = 0;
|
|
for (let i = 0; i < a.length; i++) {
|
|
dotProduct += a[i] * b[i];
|
|
normA += a[i] * a[i];
|
|
normB += b[i] * b[i];
|
|
}
|
|
normA = Math.sqrt(normA);
|
|
normB = Math.sqrt(normB);
|
|
if (normA === 0 || normB === 0)
|
|
return 0;
|
|
return dotProduct / (normA * normB);
|
|
}
|
|
/**
|
|
* Simple hash function for fallback
|
|
*/
|
|
simpleHash(text) {
|
|
let hash = 0;
|
|
for (let i = 0; i < text.length; i++) {
|
|
const char = text.charCodeAt(i);
|
|
hash = ((hash << 5) - hash) + char;
|
|
hash = hash & hash; // Convert to 32-bit integer
|
|
}
|
|
return Math.abs(hash);
|
|
}
|
|
/**
|
|
* Remove duplicate and overlapping entities
|
|
*/
|
|
deduplicateEntities(entities) {
|
|
// Sort by position and confidence
|
|
entities.sort((a, b) => {
|
|
if (a.position.start !== b.position.start) {
|
|
return a.position.start - b.position.start;
|
|
}
|
|
return b.confidence - a.confidence; // Higher confidence first
|
|
});
|
|
const result = [];
|
|
for (const entity of entities) {
|
|
// Check for overlap with already added entities
|
|
const hasOverlap = result.some(existing => (entity.position.start >= existing.position.start &&
|
|
entity.position.start < existing.position.end) ||
|
|
(entity.position.end > existing.position.start &&
|
|
entity.position.end <= existing.position.end));
|
|
if (!hasOverlap) {
|
|
result.push(entity);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
//# sourceMappingURL=entityExtractor.js.map
|