chore(8.0): final pre-RC1 sweep — API consistency, named errors, orphans, zero-cast codebase
This commit is contained in:
parent
970e08c466
commit
1f7e365a4e
237 changed files with 1951 additions and 49413 deletions
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SmartExtractor - Unified entity type extraction using ensemble of neural signals
|
||||
*
|
||||
* PRODUCTION-READY: Single orchestration class for all entity type classification
|
||||
* Single orchestration class for all entity type classification
|
||||
*
|
||||
* Design Philosophy:
|
||||
* - Simplicity over complexity (KISS principle)
|
||||
|
|
@ -645,7 +645,7 @@ export class SmartExtractor {
|
|||
return {
|
||||
type: best.type!,
|
||||
confidence: best.confidence,
|
||||
source: best.signal as any,
|
||||
source: best.signal,
|
||||
evidence: best.evidence,
|
||||
metadata: {
|
||||
formatHints: formatHints.length > 0 ? formatHints : undefined,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* SmartRelationshipExtractor - Unified relationship type extraction using ensemble of neural signals
|
||||
*
|
||||
* PRODUCTION-READY: Parallel to SmartExtractor but for verbs/relationships
|
||||
* Parallel to SmartExtractor but for verbs/relationships
|
||||
*
|
||||
* Design Philosophy:
|
||||
* - Simplicity over complexity (KISS principle)
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -3,8 +3,6 @@
|
|||
*
|
||||
* Caches entity extraction results to avoid re-processing unchanged content.
|
||||
* Uses file mtime or content hash for invalidation.
|
||||
*
|
||||
* PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
|
||||
*/
|
||||
|
||||
import { ExtractedEntity } from './entityExtractor.js'
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* Uses embeddings and similarity matching for accurate type detection
|
||||
*
|
||||
* Now powered by SmartExtractor for ultra-neural classification
|
||||
* PRODUCTION-READY with caching support
|
||||
* Entity extraction with caching support
|
||||
*/
|
||||
|
||||
import { NounType } from '../types/graphTypes.js'
|
||||
|
|
@ -293,8 +293,8 @@ export class NeuralEntityExtractor {
|
|||
const contextLower = context.toLowerCase()
|
||||
let boost = 0
|
||||
|
||||
// Context clues for each type
|
||||
const contextClues: Record<NounType, string[]> = {
|
||||
// Context clues for each type (partial: only types with known clue words)
|
||||
const contextClues: Partial<Record<NounType, string[]>> = {
|
||||
[NounType.Person]: ['mr', 'ms', 'mrs', 'dr', 'prof', 'said', 'told', 'wrote'],
|
||||
[NounType.Organization]: ['inc', 'corp', 'llc', 'ltd', 'company', 'announced'],
|
||||
[NounType.Location]: ['in', 'at', 'from', 'to', 'near', 'located', 'city', 'country'],
|
||||
|
|
@ -304,7 +304,7 @@ export class NeuralEntityExtractor {
|
|||
[NounType.Currency]: ['$', '€', '£', '¥', 'usd', 'eur', 'price', 'cost'],
|
||||
[NounType.Message]: ['email', 'message', 'sent', 'received', 'wrote', 'reply'],
|
||||
// Add more context clues as needed
|
||||
} as any
|
||||
}
|
||||
|
||||
const clues = contextClues[type] || []
|
||||
for (const clue of clues) {
|
||||
|
|
@ -401,8 +401,8 @@ export class NeuralEntityExtractor {
|
|||
this.embeddingCacheStats.misses++
|
||||
|
||||
let vector: Vector
|
||||
if ('embed' in this.brain && typeof (this.brain as any).embed === 'function') {
|
||||
vector = await (this.brain as any).embed(text)
|
||||
if ('embed' in this.brain && typeof this.brain.embed === 'function') {
|
||||
vector = await this.brain.embed(text)
|
||||
} else {
|
||||
// Fallback - create simple hash-based vector
|
||||
vector = new Array(384).fill(0)
|
||||
|
|
|
|||
|
|
@ -1003,8 +1003,12 @@ export class ImprovedNeuralAPI {
|
|||
return this._createEmptyResult(startTime, 'hierarchical')
|
||||
}
|
||||
|
||||
// Use existing HNSW level structure for natural clustering
|
||||
const level = (options as any).level || this._getOptimalClusteringLevel(itemIds.length)
|
||||
// Use existing HNSW level structure for natural clustering.
|
||||
// `level` is an undocumented expert override (not on ClusteringOptions):
|
||||
// it pins the HNSW layer used for cluster centers instead of deriving it.
|
||||
const level =
|
||||
(options as ClusteringOptions & { level?: number }).level ||
|
||||
this._getOptimalClusteringLevel(itemIds.length)
|
||||
const maxClusters = options.maxClusters || Math.min(50, Math.ceil(itemIds.length / 20))
|
||||
|
||||
// Get HNSW level representatives - these are natural cluster centers
|
||||
|
|
@ -1618,7 +1622,7 @@ export class ImprovedNeuralAPI {
|
|||
|
||||
// Get all verbs connecting the items
|
||||
for (const sourceId of itemIds) {
|
||||
const sourceVerbs = await this.brain.getRelations(sourceId)
|
||||
const sourceVerbs = await this.brain.related(sourceId)
|
||||
|
||||
for (const verb of sourceVerbs) {
|
||||
const targetId = verb.to
|
||||
|
|
@ -1882,8 +1886,9 @@ export class ImprovedNeuralAPI {
|
|||
return await this.brain.counts.entities()
|
||||
}
|
||||
|
||||
// Fallback: Get from storage statistics
|
||||
const storage = (this.brain as any).storage
|
||||
// Fallback: Get from storage statistics (brain is untyped here; storage is
|
||||
// a private Brainy field reached for its optional getStatistics capability)
|
||||
const storage = this.brain.storage
|
||||
if (storage && typeof storage.getStatistics === 'function') {
|
||||
const stats = await storage.getStatistics()
|
||||
return stats?.totalNodes || 0
|
||||
|
|
@ -2836,7 +2841,7 @@ export class ImprovedNeuralAPI {
|
|||
private async _getImportantSample(itemIds: string[], sampleSize: number): Promise<string[]> {
|
||||
const items = await Promise.all(
|
||||
itemIds.map(async id => {
|
||||
const verbs = await this.brain.getRelations(id)
|
||||
const verbs = await this.brain.related(id)
|
||||
const noun = await this.brain.get(id)
|
||||
|
||||
// Calculate importance score
|
||||
|
|
@ -3436,7 +3441,7 @@ export class ImprovedNeuralAPI {
|
|||
const sampleSize = Math.min(50, itemIds.length)
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
try {
|
||||
const verbs = await this.brain.getRelations({ from: itemIds[i] })
|
||||
const verbs = await this.brain.related({ from: itemIds[i] })
|
||||
connectionCount += verbs.length
|
||||
} catch (error) {
|
||||
// Skip items that can't be processed
|
||||
|
|
@ -3506,7 +3511,7 @@ export class ImprovedNeuralAPI {
|
|||
if (fromType !== toType) {
|
||||
for (const fromItem of fromItems.slice(0, 10)) { // Sample to avoid N^2
|
||||
try {
|
||||
const verbs = await this.brain.getRelations({ from: fromItem.id })
|
||||
const verbs = await this.brain.related({ from: fromItem.id })
|
||||
|
||||
for (const verb of verbs) {
|
||||
const toItem = toItems.find(item => item.id === verb.to)
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ export class NaturalLanguageProcessor {
|
|||
}
|
||||
|
||||
// Use brain's embed method directly to avoid recursion
|
||||
const embedding = await (this.brain as any).embed(text)
|
||||
const embedding = await this.brain.embed(text)
|
||||
|
||||
// Cache the embedding
|
||||
this.embeddingCache.set(text, embedding)
|
||||
|
|
@ -1161,8 +1161,8 @@ export class NaturalLanguageProcessor {
|
|||
const extractor = new NeuralEntityExtractor(this.brain)
|
||||
|
||||
// Convert string types to NounTypes if provided
|
||||
const nounTypes = options?.types ?
|
||||
options.types.map(t => t as any) :
|
||||
const nounTypes = options?.types ?
|
||||
options.types.map(t => t as NounType) :
|
||||
undefined
|
||||
|
||||
// Extract using neural matching
|
||||
|
|
@ -1253,7 +1253,7 @@ export class NaturalLanguageProcessor {
|
|||
const confidence = this.calculateConfidence(extractedText, type)
|
||||
|
||||
if (confidence >= minConfidence) {
|
||||
const entity = {
|
||||
const entity: (typeof extracted)[number] = {
|
||||
text: extractedText,
|
||||
type,
|
||||
position: {
|
||||
|
|
@ -1264,7 +1264,7 @@ export class NaturalLanguageProcessor {
|
|||
}
|
||||
|
||||
if (options?.includeMetadata) {
|
||||
(entity as any).metadata = {
|
||||
entity.metadata = {
|
||||
pattern: pattern.source,
|
||||
contextBefore: text.substring(Math.max(0, match.index - 20), match.index),
|
||||
contextAfter: text.substring(match.index + match[0].length, Math.min(text.length, match.index + match[0].length + 20))
|
||||
|
|
|
|||
|
|
@ -646,7 +646,7 @@ export class NeuralAPI {
|
|||
|
||||
// Run k-means iterations (simplified)
|
||||
for (let iter = 0; iter < 10; iter++) {
|
||||
const clusters = Array(k).fill(null).map(() => [])
|
||||
const clusters: (typeof items)[] = Array(k).fill(null).map(() => [])
|
||||
|
||||
// Assign items to nearest centroid
|
||||
for (const item of items) {
|
||||
|
|
@ -661,7 +661,7 @@ export class NeuralAPI {
|
|||
}
|
||||
}
|
||||
|
||||
(clusters as any[])[bestCluster].push(item)
|
||||
clusters[bestCluster].push(item)
|
||||
}
|
||||
|
||||
// Update centroids
|
||||
|
|
|
|||
|
|
@ -36,6 +36,11 @@ export interface DetectedEntity {
|
|||
suggestedId: string
|
||||
reasoning: string
|
||||
alternativeTypes: Array<{ type: string, confidence: number }>
|
||||
/**
|
||||
* Subtype tag set by the extractor, if any. Takes precedence over
|
||||
* `NeuralImportOptions.defaultSubtype` during import execution.
|
||||
*/
|
||||
subtype?: string
|
||||
}
|
||||
|
||||
export interface DetectedRelationship {
|
||||
|
|
@ -47,6 +52,11 @@ export interface DetectedRelationship {
|
|||
reasoning: string
|
||||
context: string
|
||||
metadata?: Record<string, any>
|
||||
/**
|
||||
* Subtype tag set by the extractor, if any. Takes precedence over
|
||||
* `NeuralImportOptions.defaultSubtype` during import execution.
|
||||
*/
|
||||
subtype?: string
|
||||
}
|
||||
|
||||
export interface NeuralInsight {
|
||||
|
|
@ -791,7 +801,7 @@ export class NeuralImport {
|
|||
await this.brainy.add({
|
||||
data: this.extractMainText(entity.originalData),
|
||||
type: entity.nounType as NounType,
|
||||
subtype: (entity as any).subtype ?? options.defaultSubtype ?? 'extracted',
|
||||
subtype: entity.subtype ?? options.defaultSubtype ?? 'extracted',
|
||||
metadata: {
|
||||
...entity.originalData,
|
||||
confidence: entity.confidence,
|
||||
|
|
@ -806,7 +816,7 @@ export class NeuralImport {
|
|||
from: relationship.sourceId,
|
||||
to: relationship.targetId,
|
||||
type: relationship.verbType as VerbType,
|
||||
subtype: (relationship as any).subtype ?? options.defaultSubtype ?? 'extracted',
|
||||
subtype: relationship.subtype ?? options.defaultSubtype ?? 'extracted',
|
||||
weight: relationship.weight,
|
||||
confidence: relationship.confidence, // reserved field — dedicated param, not metadata
|
||||
metadata: {
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ export class NeuralImportAugmentation {
|
|||
next: () => Promise<T>
|
||||
): Promise<T> {
|
||||
// Only process on add operations
|
||||
if (!this.operations.includes(operation as any)) {
|
||||
if (!this.operations.includes(operation)) {
|
||||
return next()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ export class PatternLibrary {
|
|||
}
|
||||
|
||||
// Use brain's embed method directly to avoid recursion
|
||||
const embedding = await (this.brain as any).embed(text)
|
||||
const embedding = await this.brain.embed(text)
|
||||
|
||||
this.embeddingCache.set(text, embedding)
|
||||
return embedding
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
/**
|
||||
* Core Pattern Library with Pre-computed Embeddings
|
||||
*
|
||||
* This file is auto-generated by scripts/buildPatterns.ts
|
||||
* DO NOT EDIT MANUALLY - edit src/patterns/comprehensive-library.json instead
|
||||
*
|
||||
* Storage strategy:
|
||||
* - Patterns are bundled directly into Brainy for zero-latency access
|
||||
* - Embeddings are pre-computed and stored as binary Float32Array
|
||||
* - Total size: ~140KB (negligible for a neural library)
|
||||
* - No external files needed, works in all environments
|
||||
*/
|
||||
|
||||
import type { Pattern } from './patternLibrary.js'
|
||||
|
||||
// Pattern data embedded directly for reliability
|
||||
export const CORE_PATTERNS: Pattern[] = [
|
||||
// Informational queries
|
||||
{
|
||||
id: "info_what_is",
|
||||
category: "informational",
|
||||
examples: ["what is artificial intelligence", "what is machine learning"],
|
||||
pattern: "what is (.+)",
|
||||
template: { like: "${1}" },
|
||||
confidence: 0.9
|
||||
},
|
||||
{
|
||||
id: "info_how_does",
|
||||
category: "informational",
|
||||
examples: ["how does neural network work", "how does deep learning work"],
|
||||
pattern: "how does (.+) work",
|
||||
template: { like: "${1}" },
|
||||
confidence: 0.85
|
||||
},
|
||||
// ... more patterns loaded from library.json at build time
|
||||
]
|
||||
|
||||
// Pre-computed embeddings as binary data
|
||||
// Generated by scripts/buildPatterns.ts using Brainy's embedding model
|
||||
export const PATTERN_EMBEDDINGS_BINARY: Uint8Array | null = null // Will be populated at build
|
||||
|
||||
// Helper to decode embeddings
|
||||
export function getPatternEmbeddings(): Map<string, Float32Array> {
|
||||
if (!PATTERN_EMBEDDINGS_BINARY) {
|
||||
return new Map() // Will compute at runtime if not pre-built
|
||||
}
|
||||
|
||||
const embeddings = new Map<string, Float32Array>()
|
||||
const view = new DataView(PATTERN_EMBEDDINGS_BINARY.buffer)
|
||||
const embeddingSize = 384 // Standard size
|
||||
|
||||
CORE_PATTERNS.forEach((pattern, index) => {
|
||||
const offset = index * embeddingSize * 4 // 4 bytes per float
|
||||
const embedding = new Float32Array(embeddingSize)
|
||||
|
||||
for (let i = 0; i < embeddingSize; i++) {
|
||||
embedding[i] = view.getFloat32(offset + i * 4, true)
|
||||
}
|
||||
|
||||
embeddings.set(pattern.id, embedding)
|
||||
})
|
||||
|
||||
return embeddings
|
||||
}
|
||||
|
||||
// Version for cache invalidation
|
||||
export const PATTERNS_VERSION = "2.0.0"
|
||||
|
||||
// Export metadata for monitoring
|
||||
export const PATTERNS_METADATA = {
|
||||
totalPatterns: CORE_PATTERNS.length,
|
||||
categories: [...new Set(CORE_PATTERNS.map(p => p.category))],
|
||||
embeddingDimensions: 384,
|
||||
storageSize: {
|
||||
patterns: "24KB",
|
||||
embeddings: "98KB",
|
||||
total: "122KB"
|
||||
}
|
||||
}
|
||||
|
|
@ -6,8 +6,6 @@
|
|||
* - Entity confidence scores
|
||||
* - Pattern matches
|
||||
* - Structural analysis
|
||||
*
|
||||
* PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
|
||||
*/
|
||||
|
||||
import { ExtractedEntity } from './entityExtractor.js'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* ContextSignal - Relationship-based entity type classification
|
||||
*
|
||||
* PRODUCTION-READY: Infers types from relationship patterns in context
|
||||
* Infers types from relationship patterns in context
|
||||
*
|
||||
* Weight: 5% (lowest, but critical for ambiguous cases)
|
||||
* Speed: Very fast (~1ms) - pure pattern matching on context
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* EmbeddingSignal - Neural entity type classification using embeddings
|
||||
*
|
||||
* PRODUCTION-READY: Merges neural + graph + temporal signals into one
|
||||
* Merges neural + graph + temporal signals into one
|
||||
* 3x faster than separate signals (single embedding lookup)
|
||||
*
|
||||
* Weight: 35% (20% neural + 10% graph + 5% temporal boost)
|
||||
|
|
@ -56,7 +56,7 @@ export interface EmbeddingSignalOptions {
|
|||
interface SourceMatch {
|
||||
type: NounType
|
||||
confidence: number
|
||||
source: string
|
||||
source: TypeSignal['source']
|
||||
metadata?: any
|
||||
}
|
||||
|
||||
|
|
@ -390,7 +390,7 @@ export class EmbeddingSignal {
|
|||
}
|
||||
|
||||
return {
|
||||
source: bestMatches.length > 1 ? 'embedding-combined' : bestMatches[0].source as any,
|
||||
source: bestMatches.length > 1 ? 'embedding-combined' : bestMatches[0].source,
|
||||
type: bestType,
|
||||
confidence: Math.min(bestCombinedScore, 1.0), // Cap at 1.0
|
||||
evidence,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
* Merges: KeywordSignal + PatternSignal from old architecture
|
||||
* Speed: Very fast (~5ms) - pre-compiled patterns
|
||||
*
|
||||
* PRODUCTION-READY: No TODOs, no mocks, real implementation
|
||||
*/
|
||||
|
||||
import type { Brainy } from '../../brainy.js'
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
* 2. Semantic compatibility (Document+Person → CreatedBy)
|
||||
* 3. Domain heuristics (Location+Organization → LocatedAt)
|
||||
*
|
||||
* PRODUCTION-READY: No TODOs, no mocks, real implementation
|
||||
*/
|
||||
|
||||
import type { Brainy } from '../../brainy.js'
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
* 2. Cosine similarity against context text
|
||||
* 3. Semantic understanding of relationship intent
|
||||
*
|
||||
* PRODUCTION-READY: No TODOs, no mocks, real implementation
|
||||
*/
|
||||
|
||||
import type { Brainy } from '../../brainy.js'
|
||||
|
|
@ -89,7 +88,12 @@ export class VerbEmbeddingSignal {
|
|||
|
||||
constructor(brain: Brainy, options?: VerbEmbeddingSignalOptions) {
|
||||
this.brain = brain
|
||||
this.distanceFn = (brain as any).distance || cosineDistance
|
||||
// Visibility boundary: Brainy keeps its distance function in a private
|
||||
// field. The signal reuses it when present so verb classification matches
|
||||
// the index geometry, falling back to cosine distance otherwise.
|
||||
this.distanceFn =
|
||||
(brain as unknown as { distance?: (a: Vector, b: Vector) => number }).distance ||
|
||||
cosineDistance
|
||||
this.options = {
|
||||
minConfidence: options?.minConfidence ?? 0.60,
|
||||
minSimilarity: options?.minSimilarity ?? 0.55,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
* 2. Prepositional phrase patterns ("in", "at", "by", "of")
|
||||
* 3. Structural patterns (parentheses, commas, formatting)
|
||||
*
|
||||
* PRODUCTION-READY: No TODOs, no mocks, real implementation
|
||||
*/
|
||||
|
||||
import type { Brainy } from '../../brainy.js'
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@ const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p]))
|
|||
const patternEmbeddings = getPatternEmbeddings()
|
||||
|
||||
/**
|
||||
* Cosine similarity between two vectors
|
||||
* Cosine similarity between two vectors.
|
||||
* Accepts any number-indexed sequence so pre-computed Float32Array pattern
|
||||
* embeddings can be compared against number[] query vectors without copying.
|
||||
*/
|
||||
function cosineSimilarity(a: Vector, b: Vector): number {
|
||||
function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number {
|
||||
if (!a || !b || a.length !== b.length) return 0
|
||||
|
||||
let dotProduct = 0
|
||||
|
|
@ -99,7 +101,7 @@ export function findBestPatterns(
|
|||
if (!patternEmbedding) continue
|
||||
|
||||
// Pass Float32Array directly, no need for Array.from()!
|
||||
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding as any)
|
||||
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding)
|
||||
if (similarity > 0.5) { // Threshold for relevance
|
||||
matches.push({ pattern, similarity })
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue