fix: update all Stage 2 references to Stage 3 CANONICAL type counts
Comprehensive update of type count references from Stage 2 (31 nouns + 40 verbs) to Stage 3 CANONICAL (42 nouns + 127 verbs) across entire codebase. Changes (23 files): - Core architecture: Memory tracking comments, speedup calculations - Tests: Type count assertions, enum index expectations, memory benchmarks - CLI: User-visible type count output - Augmentations: Type detection comments - Documentation: Architecture docs, guides, performance docs - Type embeddings: Regenerated for all 169 types (338KB) Specific updates: - 31 → 42 (noun count): 38 occurrences - 40 → 127 (verb count): 24 occurrences - 124 → 168 bytes (noun array size): 5 occurrences - 160 → 508 bytes (verb array size): 5 occurrences - 284 → 676 bytes (total type tracking): 12 occurrences - Enum indices updated to match Stage 3 reordering Type embeddings regenerated: - 42 noun embeddings (64.5 KB) - 127 verb embeddings (194.8 KB) - Total: 338 KB (was 108.8 KB) All constants, arrays, and tests now consistent with Stage 3 taxonomy. Fixes #v5.5.1-type-count-migration
This commit is contained in:
parent
f57732be90
commit
823cd5cf1b
23 changed files with 106 additions and 106 deletions
|
|
@ -23,7 +23,7 @@ export const UNIVERSAL_FIELD_PATTERNS: FieldPattern[] = [
|
|||
fields: ['firstName', 'lastName', 'fullName', 'realName'],
|
||||
displayField: 'title',
|
||||
confidence: 0.9,
|
||||
applicableTypes: [NounType.Person, NounType.Person],
|
||||
applicableTypes: [NounType.Person],
|
||||
transform: (value: any, context: FieldComputationContext) => {
|
||||
const { metadata } = context
|
||||
if (metadata.firstName && metadata.lastName) {
|
||||
|
|
@ -72,7 +72,7 @@ export const UNIVERSAL_FIELD_PATTERNS: FieldPattern[] = [
|
|||
fields: ['bio', 'biography', 'profile', 'about'],
|
||||
displayField: 'description',
|
||||
confidence: 0.85,
|
||||
applicableTypes: [NounType.Person, NounType.Person]
|
||||
applicableTypes: [NounType.Person]
|
||||
},
|
||||
{
|
||||
fields: ['content', 'text', 'body', 'message'],
|
||||
|
|
@ -105,7 +105,7 @@ export const UNIVERSAL_FIELD_PATTERNS: FieldPattern[] = [
|
|||
fields: ['role', 'position', 'jobTitle', 'occupation'],
|
||||
displayField: 'type',
|
||||
confidence: 0.8,
|
||||
applicableTypes: [NounType.Person, NounType.Person],
|
||||
applicableTypes: [NounType.Person],
|
||||
transform: (value: any) => String(value || 'Person')
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
/**
|
||||
* Universal Display Augmentation - Intelligent Computation Engine
|
||||
*
|
||||
*
|
||||
* Leverages existing Brainy AI infrastructure for intelligent field computation:
|
||||
* - BrainyTypes for semantic type detection
|
||||
* - Neural Import patterns for field analysis
|
||||
* - Neural Import patterns for field analysis
|
||||
* - JSON processing utilities for field extraction
|
||||
* - Existing NounType/VerbType taxonomy (31+40 types)
|
||||
* - Existing NounType/VerbType taxonomy (42+127 types)
|
||||
*/
|
||||
|
||||
import type {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
/**
|
||||
* BrainyTypes - Intelligent type detection using semantic embeddings
|
||||
*
|
||||
*
|
||||
* This module uses our existing TransformerEmbedding and similarity functions
|
||||
* to intelligently match data to our 31 noun types and 40 verb types.
|
||||
*
|
||||
* to intelligently match data to our 42 noun types and 127 verb types.
|
||||
*
|
||||
* Features:
|
||||
* - Semantic similarity matching using embeddings
|
||||
* - Context-aware type detection
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
/**
|
||||
* IntelligentTypeMatcher - Wrapper around BrainyTypes for testing
|
||||
*
|
||||
*
|
||||
* Provides intelligent type detection using semantic embeddings
|
||||
* for matching data to our 31 noun types and 40 verb types.
|
||||
* for matching data to our 42 noun types and 127 verb types.
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from '../../types/graphTypes.js'
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
/**
|
||||
* Universal Display Augmentation
|
||||
*
|
||||
*
|
||||
* 🎨 Provides intelligent display fields for any noun or verb using AI-powered analysis
|
||||
*
|
||||
*
|
||||
* Features:
|
||||
* - ✅ Leverages existing BrainyTypes for semantic type detection
|
||||
* - ✅ Complete icon coverage for all 31 NounTypes + 40+ VerbTypes
|
||||
* - ✅ Complete icon coverage for all 42 NounTypes + 127 VerbTypes
|
||||
* - ✅ Zero performance impact with lazy computation and intelligent caching
|
||||
* - ✅ Perfect isolation - can be disabled, replaced, or configured
|
||||
* - ✅ Clean developer experience with zero conflicts
|
||||
|
|
|
|||
|
|
@ -3901,7 +3901,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
},
|
||||
|
||||
// Phase 1b: O(1) count by type enum (Uint32Array-based, more efficient)
|
||||
// Uses fixed-size type tracking: 284 bytes vs ~35KB with Maps (99.2% reduction)
|
||||
// Uses fixed-size type tracking: 676 bytes vs ~35KB with Maps (98.1% reduction)
|
||||
byTypeEnum: (type: NounType) => {
|
||||
return this.metadataIndex.getEntityCountByTypeEnum(type)
|
||||
},
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
|
|||
|
||||
// Display nouns
|
||||
if (showNouns) {
|
||||
console.log(chalk.bold.cyan('\n📚 Noun Types (31):\n'))
|
||||
console.log(chalk.bold.cyan('\n📚 Noun Types (42):\n'))
|
||||
const nounChunks = []
|
||||
for (let i = 0; i < BrainyTypes.nouns.length; i += 3) {
|
||||
nounChunks.push(BrainyTypes.nouns.slice(i, i + 3))
|
||||
|
|
@ -43,7 +43,7 @@ export async function types(options: { json?: boolean, noun?: boolean, verb?: bo
|
|||
|
||||
// Display verbs
|
||||
if (showVerbs) {
|
||||
console.log(chalk.bold.cyan('\n🔗 Verb Types (40):\n'))
|
||||
console.log(chalk.bold.cyan('\n🔗 Verb Types (127):\n'))
|
||||
const verbChunks = []
|
||||
for (let i = 0; i < BrainyTypes.verbs.length; i += 3) {
|
||||
verbChunks.push(BrainyTypes.verbs.slice(i, i + 3))
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
* - Storage: Already type-first from Phase 1a
|
||||
*
|
||||
* Architecture:
|
||||
* - One HNSWIndex per NounType (31 total)
|
||||
* - One HNSWIndex per NounType (42 total)
|
||||
* - Lazy initialization (indexes created on first use)
|
||||
* - Type routing for optimal performance
|
||||
* - Falls back to multi-type search when type unknown
|
||||
|
|
@ -128,7 +128,7 @@ export class TypeAwareHNSWIndex {
|
|||
const typeIndex = TypeUtils.getNounIndex(type)
|
||||
if (typeIndex === undefined || typeIndex === null || typeIndex < 0) {
|
||||
throw new Error(
|
||||
`Invalid NounType: ${type}. Must be one of the 31 defined types.`
|
||||
`Invalid NounType: ${type}. Must be one of the 42 defined types.`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -193,7 +193,7 @@ export class TypeAwareHNSWIndex {
|
|||
* **All-types search** (fallback):
|
||||
* ```typescript
|
||||
* await index.search(queryVector, 10)
|
||||
* // Searches all 31 graphs (slower but comprehensive)
|
||||
* // Searches all 42 graphs (slower but comprehensive)
|
||||
* ```
|
||||
*
|
||||
* @param queryVector Query vector
|
||||
|
|
@ -393,7 +393,7 @@ export class TypeAwareHNSWIndex {
|
|||
* Rebuild HNSW indexes from storage (type-aware)
|
||||
*
|
||||
* CRITICAL: This implementation uses type-filtered pagination to avoid
|
||||
* loading ALL entities for each type (which would be 31 billion reads @ 1B scale).
|
||||
* loading ALL entities for each type (which would be 42 billion reads @ 1B scale).
|
||||
*
|
||||
* Can rebuild all types or specific types.
|
||||
* Much faster than rebuilding a monolithic index.
|
||||
|
|
@ -451,7 +451,7 @@ export class TypeAwareHNSWIndex {
|
|||
}
|
||||
|
||||
// Load ALL nouns ONCE and route to correct type indexes
|
||||
// This is O(N) instead of O(31*N) from the previous parallel approach
|
||||
// This is O(N) instead of O(42*N) from the previous parallel approach
|
||||
let cursor: string | undefined = undefined
|
||||
let hasMore = true
|
||||
let totalLoaded = 0
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ export class SmartExcelImporter {
|
|||
enableRelationshipInference: true,
|
||||
// CONCEPT EXTRACTION PRODUCTION-READY (v3.33.0+):
|
||||
// Type embeddings are now pre-computed at build time - zero runtime cost!
|
||||
// All 31 noun types + 40 verb types instantly available
|
||||
// All 42 noun types + 127 verb types instantly available
|
||||
//
|
||||
// Performance profile:
|
||||
// - Type embeddings: INSTANT (pre-computed at build time, ~100KB in-memory)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: 2025-11-06T16:58:34.845Z
|
||||
* Generated: 2025-11-06T17:38:22.619Z
|
||||
* Noun Types: 42
|
||||
* Verb Types: 127
|
||||
*
|
||||
|
|
@ -19,7 +19,7 @@ export const TYPE_METADATA = {
|
|||
verbTypes: 127,
|
||||
totalTypes: 169,
|
||||
embeddingDimensions: 384,
|
||||
generatedAt: "2025-11-06T16:58:34.845Z",
|
||||
generatedAt: "2025-11-06T17:38:22.619Z",
|
||||
sizeBytes: {
|
||||
embeddings: 259584,
|
||||
base64: 346112
|
||||
|
|
|
|||
|
|
@ -6,14 +6,14 @@
|
|||
* TypeAwareHNSWIndex graphs.
|
||||
*
|
||||
* Performance Impact:
|
||||
* - Single-type queries: 31x speedup (search 1/31 graphs)
|
||||
* - Multi-type queries: 6-15x speedup (search 2-5/31 graphs)
|
||||
* - Single-type queries: 42x speedup (search 1/42 graphs)
|
||||
* - Multi-type queries: 8-21x speedup (search 2-5/42 graphs)
|
||||
* - Overall: 40% latency reduction @ 1B scale
|
||||
*
|
||||
* Examples:
|
||||
* - "Find engineers" → single-type → [Person] → 31x speedup
|
||||
* - "People at Tesla" → multi-type → [Person, Organization] → 15.5x speedup
|
||||
* - "Everything about AI" → all-types → [all 31 types] → no speedup
|
||||
* - "Find engineers" → single-type → [Person] → 42x speedup
|
||||
* - "People at Tesla" → multi-type → [Person, Organization] → 21x speedup
|
||||
* - "Everything about AI" → all-types → [all 42 types] → no speedup
|
||||
*/
|
||||
|
||||
import { NounType, NOUN_TYPE_COUNT } from '../types/graphTypes.js'
|
||||
|
|
|
|||
|
|
@ -149,9 +149,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
// Type-first indexing support (v5.4.0)
|
||||
// Built into all storage adapters for billion-scale efficiency
|
||||
protected nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) // 124 bytes
|
||||
protected verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) // 160 bytes
|
||||
// Total: 284 bytes (99.76% reduction vs Map-based tracking)
|
||||
protected nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) // 168 bytes (Stage 3: 42 types)
|
||||
protected verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) // 508 bytes (Stage 3: 127 types)
|
||||
// Total: 676 bytes (99.2% reduction vs Map-based tracking)
|
||||
|
||||
// Type cache for O(1) lookups after first access
|
||||
protected nounTypeCache = new Map<string, NounType>()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue