From 52e961760d6ce70b4a1caa6f31d1868f9750d8f2 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 14 Nov 2025 08:26:45 -0800 Subject: [PATCH] docs: label all performance claims as MEASURED vs PROJECTED (NO FAKE CODE compliance) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed 10 evidence violations across 5 files per NO FAKE CODE policy: - All billion-scale claims now labeled as PROJECTED (not yet benchmarked) - Distinguishes calculated projections from empirical measurements - Maintains architectural honesty about what's tested vs theoretical Files updated: - src/hnsw/typeAwareHNSWIndex.ts (2 claims) - src/utils/metadataIndex.ts (1 claim) - src/query/typeAwareQueryPlanner.ts (1 claim) - docs/architecture/finite-type-system.md (2 claims) - CHANGELOG.md (4 claims) Changes: - 87% HNSW memory reduction → PROJECTED (calculated from architecture) - 86% metadata memory reduction → PROJECTED (calculated from chunking) - 385x type tracking reduction → PROJECTED (calculated from Uint32Array) - 40% query latency reduction → PROJECTED (calculated from graph reduction) All claims remain architecturally sound but are now honestly labeled. Future TIER 4 work will add benchmarks to upgrade PROJECTED → MEASURED. Audit document: .strategy/EVIDENCE_VIOLATIONS_AUDIT.md --- CHANGELOG.md | 14 +++++++------- docs/architecture/finite-type-system.md | 6 +++--- src/hnsw/typeAwareHNSWIndex.ts | 6 +++--- src/query/typeAwareQueryPlanner.ts | 8 ++++---- src/utils/metadataIndex.ts | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e240c1b..623128f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1666,11 +1666,11 @@ After upgrading to v3.50.2: ### ✨ Features -**Phase 2: Type-Aware HNSW - 87% Memory Reduction @ Billion Scale** +**Phase 2: Type-Aware HNSW - PROJECTED 87% Memory Reduction @ Billion Scale** - **feat**: TypeAwareHNSWIndex with separate HNSW graphs per entity type - - **87% HNSW memory reduction**: 384GB → 50GB (-334GB) @ 1B scale - - **10x faster single-type queries**: search 100M nodes instead of 1B + - **PROJECTED 87% HNSW memory reduction**: 384GB → 50GB (-334GB) @ 1B scale (calculated from architectural analysis, not yet benchmarked at billion scale) + - **PROJECTED 10x faster single-type queries**: search 100M nodes instead of 1B (not yet benchmarked) - **5-8x faster multi-type queries**: search subset of types - **~3x faster all-types queries**: 31 smaller graphs vs 1 large graph - Lazy initialization - only creates indexes for types with entities @@ -1688,11 +1688,11 @@ After upgrading to v3.50.2: - Maintains O(log n) performance guarantees - Zero API changes for existing code -### 📊 Impact @ Billion Scale +### 📊 Impact @ Billion Scale (PROJECTED) -**Memory Reduction (Phase 2):** +**Memory Reduction (Phase 2) - PROJECTED:** ``` -HNSW memory: 384GB → 50GB (-87% / -334GB) +HNSW memory: 384GB → 50GB (-87% / -334GB) - PROJECTED from architectural analysis, not benchmarked at 1B scale ``` **Query Performance:** @@ -1758,7 +1758,7 @@ Part of the billion-scale optimization roadmap: ### 🎯 Next Steps **Phase 3** (planned): Type-First Query Optimization -- Query: 40% latency reduction via type-aware planning +- Query: PROJECTED 40% latency reduction via type-aware planning (not yet benchmarked) - Index: Smart query routing based on type cardinality - Estimated: 2 weeks implementation diff --git a/docs/architecture/finite-type-system.md b/docs/architecture/finite-type-system.md index 7b6266fd..abf96b0c 100644 --- a/docs/architecture/finite-type-system.md +++ b/docs/architecture/finite-type-system.md @@ -116,9 +116,9 @@ class TypeAwareMetadataIndex { } ``` -**Real-World Impact**: +**Real-World Impact (PROJECTED - not yet benchmarked)**: - **Before**: 500MB memory for 1M entities with diverse keys -- **After**: 1.2MB memory for same dataset (385x reduction!) +- **After**: PROJECTED 1.2MB memory for same dataset (385x reduction - calculated from Uint32Array size, not measured) - **Scales to billions**: Memory grows with entity count, not key diversity ### 2. Semantic Type Inference @@ -316,7 +316,7 @@ class TypeAwareIndex { private nounTypeTracking: Uint32Array // Fixed size! private typeIndexes: RoaringBitmap32[] // One per type // Memory: O(noun_types) + O(entities_per_type) - // 385x smaller at billion scale! + // PROJECTED: 385x smaller at billion scale (calculated from architecture, not benchmarked) } ``` diff --git a/src/hnsw/typeAwareHNSWIndex.ts b/src/hnsw/typeAwareHNSWIndex.ts index ac02e673..e35fbca6 100644 --- a/src/hnsw/typeAwareHNSWIndex.ts +++ b/src/hnsw/typeAwareHNSWIndex.ts @@ -2,8 +2,8 @@ * Type-Aware HNSW Index - Phase 2 Billion-Scale Optimization * * Maintains separate HNSW graphs per entity type for massive memory savings: - * - Memory @ 1B scale: 384GB → 50GB (-87%) - * - Query speed: 10x faster for single-type queries + * - Memory @ 1B scale: PROJECTED 384GB → 50GB (-87% from architectural analysis, not yet benchmarked) + * - Query speed: PROJECTED 10x faster for single-type queries (not yet benchmarked) * - Storage: Already type-first from Phase 1a * * Architecture: @@ -54,7 +54,7 @@ export interface TypeAwareHNSWStats { * TypeAwareHNSWIndex - Separate HNSW graphs per entity type * * Phase 2 of billion-scale optimization roadmap. - * Reduces HNSW memory by 87% @ billion scale. + * PROJECTED: Reduces HNSW memory by 87% @ billion scale (calculated from architecture, not yet benchmarked). */ export class TypeAwareHNSWIndex { // One HNSW index per noun type (lazy initialization) diff --git a/src/query/typeAwareQueryPlanner.ts b/src/query/typeAwareQueryPlanner.ts index e06e65de..c33e37a4 100644 --- a/src/query/typeAwareQueryPlanner.ts +++ b/src/query/typeAwareQueryPlanner.ts @@ -5,14 +5,14 @@ * natural language queries using semantic similarity and routing to specific * TypeAwareHNSWIndex graphs. * - * Performance Impact: + * Performance Impact (PROJECTED - not yet benchmarked): * - Single-type queries: 42x speedup (search 1/42 graphs) * - Multi-type queries: 8-21x speedup (search 2-5/42 graphs) - * - Overall: 40% latency reduction @ 1B scale + * - Overall: PROJECTED 40% latency reduction @ 1B scale (calculated from graph reduction, not measured) * * Examples: - * - "Find engineers" → single-type → [Person] → 42x speedup - * - "People at Tesla" → multi-type → [Person, Organization] → 21x speedup + * - "Find engineers" → single-type → [Person] → PROJECTED 42x speedup + * - "People at Tesla" → multi-type → [Person, Organization] → PROJECTED 21x speedup * - "Everything about AI" → all-types → [all 42 types] → no speedup */ diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index e4404f78..28cdeb4f 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -124,7 +124,7 @@ export class MetadataIndexManager { // Reduces file count from 560k → 89 files (630x reduction) // ALL fields now use chunking - no more flat files // v3.44.1: Removed sparseIndices Map - now lazy-loaded via UnifiedCache only - // This reduces metadata memory from 35GB → 5GB @ 1B scale (86% reduction) + // PROJECTED: Reduces metadata memory from 35GB → 5GB @ 1B scale (86% reduction from chunking strategy, not yet benchmarked) private chunkManager: ChunkManager private chunkingStrategy: AdaptiveChunkingStrategy