feat(v4.0.0): Complete metadata/vector separation architecture with Azure support
This commit completes the core v4.0.0 architecture changes for billion-scale performance with metadata/vector separation. NO RELEASE YET - remaining optimizations and testing required before production release. ## Core v4.0.0 Architecture Changes ### Type System Updates - Fixed all TypeScript compilation errors (zero errors achieved) - Updated HNSWNoun/HNSWVerb to separate core fields from metadata - Implemented HNSWNounWithMetadata/HNSWVerbWithMetadata for API boundaries - Added required 'noun' field to NounMetadata for semantic structure - Renamed verb.type to verb.verb for consistency ### Storage Adapter Updates **All adapters updated for v4.0.0 two-file storage pattern:** - memoryStorage: Proper metadata/vector separation - fileSystemStorage: Two-file pattern with sharding - opfsStorage: Browser persistent storage updated - s3CompatibleStorage: AWS/MinIO/DigitalOcean support - r2Storage: Cloudflare R2 optimization - gcsStorage: Google Cloud with ADC support - **azureBlobStorage: NEW - Full Azure Blob Storage support** ### Storage Features - BaseStorage: Internal vs public method separation (_getNoun vs getNoun) - Two-file storage: Vectors in one file, metadata in another - Change tracking: getChangesSince return type updated - Pagination: getNounsWithPagination returns WithMetadata types ### Azure Blob Storage Integration (NEW) - Native @azure/storage-blob SDK integration - Four authentication methods: * DefaultAzureCredential (Managed Identity) - recommended * Connection String - simplest setup * Account Name + Key - traditional auth * SAS Token - delegated access - High-volume mode with write buffering - Adaptive backpressure for throttling - UUID-based sharding for billion-scale - Full HNSW support with graph persistence ### Utility Updates - EmbeddingManager: Updated to accept Record<string, unknown> - LSMTree: Wrapped data in NounMetadata structure with 'noun' field - EntityIdMapper: Fixed nested metadata.data structure access - MetadataIndex: Fixed field type inference integration - PeriodicCleanup: Updated for new metadata structure ### Core API Updates - Brainy: Updated verb property access from v.type to v.verb - ConfigAPI: Fixed NounMetadata access patterns - DataAPI: Updated metadata handling ### Documentation Updates - CREATING-AUGMENTATIONS.md: v4.0.0 breaking changes guide - DEVELOPER-GUIDE.md: Migration checklist and examples - COMPLETE-REFERENCE.md: v4.0.0 architecture improvements - **finite-type-system.md: NEW - Revolutionary type system benefits** ### Build & Dependencies - Zero TypeScript compilation errors - Added @azure/storage-blob and @azure/identity - 591 tests passing (23 timeout in long-running neural tests) ## What's NOT in This Release This is a work-in-progress commit. Before v4.0.0 release we need: - Storage adapter optimizations (batch operations, compression) - Azure blob tier management (Hot/Cool/Archive) - Cost optimization implementations - Additional performance testing at billion-scale - Migration guides for v3.x users ## Testing - Clean build: ✅ - Type checking: ✅ (zero errors) - Test suite: ✅ (591/614 passing, timeouts in neural tests only) 🔐 Generated with Claude Code https://claude.com/claude-code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
8d6dd07e1d
commit
92c96246fb
35 changed files with 4524 additions and 1026 deletions
|
|
@ -7,6 +7,10 @@ import {
|
|||
GraphVerb,
|
||||
HNSWNoun,
|
||||
HNSWVerb,
|
||||
NounMetadata,
|
||||
VerbMetadata,
|
||||
HNSWNounWithMetadata,
|
||||
HNSWVerbWithMetadata,
|
||||
StatisticsData
|
||||
} from '../../coreTypes.js'
|
||||
import {
|
||||
|
|
@ -266,21 +270,16 @@ export class OPFSStorage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
const node = {
|
||||
// v4.0.0: Return ONLY vector data (no metadata field)
|
||||
const node: HNSWNode = {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
level: data.level || 0
|
||||
}
|
||||
|
||||
// Get metadata (entity data in 2-file system)
|
||||
const metadata = await this.getNounMetadata(id)
|
||||
|
||||
// Combine into complete noun object
|
||||
return {
|
||||
...node,
|
||||
metadata: metadata || {}
|
||||
}
|
||||
// Return pure vector structure
|
||||
return node
|
||||
} catch (error) {
|
||||
// Noun not found or other error
|
||||
return null
|
||||
|
|
@ -444,23 +443,18 @@ export class OPFSStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
* Combines vector data from getEdge() with metadata from getVerbMetadata()
|
||||
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
||||
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
||||
*/
|
||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||
// Get vector data (lightweight)
|
||||
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
||||
const edge = await this.getEdge(id)
|
||||
if (!edge) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Get metadata (relationship data in 2-file system)
|
||||
const metadata = await this.getVerbMetadata(id)
|
||||
|
||||
// Combine into complete verb object
|
||||
return {
|
||||
...edge,
|
||||
metadata: metadata || {}
|
||||
}
|
||||
// Return pure vector + core fields structure
|
||||
return edge
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -502,7 +496,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
version: '1.0'
|
||||
}
|
||||
|
||||
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
|
||||
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
||||
return {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
|
|
@ -511,10 +505,10 @@ export class OPFSStorage extends BaseStorage {
|
|||
// CORE RELATIONAL DATA (read from vector file)
|
||||
verb: data.verb,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
targetId: data.targetId
|
||||
|
||||
// User metadata (retrieved separately via getVerbMetadata())
|
||||
metadata: data.metadata
|
||||
// ✅ NO metadata field in v4.0.0
|
||||
// User metadata retrieved separately via getVerbMetadata()
|
||||
}
|
||||
} catch (error) {
|
||||
// Edge not found or other error
|
||||
|
|
@ -563,7 +557,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
version: '1.0'
|
||||
}
|
||||
|
||||
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields
|
||||
// v4.0.0: Include core relational fields (NO metadata field)
|
||||
allEdges.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
|
|
@ -572,10 +566,10 @@ export class OPFSStorage extends BaseStorage {
|
|||
// CORE RELATIONAL DATA
|
||||
verb: data.verb,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
targetId: data.targetId
|
||||
|
||||
// User metadata
|
||||
metadata: data.metadata
|
||||
// ✅ NO metadata field in v4.0.0
|
||||
// User metadata retrieved separately via getVerbMetadata()
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading edge file ${shardName}/${fileName}:`, error)
|
||||
|
|
@ -596,7 +590,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
*/
|
||||
protected async getVerbsBySource_internal(
|
||||
sourceId: string
|
||||
): Promise<GraphVerb[]> {
|
||||
): Promise<HNSWVerbWithMetadata[]> {
|
||||
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||||
const result = await this.getVerbsWithPagination({
|
||||
filter: { sourceId: [sourceId] },
|
||||
|
|
@ -608,7 +602,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
/**
|
||||
* Get edges by source
|
||||
*/
|
||||
protected async getEdgesBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
protected async getEdgesBySource(sourceId: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// This method is deprecated and would require loading metadata for each edge
|
||||
// For now, return empty array since this is not efficiently implementable with new storage pattern
|
||||
console.warn(
|
||||
|
|
@ -622,7 +616,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
*/
|
||||
protected async getVerbsByTarget_internal(
|
||||
targetId: string
|
||||
): Promise<GraphVerb[]> {
|
||||
): Promise<HNSWVerbWithMetadata[]> {
|
||||
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||||
const result = await this.getVerbsWithPagination({
|
||||
filter: { targetId: [targetId] },
|
||||
|
|
@ -634,7 +628,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
/**
|
||||
* Get edges by target
|
||||
*/
|
||||
protected async getEdgesByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
protected async getEdgesByTarget(targetId: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// This method is deprecated and would require loading metadata for each edge
|
||||
// For now, return empty array since this is not efficiently implementable with new storage pattern
|
||||
console.warn(
|
||||
|
|
@ -646,7 +640,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
||||
protected async getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||||
const result = await this.getVerbsWithPagination({
|
||||
filter: { verbType: [type] },
|
||||
|
|
@ -658,7 +652,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
/**
|
||||
* Get edges by type
|
||||
*/
|
||||
protected async getEdgesByType(type: string): Promise<GraphVerb[]> {
|
||||
protected async getEdgesByType(type: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// This method is deprecated and would require loading metadata for each edge
|
||||
// For now, return empty array since this is not efficiently implementable with new storage pattern
|
||||
console.warn(
|
||||
|
|
@ -1515,7 +1509,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
} = {}): Promise<{
|
||||
items: HNSWNoun[]
|
||||
items: HNSWNounWithMetadata[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -1557,40 +1551,41 @@ export class OPFSStorage extends BaseStorage {
|
|||
// Get the subset of files for this page
|
||||
const pageFiles = nounFiles.slice(startIndex, startIndex + limit)
|
||||
|
||||
// Load nouns from files
|
||||
const items: HNSWNoun[] = []
|
||||
// v4.0.0: Load nouns from files and combine with metadata
|
||||
const items: HNSWNounWithMetadata[] = []
|
||||
for (const fileName of pageFiles) {
|
||||
// fileName is in format "shard/uuid.json", extract just the UUID
|
||||
const id = fileName.split('/')[1].replace('.json', '')
|
||||
const noun = await this.getNoun_internal(id)
|
||||
if (noun) {
|
||||
// Load metadata for filtering and combining
|
||||
const metadata = await this.getNounMetadata(id)
|
||||
if (!metadata) continue
|
||||
|
||||
// Apply filters if provided
|
||||
if (options.filter) {
|
||||
const metadata = await this.getNounMetadata(id)
|
||||
|
||||
// Filter by noun type
|
||||
if (options.filter.nounType) {
|
||||
const nounTypes = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType
|
||||
: [options.filter.nounType]
|
||||
if (metadata && !nounTypes.includes(metadata.type || metadata.noun)) {
|
||||
if (!nounTypes.includes((metadata.type || metadata.noun) as string)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
if (metadata && !services.includes(metadata.createdBy?.augmentation)) {
|
||||
if (!metadata.createdBy?.augmentation || !services.includes(metadata.createdBy.augmentation as string)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
if (!metadata) continue
|
||||
let matches = true
|
||||
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
|
|
@ -1601,8 +1596,17 @@ export class OPFSStorage extends BaseStorage {
|
|||
if (!matches) continue
|
||||
}
|
||||
}
|
||||
|
||||
items.push(noun)
|
||||
|
||||
// v4.0.0: Create HNSWNounWithMetadata by combining noun with metadata
|
||||
const nounWithMetadata: HNSWNounWithMetadata = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(noun.connections),
|
||||
level: noun.level || 0,
|
||||
metadata: metadata
|
||||
}
|
||||
|
||||
items.push(nounWithMetadata)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1638,7 +1642,7 @@ export class OPFSStorage extends BaseStorage {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
} = {}): Promise<{
|
||||
items: GraphVerb[]
|
||||
items: HNSWVerbWithMetadata[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -1680,73 +1684,87 @@ export class OPFSStorage extends BaseStorage {
|
|||
// Get the subset of files for this page
|
||||
const pageFiles = verbFiles.slice(startIndex, startIndex + limit)
|
||||
|
||||
// Load verbs from files and convert to GraphVerb
|
||||
const items: GraphVerb[] = []
|
||||
// v4.0.0: Load verbs from files and combine with metadata
|
||||
const items: HNSWVerbWithMetadata[] = []
|
||||
for (const fileName of pageFiles) {
|
||||
// fileName is in format "shard/uuid.json", extract just the UUID
|
||||
const id = fileName.split('/')[1].replace('.json', '')
|
||||
const hnswVerb = await this.getVerb_internal(id)
|
||||
if (hnswVerb) {
|
||||
// Convert HNSWVerb to GraphVerb
|
||||
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb)
|
||||
if (graphVerb) {
|
||||
// Apply filters if provided
|
||||
if (options.filter) {
|
||||
// Filter by verb type
|
||||
if (options.filter.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType
|
||||
: [options.filter.verbType]
|
||||
if (graphVerb.verb && !verbTypes.includes(graphVerb.verb)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by source ID
|
||||
if (options.filter.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId
|
||||
: [options.filter.sourceId]
|
||||
if (graphVerb.source && !sourceIds.includes(graphVerb.source)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by target ID
|
||||
if (options.filter.targetId) {
|
||||
const targetIds = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId
|
||||
: [options.filter.targetId]
|
||||
if (graphVerb.target && !targetIds.includes(graphVerb.target)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
if (graphVerb.createdBy?.augmentation && !services.includes(graphVerb.createdBy.augmentation)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata && graphVerb.metadata) {
|
||||
let matches = true
|
||||
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
||||
if (graphVerb.metadata[key] !== value) {
|
||||
matches = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!matches) continue
|
||||
// Load metadata for filtering and combining
|
||||
const metadata = await this.getVerbMetadata(id)
|
||||
if (!metadata) continue
|
||||
|
||||
// Apply filters if provided
|
||||
if (options.filter) {
|
||||
// Filter by verb type
|
||||
// v4.0.0: verb field is in HNSWVerb structure (NOT in metadata)
|
||||
if (options.filter.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType
|
||||
: [options.filter.verbType]
|
||||
if (!hnswVerb.verb || !verbTypes.includes(hnswVerb.verb)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
items.push(graphVerb)
|
||||
|
||||
// Filter by source ID
|
||||
// v4.0.0: sourceId field is in HNSWVerb structure (NOT in metadata)
|
||||
if (options.filter.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId
|
||||
: [options.filter.sourceId]
|
||||
if (!hnswVerb.sourceId || !sourceIds.includes(hnswVerb.sourceId)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by target ID
|
||||
// v4.0.0: targetId field is in HNSWVerb structure (NOT in metadata)
|
||||
if (options.filter.targetId) {
|
||||
const targetIds = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId
|
||||
: [options.filter.targetId]
|
||||
if (!hnswVerb.targetId || !targetIds.includes(hnswVerb.targetId)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
if (!metadata.createdBy?.augmentation || !services.includes(metadata.createdBy.augmentation as string)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
let matches = true
|
||||
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
matches = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!matches) continue
|
||||
}
|
||||
}
|
||||
|
||||
// v4.0.0: Create HNSWVerbWithMetadata by combining verb with metadata
|
||||
const verbWithMetadata: HNSWVerbWithMetadata = {
|
||||
id: hnswVerb.id,
|
||||
vector: [...hnswVerb.vector],
|
||||
connections: new Map(hnswVerb.connections),
|
||||
verb: hnswVerb.verb,
|
||||
sourceId: hnswVerb.sourceId,
|
||||
targetId: hnswVerb.targetId,
|
||||
metadata: metadata
|
||||
}
|
||||
|
||||
items.push(verbWithMetadata)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue