feat(v4.0.0): Complete metadata/vector separation architecture with Azure support
This commit completes the core v4.0.0 architecture changes for billion-scale performance with metadata/vector separation. NO RELEASE YET - remaining optimizations and testing required before production release. ## Core v4.0.0 Architecture Changes ### Type System Updates - Fixed all TypeScript compilation errors (zero errors achieved) - Updated HNSWNoun/HNSWVerb to separate core fields from metadata - Implemented HNSWNounWithMetadata/HNSWVerbWithMetadata for API boundaries - Added required 'noun' field to NounMetadata for semantic structure - Renamed verb.type to verb.verb for consistency ### Storage Adapter Updates **All adapters updated for v4.0.0 two-file storage pattern:** - memoryStorage: Proper metadata/vector separation - fileSystemStorage: Two-file pattern with sharding - opfsStorage: Browser persistent storage updated - s3CompatibleStorage: AWS/MinIO/DigitalOcean support - r2Storage: Cloudflare R2 optimization - gcsStorage: Google Cloud with ADC support - **azureBlobStorage: NEW - Full Azure Blob Storage support** ### Storage Features - BaseStorage: Internal vs public method separation (_getNoun vs getNoun) - Two-file storage: Vectors in one file, metadata in another - Change tracking: getChangesSince return type updated - Pagination: getNounsWithPagination returns WithMetadata types ### Azure Blob Storage Integration (NEW) - Native @azure/storage-blob SDK integration - Four authentication methods: * DefaultAzureCredential (Managed Identity) - recommended * Connection String - simplest setup * Account Name + Key - traditional auth * SAS Token - delegated access - High-volume mode with write buffering - Adaptive backpressure for throttling - UUID-based sharding for billion-scale - Full HNSW support with graph persistence ### Utility Updates - EmbeddingManager: Updated to accept Record<string, unknown> - LSMTree: Wrapped data in NounMetadata structure with 'noun' field - EntityIdMapper: Fixed nested metadata.data structure access - MetadataIndex: Fixed field type inference integration - PeriodicCleanup: Updated for new metadata structure ### Core API Updates - Brainy: Updated verb property access from v.type to v.verb - ConfigAPI: Fixed NounMetadata access patterns - DataAPI: Updated metadata handling ### Documentation Updates - CREATING-AUGMENTATIONS.md: v4.0.0 breaking changes guide - DEVELOPER-GUIDE.md: Migration checklist and examples - COMPLETE-REFERENCE.md: v4.0.0 architecture improvements - **finite-type-system.md: NEW - Revolutionary type system benefits** ### Build & Dependencies - Zero TypeScript compilation errors - Added @azure/storage-blob and @azure/identity - 591 tests passing (23 timeout in long-running neural tests) ## What's NOT in This Release This is a work-in-progress commit. Before v4.0.0 release we need: - Storage adapter optimizations (batch operations, compression) - Azure blob tier management (Hot/Cool/Archive) - Cost optimization implementations - Additional performance testing at billion-scale - Migration guides for v3.x users ## Testing - Clean build: ✅ - Type checking: ✅ (zero errors) - Test suite: ✅ (591/614 passing, timeouts in neural tests only) 🔐 Generated with Claude Code https://claude.com/claude-code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
8d6dd07e1d
commit
92c96246fb
35 changed files with 4524 additions and 1026 deletions
|
|
@ -9,7 +9,16 @@
|
|||
* 4. HMAC Keys (fallback for backward compatibility)
|
||||
*/
|
||||
|
||||
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js'
|
||||
import {
|
||||
GraphVerb,
|
||||
HNSWNoun,
|
||||
HNSWVerb,
|
||||
NounMetadata,
|
||||
VerbMetadata,
|
||||
HNSWNounWithMetadata,
|
||||
HNSWVerbWithMetadata,
|
||||
StatisticsData
|
||||
} from '../../coreTypes.js'
|
||||
import {
|
||||
BaseStorage,
|
||||
NOUNS_DIR,
|
||||
|
|
@ -472,7 +481,7 @@ export class GcsStorage extends BaseStorage {
|
|||
// Increment noun count
|
||||
const metadata = await this.getNounMetadata(node.id)
|
||||
if (metadata && metadata.type) {
|
||||
await this.incrementEntityCountSafe(metadata.type)
|
||||
await this.incrementEntityCountSafe(metadata.type as string)
|
||||
}
|
||||
|
||||
this.logger.trace(`Node ${node.id} saved successfully`)
|
||||
|
|
@ -493,23 +502,18 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Get a noun from storage (internal implementation)
|
||||
* Combines vector data from getNode() with metadata from getNounMetadata()
|
||||
* v4.0.0: Returns ONLY vector data (no metadata field)
|
||||
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
||||
*/
|
||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||
// Get vector data (lightweight)
|
||||
// v4.0.0: Return ONLY vector data (no metadata field)
|
||||
const node = await this.getNode(id)
|
||||
if (!node) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Get metadata (entity data in 2-file system)
|
||||
const metadata = await this.getNounMetadata(id)
|
||||
|
||||
// Combine into complete noun object
|
||||
return {
|
||||
...node,
|
||||
metadata: metadata || {}
|
||||
}
|
||||
// Return pure vector structure
|
||||
return node
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -644,7 +648,7 @@ export class GcsStorage extends BaseStorage {
|
|||
// Decrement noun count
|
||||
const metadata = await this.getNounMetadata(id)
|
||||
if (metadata && metadata.type) {
|
||||
await this.decrementEntityCountSafe(metadata.type)
|
||||
await this.decrementEntityCountSafe(metadata.type as string)
|
||||
}
|
||||
|
||||
this.logger.trace(`Noun ${id} deleted successfully`)
|
||||
|
|
@ -847,7 +851,7 @@ export class GcsStorage extends BaseStorage {
|
|||
// Increment verb count
|
||||
const metadata = await this.getVerbMetadata(edge.id)
|
||||
if (metadata && metadata.type) {
|
||||
await this.incrementVerbCount(metadata.type)
|
||||
await this.incrementVerbCount(metadata.type as string)
|
||||
}
|
||||
|
||||
this.logger.trace(`Edge ${edge.id} saved successfully`)
|
||||
|
|
@ -867,23 +871,18 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Get a verb from storage (internal implementation)
|
||||
* Combines vector data from getEdge() with metadata from getVerbMetadata()
|
||||
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
||||
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
||||
*/
|
||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||
// Get vector data (lightweight)
|
||||
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
||||
const edge = await this.getEdge(id)
|
||||
if (!edge) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Get metadata (relationship data in 2-file system)
|
||||
const metadata = await this.getVerbMetadata(id)
|
||||
|
||||
// Combine into complete verb object
|
||||
return {
|
||||
...edge,
|
||||
metadata: metadata || {}
|
||||
}
|
||||
// Return pure vector + core fields structure
|
||||
return edge
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -920,7 +919,7 @@ export class GcsStorage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(verbIds as string[]))
|
||||
}
|
||||
|
||||
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
|
||||
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
||||
const edge: Edge = {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
|
|
@ -929,10 +928,10 @@ export class GcsStorage extends BaseStorage {
|
|||
// CORE RELATIONAL DATA (read from vector file)
|
||||
verb: data.verb,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
targetId: data.targetId
|
||||
|
||||
// User metadata (retrieved separately via getVerbMetadata())
|
||||
metadata: data.metadata
|
||||
// ✅ NO metadata field in v4.0.0
|
||||
// User metadata retrieved separately via getVerbMetadata()
|
||||
}
|
||||
|
||||
// Update cache
|
||||
|
|
@ -984,7 +983,7 @@ export class GcsStorage extends BaseStorage {
|
|||
// Decrement verb count
|
||||
const metadata = await this.getVerbMetadata(id)
|
||||
if (metadata && metadata.type) {
|
||||
await this.decrementVerbCount(metadata.type)
|
||||
await this.decrementVerbCount(metadata.type as string)
|
||||
}
|
||||
|
||||
this.logger.trace(`Verb ${id} deleted successfully`)
|
||||
|
|
@ -1010,6 +1009,7 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Get nouns with pagination
|
||||
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
||||
* Iterates through all UUID-based shards (00-ff) for consistent pagination
|
||||
*/
|
||||
public async getNounsWithPagination(options: {
|
||||
|
|
@ -1021,7 +1021,7 @@ export class GcsStorage extends BaseStorage {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
} = {}): Promise<{
|
||||
items: HNSWNoun[]
|
||||
items: HNSWNounWithMetadata[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -1038,31 +1038,54 @@ export class GcsStorage extends BaseStorage {
|
|||
useCache: true
|
||||
})
|
||||
|
||||
// Apply filters if provided
|
||||
let filteredNodes = result.nodes
|
||||
// v4.0.0: Combine nodes with metadata to create HNSWNounWithMetadata[]
|
||||
const items: HNSWNounWithMetadata[] = []
|
||||
|
||||
if (options.filter) {
|
||||
// Filter by noun type
|
||||
if (options.filter.nounType) {
|
||||
const nounTypes = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType
|
||||
: [options.filter.nounType]
|
||||
for (const node of result.nodes) {
|
||||
const metadata = await this.getNounMetadata(node.id)
|
||||
if (!metadata) continue
|
||||
|
||||
const filteredByType: HNSWNoun[] = []
|
||||
for (const node of filteredNodes) {
|
||||
const metadata = await this.getNounMetadata(node.id)
|
||||
if (metadata && nounTypes.includes(metadata.type || metadata.noun)) {
|
||||
filteredByType.push(node)
|
||||
// Apply filters if provided
|
||||
if (options.filter) {
|
||||
// Filter by noun type
|
||||
if (options.filter.nounType) {
|
||||
const nounTypes = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType
|
||||
: [options.filter.nounType]
|
||||
|
||||
const nounType = (metadata as any).type || (metadata as any).noun
|
||||
if (!nounType || !nounTypes.includes(nounType)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
filteredNodes = filteredByType
|
||||
|
||||
// Filter by metadata fields if specified
|
||||
if (options.filter.metadata) {
|
||||
let metadataMatch = true
|
||||
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
||||
const metadataValue = (metadata as any)[key]
|
||||
if (metadataValue !== value) {
|
||||
metadataMatch = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!metadataMatch) continue
|
||||
}
|
||||
}
|
||||
|
||||
// Additional filter logic can be added here
|
||||
// Combine node with metadata
|
||||
const nounWithMetadata: HNSWNounWithMetadata = {
|
||||
id: node.id,
|
||||
vector: [...node.vector],
|
||||
connections: new Map(node.connections),
|
||||
level: node.level || 0,
|
||||
metadata: metadata
|
||||
}
|
||||
items.push(nounWithMetadata)
|
||||
}
|
||||
|
||||
return {
|
||||
items: filteredNodes,
|
||||
items,
|
||||
totalCount: result.totalCount,
|
||||
hasMore: result.hasMore,
|
||||
nextCursor: result.nextCursor
|
||||
|
|
@ -1203,7 +1226,7 @@ export class GcsStorage extends BaseStorage {
|
|||
/**
|
||||
* Get verbs by source ID (internal implementation)
|
||||
*/
|
||||
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
|
||||
protected async getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||||
const result = await this.getVerbsWithPagination({
|
||||
limit: Number.MAX_SAFE_INTEGER,
|
||||
|
|
@ -1216,7 +1239,7 @@ export class GcsStorage extends BaseStorage {
|
|||
/**
|
||||
* Get verbs by target ID (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
|
||||
protected async getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||||
const result = await this.getVerbsWithPagination({
|
||||
limit: Number.MAX_SAFE_INTEGER,
|
||||
|
|
@ -1229,7 +1252,7 @@ export class GcsStorage extends BaseStorage {
|
|||
/**
|
||||
* Get verbs by type (internal implementation)
|
||||
*/
|
||||
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
||||
protected async getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]> {
|
||||
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
||||
const result = await this.getVerbsWithPagination({
|
||||
limit: Number.MAX_SAFE_INTEGER,
|
||||
|
|
@ -1241,6 +1264,7 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Get verbs with pagination
|
||||
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
||||
*/
|
||||
public async getVerbsWithPagination(options: {
|
||||
limit?: number
|
||||
|
|
@ -1253,7 +1277,7 @@ export class GcsStorage extends BaseStorage {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
} = {}): Promise<{
|
||||
items: GraphVerb[]
|
||||
items: HNSWVerbWithMetadata[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -1303,56 +1327,70 @@ export class GcsStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// Convert HNSWVerbs to GraphVerbs by combining with metadata
|
||||
const graphVerbs: GraphVerb[] = []
|
||||
// v4.0.0: Combine HNSWVerbs with metadata to create HNSWVerbWithMetadata[]
|
||||
const items: HNSWVerbWithMetadata[] = []
|
||||
for (const hnswVerb of hnswVerbs) {
|
||||
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb)
|
||||
if (graphVerb) {
|
||||
graphVerbs.push(graphVerb)
|
||||
const metadata = await this.getVerbMetadata(hnswVerb.id)
|
||||
|
||||
// Apply filters
|
||||
if (options.filter) {
|
||||
// v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb structure
|
||||
if (options.filter.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId
|
||||
: [options.filter.sourceId]
|
||||
if (!hnswVerb.sourceId || !sourceIds.includes(hnswVerb.sourceId)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if (options.filter.targetId) {
|
||||
const targetIds = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId
|
||||
: [options.filter.targetId]
|
||||
if (!hnswVerb.targetId || !targetIds.includes(hnswVerb.targetId)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if (options.filter.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType
|
||||
: [options.filter.verbType]
|
||||
if (!hnswVerb.verb || !verbTypes.includes(hnswVerb.verb)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by metadata fields if specified
|
||||
if (options.filter.metadata && metadata) {
|
||||
let metadataMatch = true
|
||||
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
||||
const metadataValue = (metadata as any)[key]
|
||||
if (metadataValue !== value) {
|
||||
metadataMatch = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!metadataMatch) continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filters
|
||||
let filteredVerbs = graphVerbs
|
||||
if (options.filter) {
|
||||
filteredVerbs = graphVerbs.filter((graphVerb) => {
|
||||
// Filter by sourceId
|
||||
if (options.filter!.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter!.sourceId)
|
||||
? options.filter!.sourceId
|
||||
: [options.filter!.sourceId]
|
||||
if (!sourceIds.includes(graphVerb.sourceId)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by targetId
|
||||
if (options.filter!.targetId) {
|
||||
const targetIds = Array.isArray(options.filter!.targetId)
|
||||
? options.filter!.targetId
|
||||
: [options.filter!.targetId]
|
||||
if (!targetIds.includes(graphVerb.targetId)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by verbType
|
||||
if (options.filter!.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter!.verbType)
|
||||
? options.filter!.verbType
|
||||
: [options.filter!.verbType]
|
||||
const verbType = graphVerb.verb || graphVerb.type || ''
|
||||
if (!verbTypes.includes(verbType)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
// Combine verb with metadata
|
||||
const verbWithMetadata: HNSWVerbWithMetadata = {
|
||||
id: hnswVerb.id,
|
||||
vector: [...hnswVerb.vector],
|
||||
connections: new Map(hnswVerb.connections),
|
||||
verb: hnswVerb.verb,
|
||||
sourceId: hnswVerb.sourceId,
|
||||
targetId: hnswVerb.targetId,
|
||||
metadata: metadata || {}
|
||||
}
|
||||
items.push(verbWithMetadata)
|
||||
}
|
||||
|
||||
return {
|
||||
items: filteredVerbs,
|
||||
items,
|
||||
totalCount: this.totalVerbCount,
|
||||
hasMore: !!response?.nextPageToken,
|
||||
nextCursor: response?.nextPageToken
|
||||
|
|
@ -1395,6 +1433,7 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Get verbs with filtering and pagination (public API)
|
||||
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
||||
*/
|
||||
public async getVerbs(options?: {
|
||||
pagination?: {
|
||||
|
|
@ -1410,7 +1449,7 @@ export class GcsStorage extends BaseStorage {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: GraphVerb[]
|
||||
items: HNSWVerbWithMetadata[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue