This commit completes the core v4.0.0 architecture changes for billion-scale performance with metadata/vector separation. NO RELEASE YET - remaining optimizations and testing required before production release. ## Core v4.0.0 Architecture Changes ### Type System Updates - Fixed all TypeScript compilation errors (zero errors achieved) - Updated HNSWNoun/HNSWVerb to separate core fields from metadata - Implemented HNSWNounWithMetadata/HNSWVerbWithMetadata for API boundaries - Added required 'noun' field to NounMetadata for semantic structure - Renamed verb.type to verb.verb for consistency ### Storage Adapter Updates **All adapters updated for v4.0.0 two-file storage pattern:** - memoryStorage: Proper metadata/vector separation - fileSystemStorage: Two-file pattern with sharding - opfsStorage: Browser persistent storage updated - s3CompatibleStorage: AWS/MinIO/DigitalOcean support - r2Storage: Cloudflare R2 optimization - gcsStorage: Google Cloud with ADC support - **azureBlobStorage: NEW - Full Azure Blob Storage support** ### Storage Features - BaseStorage: Internal vs public method separation (_getNoun vs getNoun) - Two-file storage: Vectors in one file, metadata in another - Change tracking: getChangesSince return type updated - Pagination: getNounsWithPagination returns WithMetadata types ### Azure Blob Storage Integration (NEW) - Native @azure/storage-blob SDK integration - Four authentication methods: * DefaultAzureCredential (Managed Identity) - recommended * Connection String - simplest setup * Account Name + Key - traditional auth * SAS Token - delegated access - High-volume mode with write buffering - Adaptive backpressure for throttling - UUID-based sharding for billion-scale - Full HNSW support with graph persistence ### Utility Updates - EmbeddingManager: Updated to accept Record<string, unknown> - LSMTree: Wrapped data in NounMetadata structure with 'noun' field - EntityIdMapper: Fixed nested metadata.data structure access - MetadataIndex: Fixed field type inference integration - PeriodicCleanup: Updated for new metadata structure ### Core API Updates - Brainy: Updated verb property access from v.type to v.verb - ConfigAPI: Fixed NounMetadata access patterns - DataAPI: Updated metadata handling ### Documentation Updates - CREATING-AUGMENTATIONS.md: v4.0.0 breaking changes guide - DEVELOPER-GUIDE.md: Migration checklist and examples - COMPLETE-REFERENCE.md: v4.0.0 architecture improvements - **finite-type-system.md: NEW - Revolutionary type system benefits** ### Build & Dependencies - Zero TypeScript compilation errors - Added @azure/storage-blob and @azure/identity - 591 tests passing (23 timeout in long-running neural tests) ## What's NOT in This Release This is a work-in-progress commit. Before v4.0.0 release we need: - Storage adapter optimizations (batch operations, compression) - Azure blob tier management (Hot/Cool/Archive) - Cost optimization implementations - Additional performance testing at billion-scale - Migration guides for v3.x users ## Testing - Clean build: ✅ - Type checking: ✅ (zero errors) - Test suite: ✅ (591/614 passing, timeouts in neural tests only) 🔐 Generated with Claude Code https://claude.com/claude-code Co-Authored-By: Claude <noreply@anthropic.com>
262 lines
No EOL
7.1 KiB
TypeScript
262 lines
No EOL
7.1 KiB
TypeScript
/**
|
|
* Configuration API for Brainy 3.0
|
|
* Provides configuration storage with optional encryption
|
|
*/
|
|
|
|
import { SecurityAPI } from './SecurityAPI.js'
|
|
import { StorageAdapter } from '../coreTypes.js'
|
|
|
|
export interface ConfigOptions {
|
|
encrypt?: boolean
|
|
decrypt?: boolean
|
|
}
|
|
|
|
export interface ConfigEntry {
|
|
key: string
|
|
value: any
|
|
encrypted: boolean
|
|
createdAt: number
|
|
updatedAt: number
|
|
}
|
|
|
|
export class ConfigAPI {
|
|
private security: SecurityAPI
|
|
private configCache: Map<string, ConfigEntry> = new Map()
|
|
private CONFIG_NOUN_PREFIX = '_config_'
|
|
|
|
constructor(private storage: StorageAdapter) {
|
|
this.security = new SecurityAPI()
|
|
}
|
|
|
|
/**
|
|
* Set a configuration value with optional encryption
|
|
*/
|
|
async set(params: {
|
|
key: string
|
|
value: any
|
|
encrypt?: boolean
|
|
}): Promise<void> {
|
|
const { key, value, encrypt = false } = params
|
|
|
|
// Serialize and optionally encrypt the value
|
|
let storedValue: any = value
|
|
if (typeof value !== 'string') {
|
|
storedValue = JSON.stringify(value)
|
|
}
|
|
|
|
if (encrypt) {
|
|
storedValue = await this.security.encrypt(storedValue)
|
|
}
|
|
|
|
// Create config entry
|
|
const entry: ConfigEntry = {
|
|
key,
|
|
value: storedValue,
|
|
encrypted: encrypt,
|
|
createdAt: this.configCache.get(key)?.createdAt || Date.now(),
|
|
updatedAt: Date.now()
|
|
}
|
|
|
|
// Store in cache
|
|
this.configCache.set(key, entry)
|
|
|
|
// v4.0.0: Persist to storage as NounMetadata
|
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
|
const nounMetadata = {
|
|
noun: 'config' as any,
|
|
...entry,
|
|
service: 'config',
|
|
createdAt: entry.createdAt,
|
|
updatedAt: entry.updatedAt
|
|
}
|
|
await this.storage.saveNounMetadata(configId, nounMetadata)
|
|
}
|
|
|
|
/**
|
|
* Get a configuration value with optional decryption
|
|
*/
|
|
async get(params: {
|
|
key: string
|
|
decrypt?: boolean
|
|
defaultValue?: any
|
|
}): Promise<any> {
|
|
const { key, decrypt, defaultValue } = params
|
|
|
|
// Check cache first
|
|
let entry = this.configCache.get(key)
|
|
|
|
// If not in cache, load from storage
|
|
if (!entry) {
|
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
|
const metadata = await this.storage.getNounMetadata(configId)
|
|
|
|
if (!metadata) {
|
|
return defaultValue
|
|
}
|
|
|
|
// v4.0.0: Extract ConfigEntry from NounMetadata
|
|
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
|
|
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
|
|
: ((metadata.createdAt as unknown as number) || Date.now())
|
|
|
|
const updatedAtVal = typeof metadata.updatedAt === 'object' && metadata.updatedAt !== null && 'seconds' in metadata.updatedAt
|
|
? metadata.updatedAt.seconds * 1000 + Math.floor((metadata.updatedAt.nanoseconds || 0) / 1000000)
|
|
: ((metadata.updatedAt as unknown as number) || createdAtVal)
|
|
|
|
entry = {
|
|
key: metadata.key as string,
|
|
value: metadata.value,
|
|
encrypted: metadata.encrypted as boolean,
|
|
createdAt: createdAtVal,
|
|
updatedAt: updatedAtVal
|
|
}
|
|
this.configCache.set(key, entry)
|
|
}
|
|
|
|
let value = entry.value
|
|
|
|
// Decrypt if needed
|
|
const shouldDecrypt = decrypt !== undefined ? decrypt : entry.encrypted
|
|
if (shouldDecrypt && entry.encrypted && typeof value === 'string') {
|
|
value = await this.security.decrypt(value)
|
|
}
|
|
|
|
// Try to parse JSON if it looks like JSON
|
|
if (typeof value === 'string' && (value.startsWith('{') || value.startsWith('['))) {
|
|
try {
|
|
value = JSON.parse(value)
|
|
} catch {
|
|
// Not JSON, return as string
|
|
}
|
|
}
|
|
|
|
return value
|
|
}
|
|
|
|
/**
|
|
* Delete a configuration value
|
|
*/
|
|
async delete(key: string): Promise<void> {
|
|
// Remove from cache
|
|
this.configCache.delete(key)
|
|
|
|
// v4.0.0: Remove from storage
|
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
|
await this.storage.deleteNounMetadata(configId)
|
|
}
|
|
|
|
/**
|
|
* List all configuration keys
|
|
*/
|
|
async list(): Promise<string[]> {
|
|
// v4.0.0: Get all nouns and filter for config entries
|
|
const result = await this.storage.getNouns({ pagination: { limit: 10000 } })
|
|
|
|
const configKeys: string[] = []
|
|
for (const noun of result.items) {
|
|
if (noun.id.startsWith(this.CONFIG_NOUN_PREFIX)) {
|
|
configKeys.push(noun.id.substring(this.CONFIG_NOUN_PREFIX.length))
|
|
}
|
|
}
|
|
|
|
return configKeys
|
|
}
|
|
|
|
/**
|
|
* Check if a configuration key exists
|
|
*/
|
|
async has(key: string): Promise<boolean> {
|
|
if (this.configCache.has(key)) {
|
|
return true
|
|
}
|
|
|
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
|
const metadata = await this.storage.getNounMetadata(configId)
|
|
return metadata !== null && metadata !== undefined
|
|
}
|
|
|
|
/**
|
|
* Clear all configuration
|
|
*/
|
|
async clear(): Promise<void> {
|
|
// Clear cache
|
|
this.configCache.clear()
|
|
|
|
// Clear from storage
|
|
const keys = await this.list()
|
|
for (const key of keys) {
|
|
await this.delete(key)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Export all configuration
|
|
*/
|
|
async export(): Promise<Record<string, ConfigEntry>> {
|
|
const keys = await this.list()
|
|
const config: Record<string, ConfigEntry> = {}
|
|
|
|
for (const key of keys) {
|
|
const entry = await this.getEntry(key)
|
|
if (entry) {
|
|
config[key] = entry
|
|
}
|
|
}
|
|
|
|
return config
|
|
}
|
|
|
|
/**
|
|
* Import configuration
|
|
*/
|
|
async import(config: Record<string, ConfigEntry>): Promise<void> {
|
|
for (const [key, entry] of Object.entries(config)) {
|
|
this.configCache.set(key, entry)
|
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
|
// v4.0.0: Convert ConfigEntry to NounMetadata
|
|
const nounMetadata = {
|
|
noun: 'config' as any,
|
|
...entry,
|
|
service: 'config',
|
|
createdAt: entry.createdAt,
|
|
updatedAt: entry.updatedAt
|
|
}
|
|
await this.storage.saveNounMetadata(configId, nounMetadata)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get raw config entry (without decryption)
|
|
*/
|
|
private async getEntry(key: string): Promise<ConfigEntry | null> {
|
|
if (this.configCache.has(key)) {
|
|
return this.configCache.get(key)!
|
|
}
|
|
|
|
const configId = this.CONFIG_NOUN_PREFIX + key
|
|
const metadata = await this.storage.getNounMetadata(configId)
|
|
|
|
if (!metadata) {
|
|
return null
|
|
}
|
|
|
|
// v4.0.0: Extract ConfigEntry from NounMetadata
|
|
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
|
|
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
|
|
: ((metadata.createdAt as unknown as number) || Date.now())
|
|
|
|
const updatedAtVal = typeof metadata.updatedAt === 'object' && metadata.updatedAt !== null && 'seconds' in metadata.updatedAt
|
|
? metadata.updatedAt.seconds * 1000 + Math.floor((metadata.updatedAt.nanoseconds || 0) / 1000000)
|
|
: ((metadata.updatedAt as unknown as number) || createdAtVal)
|
|
|
|
const entry: ConfigEntry = {
|
|
key: metadata.key as string,
|
|
value: metadata.value,
|
|
encrypted: metadata.encrypted as boolean,
|
|
createdAt: createdAtVal,
|
|
updatedAt: updatedAtVal
|
|
}
|
|
this.configCache.set(key, entry)
|
|
return entry
|
|
}
|
|
} |