BREAKING CHANGE: System metadata location changed from 'index/' to '_system/' directory
- Rename INDEX_DIR to SYSTEM_DIR following database conventions
- Implement dual-read/write strategy for zero-downtime migration
- Add automatic migration from old to new location on first access
- Support mixed service versions sharing S3/cloud storage
- Add 30-day grace period for gradual rollout (configurable)
- Store distributed config alongside statistics in _system folder
- Add comprehensive migration guide and documentation
Migration features:
- Read from both locations (new first, fallback to old)
- Write to both during migration period
- Automatic data migration when found only in old location
- Services can update independently without coordination
- Full backward compatibility for production deployments
The change improves clarity ('_system' better represents system metadata than 'index')
and follows standard database conventions (MongoDB's _system, PostgreSQL's pg_*).
138 lines
No EOL
4.5 KiB
TypeScript
138 lines
No EOL
4.5 KiB
TypeScript
/**
|
|
* Tests for distributed configuration migration to index folder
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { DistributedConfigManager } from '../src/distributed/configManager.js'
|
|
import { MemoryStorage } from '../src/storage/adapters/memoryStorage.js'
|
|
import { SharedConfig } from '../src/types/distributedTypes.js'
|
|
|
|
describe('Distributed Config Migration', () => {
|
|
let storage: MemoryStorage
|
|
let configManager: DistributedConfigManager
|
|
|
|
beforeEach(async () => {
|
|
storage = new MemoryStorage()
|
|
await storage.init()
|
|
})
|
|
|
|
afterEach(async () => {
|
|
if (configManager) {
|
|
await configManager.cleanup()
|
|
}
|
|
})
|
|
|
|
it('should migrate config from legacy location to index folder', async () => {
|
|
// Create a legacy config in the old location
|
|
const legacyConfig: SharedConfig = {
|
|
version: 1,
|
|
updated: new Date().toISOString(),
|
|
settings: {
|
|
partitionStrategy: 'hash',
|
|
partitionCount: 100,
|
|
embeddingModel: 'text-embedding-ada-002',
|
|
dimensions: 1536,
|
|
distanceMetric: 'cosine',
|
|
hnswParams: {
|
|
M: 16,
|
|
efConstruction: 200
|
|
}
|
|
},
|
|
instances: {}
|
|
}
|
|
|
|
// Save to legacy location
|
|
await storage.saveMetadata('_distributed_config', legacyConfig)
|
|
|
|
// Create config manager
|
|
configManager = new DistributedConfigManager(
|
|
storage,
|
|
{ role: 'reader' }
|
|
)
|
|
|
|
// Initialize - should trigger migration
|
|
const config = await configManager.initialize()
|
|
|
|
// Verify config was loaded (version gets incremented during save)
|
|
expect(config).toBeDefined()
|
|
expect(config.version).toBeGreaterThanOrEqual(2) // Incremented during migration save
|
|
expect(config.settings.partitionStrategy).toBe('hash')
|
|
|
|
// Verify config is now in statistics
|
|
const stats = await storage.getStatistics()
|
|
expect(stats).toBeDefined()
|
|
expect(stats?.distributedConfig).toBeDefined()
|
|
expect(stats?.distributedConfig?.version).toBeGreaterThanOrEqual(2)
|
|
expect(stats?.distributedConfig?.settings.partitionStrategy).toBe('hash')
|
|
})
|
|
|
|
it('should create new config in index folder if no legacy exists', async () => {
|
|
// Create config manager without legacy config
|
|
configManager = new DistributedConfigManager(
|
|
storage,
|
|
{ role: 'writer' }
|
|
)
|
|
|
|
// Initialize - should create new config
|
|
const config = await configManager.initialize()
|
|
|
|
// Verify config was created (version gets incremented during save)
|
|
expect(config).toBeDefined()
|
|
expect(config.version).toBeGreaterThanOrEqual(2) // Incremented during initial save
|
|
expect(config.settings.partitionStrategy).toBe('hash')
|
|
|
|
// Verify config is in statistics
|
|
const stats = await storage.getStatistics()
|
|
expect(stats).toBeDefined()
|
|
expect(stats?.distributedConfig).toBeDefined()
|
|
expect(stats?.distributedConfig?.version).toBeGreaterThanOrEqual(1)
|
|
})
|
|
|
|
it('should update config in index folder on save', async () => {
|
|
// Create config manager with short heartbeat interval for testing
|
|
configManager = new DistributedConfigManager(
|
|
storage,
|
|
{
|
|
role: 'hybrid',
|
|
heartbeatInterval: 50 // Short interval for testing
|
|
}
|
|
)
|
|
|
|
// Initialize
|
|
const config = await configManager.initialize()
|
|
const initialVersion = config.version
|
|
|
|
// Get config to verify it's accessible
|
|
const currentConfig = configManager.getConfig()
|
|
expect(currentConfig).toBeDefined()
|
|
|
|
// Config should already be in statistics from initialization
|
|
const stats = await storage.getStatistics()
|
|
expect(stats).toBeDefined()
|
|
expect(stats?.distributedConfig).toBeDefined()
|
|
expect(stats?.distributedConfig?.version).toBeGreaterThanOrEqual(initialVersion)
|
|
})
|
|
|
|
it('should load config from index folder on subsequent reads', async () => {
|
|
// First, create a config
|
|
configManager = new DistributedConfigManager(
|
|
storage,
|
|
{ role: 'reader' }
|
|
)
|
|
const config1 = await configManager.initialize()
|
|
await configManager.cleanup()
|
|
|
|
// Create a new manager and verify it loads from index folder
|
|
const configManager2 = new DistributedConfigManager(
|
|
storage,
|
|
{ role: 'reader' }
|
|
)
|
|
const config2 = await configManager2.initialize()
|
|
|
|
// Config2 should load the same config (version may be same or slightly higher due to heartbeat)
|
|
expect(config2.version).toBeGreaterThanOrEqual(config1.version - 1) // Allow for timing differences
|
|
expect(config2.settings.partitionStrategy).toBe(config1.settings.partitionStrategy)
|
|
|
|
await configManager2.cleanup()
|
|
})
|
|
}) |