feat(8.0)!: delete fork/branch/commit/history/versions — superseded by the Db API
The COW version-control surface (fork, branches, checkout, commit,
getHistory/streamHistory, asOfCommit, brain.versions) is gone, along with
its subsystems: src/versioning/, the COW object store (CommitLog,
CommitObject, RefManager, TreeObject), HistoricalStorageAdapter, and the
TypeAwareHNSWIndex path it kept alive. The Db API (now/transact/asOf/
with/persist/restore) is the one versioning model in 8.0.
Survivors and replacements:
- BlobStorage survives (the VFS stores file content through it), relocated
to src/storage/blobStorage.ts with binaryDataCodec.ts; its adapter
interface is now BlobStoreAdapter, slimmed to the consumed surface
(write/read/has/delete/getMetadata + MIME-aware compression policy).
- brain.migrate() backup branches are replaced by persist-before-migrate:
MigrateOptions.backupTo persists a hard-link snapshot of the current
generation before any transform runs; MigrationResult.backupPath reports
it, and brain.restore(path) brings it back wholesale.
- CLI: cow.ts (fork/branch/checkout/history/migrate) is replaced by
snapshot.ts — snapshot <path>, restore <path>, history (tx-log),
generation.
- New public read API: brain.transactionLog({limit}) exposes the reified
tx-log (generation/timestamp/meta, newest first) that backs the CLI
history command; TxLogEntry is exported.
Tests: superseded suites deleted; fork/commit blocks excised from shared
suites; BlobStorage tests relocated + reworked against the slimmed store;
migration tests now prove the backupTo snapshot/restore round trip; new
transactionLog coverage in db-mvcc.
This commit is contained in:
parent
431cd64406
commit
8f93add705
64 changed files with 1444 additions and 16313 deletions
|
|
@ -59,10 +59,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
// save, so the migration converges under live traffic with no big-bang.
|
||||
private connectionsCodec: ConnectionsCodec | null = null
|
||||
|
||||
// COW (Copy-on-Write) support
|
||||
private cowEnabled: boolean = false
|
||||
private cowModifiedNodes: Set<string> = new Set()
|
||||
private cowParent: JsHnswVectorIndex | null = null
|
||||
|
||||
// Deferred HNSW persistence for cloud storage performance
|
||||
// In deferred mode, HNSW connections are only persisted on flush/close
|
||||
|
|
@ -301,99 +297,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
return this.persistMode
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
|
||||
*
|
||||
* Snowflake-style instant fork: O(1) shallow copy of Maps, lazy deep copy on write.
|
||||
*
|
||||
* @param parent - Parent HNSW index to copy from
|
||||
*
|
||||
* Performance:
|
||||
* - Fork time: <10ms for 1M+ nodes (just copies Map references)
|
||||
* - Memory: Shared reads, only modified nodes duplicated (~10-20% overhead)
|
||||
* - Reads: Same speed as parent (shared data structures)
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const parent = new JsHnswVectorIndex(config)
|
||||
* // ... parent has 1M nodes ...
|
||||
*
|
||||
* const fork = new JsHnswVectorIndex(config)
|
||||
* fork.enableCOW(parent) // <10ms - instant!
|
||||
*
|
||||
* // Reads share data
|
||||
* await fork.search(query) // Fast, uses parent's data
|
||||
*
|
||||
* // Writes trigger COW
|
||||
* await fork.addItem(newItem) // Deep copies only modified nodes
|
||||
* ```
|
||||
*/
|
||||
public enableCOW(parent: JsHnswVectorIndex): void {
|
||||
this.cowEnabled = true
|
||||
this.cowParent = parent
|
||||
|
||||
// Shallow copy Maps - O(1) per Map, just copies references
|
||||
// All nodes/connections are shared until first write
|
||||
this.nouns = new Map(parent.nouns)
|
||||
this.highLevelNodes = new Map()
|
||||
for (const [level, nodeSet] of parent.highLevelNodes.entries()) {
|
||||
this.highLevelNodes.set(level, new Set(nodeSet))
|
||||
}
|
||||
|
||||
// Copy scalar values
|
||||
this.entryPointId = parent.entryPointId
|
||||
this.maxLevel = parent.maxLevel
|
||||
this.dimension = parent.dimension
|
||||
|
||||
// Share cache (COW at cache level)
|
||||
this.unifiedCache = parent.unifiedCache
|
||||
|
||||
// Share config and distance function
|
||||
this.config = parent.config
|
||||
this.distanceFunction = parent.distanceFunction
|
||||
this.useParallelization = parent.useParallelization
|
||||
|
||||
prodLog.info(`HNSW COW enabled: ${parent.nouns.size} nodes shallow copied`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure node is copied before modification (lazy COW)
|
||||
*
|
||||
* Deep copies a node only when first modified. Subsequent modifications
|
||||
* use the already-copied node.
|
||||
*
|
||||
* @param nodeId - Node ID to ensure is copied
|
||||
* @private
|
||||
*/
|
||||
private ensureCOW(nodeId: string): void {
|
||||
if (!this.cowEnabled) return
|
||||
if (this.cowModifiedNodes.has(nodeId)) return // Already copied
|
||||
|
||||
const original = this.nouns.get(nodeId)
|
||||
if (!original) return
|
||||
|
||||
// Deep copy connections Map (separate Map + Sets for each level)
|
||||
const connectionsCopy = new Map<number, Set<string>>()
|
||||
for (const [level, ids] of original.connections.entries()) {
|
||||
connectionsCopy.set(level, new Set(ids))
|
||||
}
|
||||
|
||||
// Deep copy node
|
||||
const nodeCopy: HNSWNoun = {
|
||||
id: original.id,
|
||||
vector: [...original.vector], // Deep copy vector array
|
||||
connections: connectionsCopy,
|
||||
level: original.level,
|
||||
// Copy SQ8 quantized data if present
|
||||
quantizedVector: original.quantizedVector ? new Uint8Array(original.quantizedVector) : undefined,
|
||||
codebookMin: original.codebookMin,
|
||||
codebookMax: original.codebookMax
|
||||
}
|
||||
|
||||
this.nouns.set(nodeId, nodeCopy)
|
||||
this.cowModifiedNodes.add(nodeId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate distances between a query vector and multiple vectors in parallel
|
||||
* This is used to optimize performance for search operations
|
||||
|
|
@ -602,8 +505,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
continue
|
||||
}
|
||||
|
||||
// COW: Ensure neighbor is copied before modification
|
||||
this.ensureCOW(neighborId)
|
||||
|
||||
noun.connections.get(level)!.add(neighborId)
|
||||
|
||||
|
|
@ -965,16 +866,12 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
return false
|
||||
}
|
||||
|
||||
// COW: Ensure node is copied before modification
|
||||
this.ensureCOW(id)
|
||||
|
||||
const noun = this.nouns.get(id)!
|
||||
|
||||
// Remove connections to this noun from all neighbors
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
for (const neighborId of connections) {
|
||||
// COW: Ensure neighbor is copied before modification
|
||||
this.ensureCOW(neighborId)
|
||||
|
||||
const neighbor = this.nouns.get(neighborId)
|
||||
if (!neighbor) {
|
||||
|
|
@ -994,8 +891,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
for (const [nounId, otherNoun] of this.nouns.entries()) {
|
||||
if (nounId === id) continue // Skip the noun being removed
|
||||
|
||||
// COW: Ensure noun is copied before modification
|
||||
this.ensureCOW(nounId)
|
||||
|
||||
for (const [level, connections] of otherNoun.connections.entries()) {
|
||||
if (connections.has(id)) {
|
||||
|
|
@ -1881,8 +1776,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
* Ensure a noun doesn't have too many connections at a given level
|
||||
*/
|
||||
private async pruneConnections(noun: HNSWNoun, level: number): Promise<void> {
|
||||
// COW: Ensure noun is copied before modification
|
||||
this.ensureCOW(noun.id)
|
||||
|
||||
const connections = noun.connections.get(level)!
|
||||
if (connections.size <= this.config.M) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue