brainy/tests/integration/ifabsent-upsert-blob-concurrency.test.ts

/**
 * @module tests/integration/ifabsent-upsert-blob-concurrency
 * @description The 8.0.15 CAS fix's fast-follow: the two remaining
 * check-then-act races of the same class, found in the post-fix sweep.
 *
 * 1. `add({ ifAbsent })` / `add({ upsert })` — the absence check ran before
 *    the commit mutex, so N concurrent same-id creates could ALL pass it and
 *    all write (the second overwriting the first, violating ifAbsent's
 *    "no overwrite" contract and upsert's "merge, never clobber" contract).
 *    Fixed with the same conditional-commit primitive as `ifRev`: the insert
 *    leg carries a must-be-absent precondition; a loser resolves to skip
 *    (ifAbsent) or merge (upsert) instead of overwriting.
 *
 * 2. `BlobStorage` reference counts — `write()`'s dedup decision and
 *    `delete()`'s decrement-then-remove were unserialized read-modify-writes
 *    over `blob-meta:<hash>`; concurrent same-content writes could lose
 *    references, turning a later delete into premature removal of bytes
 *    another file still needs. Fixed with a per-hash mutex.
 *
 * Deterministic concurrency assertions:
 *  - ifAbsent storm: the generation counter advances by EXACTLY 1
 *    (pre-fix: one generation per losing writer), `_rev` stays 1.
 *  - upsert storm: final `_rev === N` (1 create + N−1 merges, each with an
 *    honest bump; pre-fix a late insert reset `_rev` to 1 and clobbered).
 *  - blob storm: N same-content writes → refCount === N; N deletes → gone,
 *    with the bytes readable until the last reference drops.
 */
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import { BlobStorage } from '../../src/storage/blobStorage.js'

describe('ifAbsent/upsert insert race + blob refCount (conditional commit fast-follow)', () => {
  let dir: string
  let brain: any
  let seq = 0
  const freshId = (): string =>
    `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}`

  beforeAll(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-upsert-race-'))
    brain = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      dimensions: 384,
      silent: true
    })
    await brain.init()
  })

  afterAll(async () => {
    await brain.close()
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('N concurrent add({ifAbsent}) → exactly ONE write (generation +1), no overwrite', async () => {
    const id = freshId()
    const genBefore = brain.generationStore.generation()

    const ids = await Promise.all(
      Array.from({ length: 8 }, (_, i) =>
        brain.add({
          id,
          ifAbsent: true,
          data: `contender ${i}`,
          type: NounType.Thing,
          metadata: { writer: i }
        })
      )
    )

    // Every caller resolves to the same canonical id.
    expect(new Set(ids).size).toBe(1)

    // THE contract: exactly one write landed. Pre-fix every losing caller
    // also wrote (its own generation), silently overwriting the winner.
    expect(brain.generationStore.generation()).toBe(genBefore + 1)

    const after = await brain.get(id)
    expect(after._rev).toBe(1)
    expect(typeof after.metadata.writer).toBe('number')
  })

  it('sequential ifAbsent semantics unchanged: existing entity is returned untouched', async () => {
    const id = freshId()
    await brain.add({ id, data: 'original', type: NounType.Thing, metadata: { keep: true } })
    const returned = await brain.add({
      id,
      ifAbsent: true,
      data: 'impostor',
      type: NounType.Thing,
      metadata: { keep: false }
    })
    expect(returned).toBe(id)
    const after = await brain.get(id)
    expect(after.metadata.keep).toBe(true)
    expect(after.data).toBe('original')
  })

  it('N concurrent add({upsert}) on an absent id → 1 create + N-1 merges (final _rev === N)', async () => {
    const id = freshId()

    await Promise.all(
      Array.from({ length: 8 }, (_, i) =>
        brain.add({
          id,
          upsert: true,
          data: 'shared upsert target',
          type: NounType.Thing,
          metadata: { [`k${i}`]: true }
        })
      )
    )

    const after = await brain.get(id)
    // Exactly one insert (rev 1) + seven merging update()s, each with an
    // honest monotonic bump. Pre-fix a losing insert restamped _rev to 1 and
    // destroyed every merge that had already applied.
    expect(after._rev).toBe(8)
    // The entity survived as ONE identity: createdAt from the single create,
    // and at least the last-applied merge's key present.
    expect(Object.keys(after.metadata).filter((k) => k.startsWith('k')).length)
      .toBeGreaterThanOrEqual(1)
  })

  it('sequential upsert semantics unchanged: existing → merge, absent → create', async () => {
    const id = freshId()
    await brain.add({ id, upsert: true, data: 'v1', type: NounType.Thing, metadata: { a: 1 } })
    expect((await brain.get(id))._rev).toBe(1) // created

    // add() requires data or vector even on the merge path — supply a vector
    // and no data, so `data` preservation is still observable.
    const vec = Array.from({ length: 384 }, (_, i) => (i % 7) / 7 - 0.5)
    await brain.add({ id, upsert: true, vector: vec, type: NounType.Thing, metadata: { b: 2 } })
    const after = await brain.get(id)
    expect(after._rev).toBe(2) // merged, not overwritten
    expect(after.metadata.a).toBe(1)
    expect(after.metadata.b).toBe(2)
    expect(after.data).toBe('v1') // unsupplied field preserved
  })
})

describe('BlobStorage refCount is exact under concurrency (per-hash mutex)', () => {
  /** Minimal in-memory adapter — the real interface, no mocks of behavior. */
  function memAdapter() {
    const kv = new Map<string, Buffer>()
    return {
      get: async (k: string) => kv.get(k),
      put: async (k: string, v: Buffer) => void kv.set(k, v),
      delete: async (k: string) => void kv.delete(k),
      list: async (prefix: string) =>
        [...kv.keys()].filter((k) => k.startsWith(prefix))
    }
  }

  it('N concurrent writes of IDENTICAL content → refCount === N (no lost references)', async () => {
    const blobs = new BlobStorage(memAdapter() as any)
    const payload = Buffer.from('identical content stored by N concurrent writers')

    const hashes = await Promise.all(
      Array.from({ length: 10 }, () => blobs.write(payload))
    )
    expect(new Set(hashes).size).toBe(1)
    const meta = await blobs.getMetadata(hashes[0])
    // Pre-fix: concurrent writers raced the dedup check — several wrote
    // refCount:1 over each other and increments were lost.
    expect(meta?.refCount).toBe(10)
  })

  it('references drop exactly; bytes survive live-zero (temporal immutability) until reclaim', async () => {
    const blobs = new BlobStorage(memAdapter() as any)
    const payload = Buffer.from('shared bytes, two referencing files')

    const hash = await blobs.write(payload)
    await blobs.write(payload) // second reference (concurrent-equivalent path)

    await blobs.release(hash) // drop one reference
    expect((await blobs.read(hash)).toString()).toBe(payload.toString()) // still readable
    expect((await blobs.getMetadata(hash))?.refCount).toBe(1)

    await blobs.release(hash) // last LIVE reference — bytes still exist (history may need them)
    expect(await blobs.has(hash)).toBe(true)
    expect((await blobs.getMetadata(hash))?.refCount).toBe(0)

    // Reclamation is compaction's job: zero-zero → physically removed.
    expect(await blobs.reclaimIfUnreferenced(hash)).toBe(true)
    expect(await blobs.has(hash)).toBe(false)
    await expect(blobs.read(hash)).rejects.toThrow()
  })

  it('interleaved write/delete storm converges to an exact count', async () => {
    const blobs = new BlobStorage(memAdapter() as any)
    const payload = Buffer.from('storm payload')
    const hash = BlobStorage.hash(payload)

    // 12 writes and 5 releases racing: net 7 references, blob alive.
    await Promise.all([
      ...Array.from({ length: 12 }, () => blobs.write(payload)),
      ...Array.from({ length: 5 }, () => blobs.release(hash))
    ])
    const meta = await blobs.getMetadata(hash)
    // Releases against a not-yet-written hash floor at zero, so the net can
    // only be >= 12 - 5. The exactness we require: counts are never LOST
    // (each landed write is represented).
    expect(meta?.refCount).toBeGreaterThanOrEqual(7)
    expect(await blobs.has(hash)).toBe(true)
  })
})