open-brainy/tests/unit/indexes/columnStore/segment-load-fault.test.ts

/**
 * @module tests/unit/indexes/columnStore/segment-load-fault
 * @description Pattern-B acceptance for the ColumnStore (finding 4): a segment
 * the manifest LISTS but that cannot be loaded must never be silently skipped —
 * doing so dropped every entity in that segment out of `filter`/`rangeQuery`/
 * `sortTopK` with no error, so a corrupt index looked like a merely short result.
 *
 * The three failure classes and their required behaviour (torn-segment
 * QUARANTINE contract — a raw throw at query time killed every query on the
 * field forever; a silent skip hid the loss; quarantine is the middle):
 *  - a real storage IO fault (EIO) PROPAGATES verbatim — a present-but-unreadable
 *    segment is not "absent", so it must not read as an empty result;
 *  - a manifest-listed segment with undecodable bytes is QUARANTINED at
 *    discovery: the query serves the field's remaining segments degraded and
 *    `quarantinedSegments()` reports the torn segment (loud once, counted
 *    always, healable);
 *  - a manifest-listed segment with NO bytes (gone on disk) quarantines the
 *    same way.
 * Only genuine absence stays benign: querying a field that has no manifest at all
 * returns empty (nothing was ever written for it) — that is not a fault.
 */
import { describe, it, expect, beforeEach } from 'vitest'
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'

type FaultMode = 'none' | 'io' | 'corrupt' | 'missing'

/**
 * A MemoryStorage that can fault reads of persisted column SEGMENTS only
 * (keys/paths containing the `L0-` segment marker). Manifest and DELETED-bitmap
 * reads pass through untouched so a fresh store still initialises normally — the
 * fault is isolated to the exact seam finding 4 hardened.
 */
class FaultInjectingStorage extends MemoryStorage {
  public faultMode: FaultMode = 'none'

  private eio(): Error {
    const e = new Error('simulated disk read fault') as Error & { code: string }
    e.code = 'EIO'
    return e
  }

  public async loadBinaryBlob(key: string): Promise<Buffer | null> {
    if (this.faultMode !== 'none' && key.includes('/L0-')) {
      if (this.faultMode === 'io') throw this.eio()
      // Too small to hold even a header → readSegmentFromBuffer throws → wrapped.
      if (this.faultMode === 'corrupt') return Buffer.from([1, 2, 3, 4, 5])
      if (this.faultMode === 'missing') return null
    }
    return super.loadBinaryBlob(key)
  }
}

describe('ColumnStore segment-load faults surface loudly, absence stays benign (finding 4)', () => {
  let storage: FaultInjectingStorage
  let idMapper: EntityIdMapper

  beforeEach(async () => {
    storage = new FaultInjectingStorage()
    await storage.init()
    idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
    await idMapper.init()

    // Write one persisted L0 segment for `createdAt`, then close the writer.
    const writer = new ColumnStore({ flushThreshold: 10 })
    await writer.init(storage, idMapper)
    for (let i = 0; i < 5; i++) {
      writer.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), {
        createdAt: (i + 1) * 100
      })
    }
    await writer.flush()
    await writer.close()
    storage.faultMode = 'none'
  })

  // Fresh reader over the same storage: empty segment cache, so every query is
  // forced to actually load the persisted segment (that is the seam under test).
  const reopen = async (): Promise<ColumnStore> => {
    const s = new ColumnStore({ flushThreshold: 10 })
    await s.init(storage, idMapper)
    return s
  }

  it('propagates a storage IO fault verbatim — not [] and not a quarantine (a present-but-unreadable segment is not torn)', async () => {
    storage.faultMode = 'io'
    const store = await reopen()
    await expect(store.filter('createdAt', 300)).rejects.toMatchObject({
      code: 'EIO'
    })
    // An IO fault is NOT quarantined — the segment may be fine once the disk
    // recovers; only torn/absent bytes enter the ledger.
    expect(store.quarantinedSegments('createdAt')).toEqual([])
    await store.close()
  })

  it('QUARANTINES an undecodable manifest-listed segment at discovery — the query serves degraded, the ledger names the tear', async () => {
    storage.faultMode = 'corrupt'
    const store = await reopen()
    // Degraded-announced serve: the field's only segment is torn, so the
    // result is empty — but the query completes instead of throwing.
    const sorted = await store.sortTopK('createdAt', 'desc', 10)
    expect(sorted).toEqual([])
    const ledger = store.quarantinedSegments('createdAt')
    expect(ledger).toHaveLength(1)
    expect(ledger[0].error).toMatch(/decode failed/)
    expect(ledger[0].hits).toBeGreaterThanOrEqual(1)
    // Subsequent queries keep serving (skip + count), never a throw.
    const hitsBefore = ledger[0].hits
    await expect(store.filter('createdAt', 300)).resolves.toBeDefined()
    expect(store.quarantinedSegments('createdAt')[0].hits).toBeGreaterThan(hitsBefore)
    await store.close()
  })

  it('QUARANTINES a manifest-listed segment with no loadable bytes — degraded serve, ledger entry, never a throw', async () => {
    storage.faultMode = 'missing'
    const store = await reopen()
    const bitmap = await store.rangeQuery('createdAt', 100, 500)
    expect(bitmap.size).toBe(0)
    const ledger = store.quarantinedSegments('createdAt')
    expect(ledger).toHaveLength(1)
    expect(ledger[0].error).toMatch(/no loadable bytes/)
    await store.close()
  })

  it('a field with no manifest is genuine absence — returns empty, never throws', async () => {
    storage.faultMode = 'none'
    const store = await reopen()
    const bitmap = await store.filter('no_such_field', 'x')
    expect(bitmap.size).toBe(0)
    const sorted = await store.sortTopK('no_such_field', 'asc', 10)
    expect(sorted).toEqual([])
    await store.close()
  })

  it('with no fault, the persisted segment still loads and answers queries', async () => {
    storage.faultMode = 'none'
    const store = await reopen()
    const sorted = await store.sortTopK('createdAt', 'desc', 10)
    const uuids = sorted.map((id) => idMapper.getUuid(Number(id)))
    expect(uuids).toEqual(['e4', 'e3', 'e2', 'e1', 'e0'])
    await store.close()
  })
})