open-brainy/src/utils/entityIdMapper.ts

/**
 * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs.
 *
 * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings
 * as canonical entity IDs. This class provides efficient O(1) bidirectional
 * mapping with persistence — and, importantly, a stability guarantee that any
 * persisted int-keyed data can rely on.
 *
 * **Stability guarantee (the foundation 2.4.0 vector-mmap, graph-link-compression,
 * and column-store interchange all key off):**
 *
 * - `getOrAssign(uuid)` is **append-only**: once a UUID is assigned an int, the
 *   mapping never changes. Subsequent `getOrAssign` calls for the same UUID
 *   return the same int.
 * - `nextId` is **monotonically increasing**. New UUIDs always get an int greater
 *   than any previously assigned, so a removed-then-re-added UUID is treated as
 *   a fresh entity (and gets a fresh int — there is no automatic "revive").
 * - `remove(uuid)` removes the mapping but does **not** decrement `nextId` or
 *   recycle the int. The removed int becomes a permanent hole in `intToUuid` —
 *   downstream consumers seeing `getUuid(int) === undefined` know the entity
 *   was deleted.
 * - A metadata-index `rebuild()` does **not** clear the mapper (the rebuild path
 *   re-iterates entities via `getOrAssign`, which returns existing ints unchanged).
 *   Only the explicit `clear()` method renumbers — used by `clearAllIndexData()`
 *   as the nuclear recovery path with a documented warning.
 *
 * Features:
 * - O(1) lookup in both directions.
 * - Persistent storage via storage adapter.
 * - Atomic, monotonic, append-only int counter.
 * - Serialization/deserialization support.
 *
 * @module utils/entityIdMapper
 */

import type { StorageAdapter } from '../coreTypes.js'
import type { EntityIdMapperProvider } from '../plugin.js'

/**
 * The largest entity int the JS fallback `EntityIdMapper` will allocate.
 * The metadata index's roaring bitmaps are 32-bit-keyed (`Roaring32`), so
 * allowing the JS counter past this point would silently corrupt every
 * downstream bitmap. The cor 3.0 binary mapper supports a U64 IdSpace
 * for brains above this ceiling — see the
 * [`EntityIdSpaceExceeded`](#EntityIdSpaceExceeded) message for the
 * migration pointer.
 */
export const U32_ENTITY_ID_MAX = 0xffff_ffff

/**
 * Thrown by the JS fallback `EntityIdMapper` when `nextId` would exceed
 * [`U32_ENTITY_ID_MAX`](#U32_ENTITY_ID_MAX). The JS path is the
 * cor-free fallback; once a brain has more than ~4.29 B entities,
 * callers MUST install the cor 3.0 `NativeBinaryEntityIdMapper` with
 * `idSpace: 'u64'` (mmap-backed extendible-hash KV; persists the full
 * u64 range losslessly).
 *
 * Brainy 8.0 surfaces this loudly rather than silently widening past
 * u32::MAX, because the metadata index's roaring bitmaps would silently
 * truncate entity ids and zero-out query results.
 */
export class EntityIdSpaceExceeded extends Error {
  /** The u32 entity-id ceiling. */
  readonly ceiling: number = U32_ENTITY_ID_MAX
  /**
   * The would-be `nextId` value the mapper was about to assign — always
   * `U32_ENTITY_ID_MAX + 1`.
   */
  readonly attempted: number

  constructor(attempted: number) {
    super(
      `EntityIdMapper: nextId ${attempted} would exceed u32::MAX ` +
        `(${U32_ENTITY_ID_MAX}). The JS fallback mapper caps at u32 to ` +
        `match the metadata index's Roaring32 bitmap width. For >4.29 B ` +
        `entities, install @soulcraft/cor and configure the binary ` +
        `mapper with idSpace: 'u64' (mmap-backed extendible-hash KV).`,
    )
    this.name = 'EntityIdSpaceExceeded'
    this.attempted = attempted
  }
}

export interface EntityIdMapperOptions {
  storage: StorageAdapter
  storageKey?: string
}

export interface EntityIdMapperData {
  nextId: number
  uuidToInt: Record<string, number>
  intToUuid: Record<number, string>
}

/**
 * Maps entity UUIDs to integer IDs for use with Roaring Bitmaps.
 *
 * Implements {@link EntityIdMapperProvider}: the surface a registered
 * `'entityIdMapper'` provider (e.g. Cor's native mapper) must also satisfy.
 */
export class EntityIdMapper implements EntityIdMapperProvider {
  private storage: StorageAdapter
  private storageKey: string

  // Bidirectional maps
  private uuidToInt = new Map<string, number>()
  private intToUuid = new Map<number, string>()

  // Atomic counter for next ID
  private nextId = 1

  // Dirty flag for persistence
  private dirty = false

  constructor(options: EntityIdMapperOptions) {
    this.storage = options.storage
    this.storageKey = options.storageKey || 'brainy:entityIdMapper'
  }

  /**
   * Initialize the mapper by loading from storage
   */
  async init(): Promise<void> {
    try {
      const metadata = await this.storage.getMetadata(this.storageKey)
      // metadata IS the data (no nested 'data' property)
      if (metadata && metadata.nextId !== undefined) {
        // Typed boundary: mapper state round-trips through the storage
        // metadata channel as plain JSON; the `nextId` probe above identifies
        // the persisted EntityIdMapperData shape.
        const data = metadata as unknown as EntityIdMapperData

        // TORN-STATE VALIDATION (power-loss survivor): a torn mapper file
        // can carry NaN/garbage where integers belong — unvalidated, those
        // NaNs reach BigInt() on the graph's int-resolution (reopen) and
        // the mint path (first write after recovery) and kill both with
        // RangeErrors. A torn mapper is DISCARDED with narration and the
        // maps re-derive through the existing rebuild path (under log
        // authority the mint-at-append records reproduce assignments
        // exactly; under tree authority the metadata-index reconstruction
        // rebuilds them — the same path a missing mapper file takes).
        const validInt = (v: unknown): v is number =>
          typeof v === 'number' && Number.isSafeInteger(v) && v >= 0
        let torn = !validInt(data.nextId)
        const uuidToInt = new Map<string, number>()
        const intToUuid = new Map<number, string>()
        if (!torn) {
          for (const [k, v] of Object.entries(data.uuidToInt ?? {})) {
            const n = Number(v)
            if (!validInt(n)) { torn = true; break }
            uuidToInt.set(k, n)
          }
        }
        if (!torn) {
          for (const [k, v] of Object.entries(data.intToUuid ?? {})) {
            const n = Number(k)
            if (!validInt(n) || typeof v !== 'string') { torn = true; break }
            intToUuid.set(n, v)
          }
        }
        if (torn) {
          console.warn(
            `[EntityIdMapper] persisted mapper state is TORN (non-integer ids — ` +
              `power-loss survivor); discarding and re-deriving via the rebuild ` +
              `path. Never a RangeError at reopen or first write.`
          )
          this.nextId = 1
          this.uuidToInt = new Map()
          this.intToUuid = new Map()
        } else {
          this.nextId = data.nextId
          this.uuidToInt = uuidToInt
          this.intToUuid = intToUuid
        }
      } else {
        // Guard: mapper file missing but entities may exist on disk.
        // If we start from nextId=1 with existing entities, roaring bitmap
        // queries will use wrong integer IDs → silent data corruption.
        // Probe storage to detect this case and log a warning.
        try {
          const probe = await this.storage.getNouns({ pagination: { limit: 1, offset: 0 } })
          if ((probe.totalCount ?? 0) > 0 || probe.items.length > 0) {
            console.warn(
              `[EntityIdMapper] Mapper file missing but entities exist on disk. ` +
              `IDs will be rebuilt during metadata index reconstruction.`
            )
          }
        } catch {
          // Storage not ready
        }
      }
    } catch (error) {
      // First time initialization - maps are empty, nextId = 1
    }
  }

  /**
   * Get integer ID for UUID, assigning a new ID if not exists.
   *
   * The JS fallback mapper caps at [`U32_ENTITY_ID_MAX`](#U32_ENTITY_ID_MAX)
   * to match the metadata index's Roaring32 bitmap width — once `nextId`
   * would exceed that, throws {@link EntityIdSpaceExceeded} so the caller
   * loudly migrates to cor's binary mapper with `idSpace: 'u64'`
   * rather than silently truncating entity ids.
   *
   * @param generation - Brainy's commit generation current at mint time
   *   (contract parity with the `EntityIdMapperProvider` surface). This JS
   *   mapper keeps a snapshot file, not a per-record delta log, so there is
   *   no natural slot to store it — accepted and ignored; a native mapper
   *   stamps its assignment records with it.
   */
  getOrAssign(uuid: string, generation?: bigint): number {
    void generation // Contract parity — no per-record log in the JS mapper.
    const existing = this.uuidToInt.get(uuid)
    if (existing !== undefined) {
      return existing
    }

    // Assign new ID. Source guard: nextId must be a finite positive integer
    // — the load path validates persisted state, but a NaN here would mint
    // poison ints that reach BigInt() downstream; heal to the map-derived
    // floor with narration rather than propagate.
    if (!Number.isSafeInteger(this.nextId) || this.nextId < 1) {
      let floor = 1
      for (const n of this.intToUuid.keys()) if (n >= floor) floor = n + 1
      console.warn(
        `[EntityIdMapper] nextId was non-integer (${String(this.nextId)}) — ` +
          `healed to ${floor} from the live map; torn-state survivor`
      )
      this.nextId = floor
    }
    if (this.nextId > U32_ENTITY_ID_MAX) {
      throw new EntityIdSpaceExceeded(this.nextId)
    }
    const newId = this.nextId++
    this.uuidToInt.set(uuid, newId)
    this.intToUuid.set(newId, uuid)
    this.dirty = true

    return newId
  }

  /**
   * Get integer ID for UUID with immediate persistence guarantee
   * Unlike getOrAssign(), this method flushes to storage immediately after assigning
   * a new ID. This prevents UUID→int mapping divergence if the process crashes
   * before a normal flush() occurs.
   *
   * Use this for critical operations where data integrity is paramount.
   * Normal operations can use getOrAssign() with batched flushing for better performance.
   */
  async getOrAssignSync(uuid: string): Promise<number> {
    const id = this.getOrAssign(uuid)

    // If a new ID was assigned, immediately persist to storage
    if (this.dirty) {
      await this.flush()
    }

    return id
  }

  /**
   * Get UUID for integer ID
   */
  getUuid(intId: number): string | undefined {
    return this.intToUuid.get(intId)
  }

  /**
   * Get integer ID for UUID (without assigning if not exists)
   */
  getInt(uuid: string): number | undefined {
    return this.uuidToInt.get(uuid)
  }

  /**
   * Check if UUID has been assigned an integer ID
   */
  has(uuid: string): boolean {
    return this.uuidToInt.has(uuid)
  }

  /**
   * Remove mapping for UUID
   *
   * @param generation - Brainy's commit generation for this removal (contract
   *   parity with the `EntityIdMapperProvider` surface). Accepted and ignored —
   *   this JS mapper removes immediately; a native mapper tombstones the
   *   mapping at this generation in its version chain.
   */
  remove(uuid: string, generation?: bigint): boolean {
    void generation // Contract parity — no per-key version chain in the JS mapper.
    const intId = this.uuidToInt.get(uuid)
    if (intId === undefined) {
      return false
    }

    this.uuidToInt.delete(uuid)
    this.intToUuid.delete(intId)
    this.dirty = true

    return true
  }

  /**
   * Get total number of mappings
   */
  get size(): number {
    return this.uuidToInt.size
  }

  /**
   * Get all mapped integer IDs without storage reads.
   * Used for bitmap negation operations (ne, exists:false, missing:true)
   * to avoid full-table getAllIds() scans.
   */
  getAllIntIds(): number[] {
    return Array.from(this.intToUuid.keys())
  }

  /**
   * Convert array of UUIDs to array of integers
   */
  uuidsToInts(uuids: string[]): number[] {
    return uuids.map(uuid => this.getOrAssign(uuid))
  }

  /**
   * Convert array of integers to array of UUIDs
   */
  intsToUuids(ints: number[]): string[] {
    const result: string[] = []
    for (const intId of ints) {
      const uuid = this.intToUuid.get(intId)
      if (uuid) {
        result.push(uuid)
      }
    }
    return result
  }

  /**
   * Convert iterable of integers to array of UUIDs (for roaring bitmap iteration)
   */
  intsIterableToUuids(ints: Iterable<number>): string[] {
    const result: string[] = []
    for (const intId of ints) {
      const uuid = this.intToUuid.get(intId)
      if (uuid) {
        result.push(uuid)
      }
    }
    return result
  }

  /**
   * @description Batch reverse-resolve u64 entity ints (a `BigInt64Array`) → UUID
   * strings — the bigint counterpart of {@link EntityIdMapper.intsIterableToUuids},
   * for the native graph engine whose `Subgraph.nodes` is a `BigInt64Array`. The JS
   * mapper is keyed by `number` (u32-era); each int is narrowed via `Number()` — safe
   * for the JS fallback's id range. Order-preserving (one entry per input): an int the
   * mapper has not assigned yields `''` (does not occur for graph-engine results, which
   * reference assigned ints only).
   * @param nodeInts - Entity ints as returned in a graph `Subgraph`.
   * @returns One UUID per input, in order (`''` for a never-assigned int).
   * @example
   * const uuids = mapper.entityIntsToUuids(subgraph.nodes)
   */
  entityIntsToUuids(nodeInts: BigInt64Array): string[] {
    const result: string[] = new Array(nodeInts.length)
    for (let i = 0; i < nodeInts.length; i++) {
      result[i] = this.intToUuid.get(Number(nodeInts[i])) ?? ''
    }
    return result
  }

  /**
   * Flush mappings to storage
   */
  async flush(): Promise<void> {
    if (!this.dirty) {
      return
    }

    // Convert maps to plain objects for serialization
    // Add required 'noun' property for NounMetadata
    const data = {
      noun: 'EntityIdMapper',
      nextId: this.nextId,
      uuidToInt: Object.fromEntries(this.uuidToInt),
      intToUuid: Object.fromEntries(this.intToUuid)
    }

    await this.storage.saveMetadata(this.storageKey, data)
    this.dirty = false
  }

  /**
   * Clear all mappings
   */
  async clear(): Promise<void> {
    this.uuidToInt.clear()
    this.intToUuid.clear()
    this.nextId = 1
    this.dirty = true
    await this.flush()
  }

  // No `rebuild()` on the JS mapper by design (CTX-BR-RESTORE-REBUILD): after a
  // `brain.restore()`, `MetadataIndex.rebuild()` re-derives this mapper from the
  // restored entities via append-only `getOrAssign` (the ints it picks are
  // internally consistent with the bitmaps it builds), so the JS path needs no
  // explicit post-restore reload — and forcing one (blanking + reloading) would
  // drop a still-referenced mapping when the snapshot omits the mapper file. The
  // `rebuild()` reload is the NATIVE mapper's concern: cor's binary KV holds the
  // authoritative int↔uuid the native adjacency is keyed on, so it must reload
  // from the restored KV before the native graph rebuild. `restore()` therefore
  // calls `rebuild()` only when the provider implements it (see brainy.ts).

  /**
   * Get statistics about the mapper
   */
  getStats() {
    return {
      mappings: this.uuidToInt.size,
      nextId: this.nextId,
      dirty: this.dirty,
      memoryEstimate: this.uuidToInt.size * (36 + 8 + 4 + 8) // uuid string + map overhead + int + map overhead
    }
  }
}