brainy/src/graph/graphAudit.ts

/**
 * @module graph/graphAudit
 * @description Read-only graph-truth audit — the graph sibling of `repairIndex()`'s
 * diagnosis half. Verifies three layers against each other without mutating anything:
 *
 *   1. CANONICAL verb records (the storage walk — the source of truth)
 *   2. the RELATIONSHIP READ PATH (`related()` with all visibility tiers — exactly
 *      what application reads like a VFS `readdir` consult)
 *   3. ENTITY ENDPOINTS (does each verb's source/target still exist?)
 *
 * and classifies every discrepancy into the three failure families production
 * incidents have shown:
 *
 *   - `missingFromReads` — a canonical verb record the read path does NOT return
 *     for its source: PRESENT BUT INVISIBLE (adjacency/membership staleness).
 *   - `danglingEndpoints` — a canonical verb whose endpoint entity is gone:
 *     the SCAR class (write-path loss / partial delete).
 *   - `readOnlyVerbIds` — the read path returns an edge with NO canonical
 *     record: GHOST edges (stale index entries).
 *
 * `visibilityHiddenCount` is reported separately: an internal/system edge that is
 * indexed and present but hidden from DEFAULT reads is working as designed — the
 * audit reads with all tiers included so design-hiding is never misclassified as
 * index loss.
 *
 * Full counts are always exact; only the example LISTS are capped (`maxExamples`)
 * — a capped report says so via `truncatedExamples`, never silently.
 */

import { prodLog } from '../utils/logger.js'

/** One discrepant relationship, identified fully enough to inspect by hand. */
export interface GraphAuditDiscrepancy {
  verbId: string
  from: string
  to: string
  type: string
}

export interface GraphAuditReport {
  /** True iff every discrepancy count is zero — `related()` returns canonical truth. */
  coherent: boolean
  verbsInCanonical: number
  entitiesInCanonical: number
  /** Distinct source entities whose read path was actually consulted (coverage honesty). */
  sourcesChecked: number

  /** PRESENT BUT INVISIBLE: canonical records the read path omits. */
  missingFromReadsCount: number
  missingFromReads: GraphAuditDiscrepancy[]

  /** SCAR CLASS: canonical verbs with a missing endpoint entity. */
  danglingEndpointsCount: number
  danglingEndpoints: Array<GraphAuditDiscrepancy & { missingEnd: 'from' | 'to' | 'both' }>

  /** GHOST EDGES: read-path verb ids with no canonical record. */
  readOnlyCount: number
  readOnlyVerbIds: string[]

  /** Canonical verbs hidden from DEFAULT reads by design (internal/system visibility). */
  visibilityHiddenCount: number

  /** Example lists above were capped at maxExamples; counts remain exact. */
  truncatedExamples: boolean
  durationMs: number
}

/** A canonical verb record, as the audit needs it. */
export interface AuditVerbRecord {
  id: string
  type: string
  sourceId: string
  targetId: string
  visibility?: string
}

/** The seams the audit runs over — injected so the walk is testable in isolation. */
export interface GraphAuditDeps {
  /** Stream every canonical entity id (id-only; no per-entity reads needed). */
  eachNounId(consume: (id: string) => void): Promise<void>
  /** Stream every canonical verb record. */
  eachVerb(consume: (verb: AuditVerbRecord) => void): Promise<void>
  /**
   * The END-TO-END relationship read for one source, ALL visibility tiers
   * included — must be the same path application reads consult.
   */
  readRelationsFrom(sourceId: string): Promise<Array<{ id: string }>>
}

export interface GraphAuditOptions {
  /** Cap on entries per example list (counts stay exact). Default 100. */
  maxExamples?: number
}

export async function runGraphAudit(
  deps: GraphAuditDeps,
  options: GraphAuditOptions = {}
): Promise<GraphAuditReport> {
  const maxExamples = options.maxExamples ?? 100
  const started = Date.now()

  // 1. Canonical entity ids — endpoint existence oracle.
  const entityIds = new Set<string>()
  await deps.eachNounId((id) => entityIds.add(id))

  // 2. Canonical verb walk: group by source, check endpoints, note visibility.
  const canonicalVerbIds = new Set<string>()
  const bySource = new Map<string, AuditVerbRecord[]>()
  let verbsInCanonical = 0
  let visibilityHiddenCount = 0
  let danglingEndpointsCount = 0
  const danglingEndpoints: GraphAuditReport['danglingEndpoints'] = []

  await deps.eachVerb((verb) => {
    verbsInCanonical++
    canonicalVerbIds.add(verb.id)
    const list = bySource.get(verb.sourceId)
    if (list) list.push(verb)
    else bySource.set(verb.sourceId, [verb])

    if (verb.visibility === 'internal' || verb.visibility === 'system') {
      visibilityHiddenCount++
    }

    const fromMissing = !entityIds.has(verb.sourceId)
    const toMissing = !entityIds.has(verb.targetId)
    if (fromMissing || toMissing) {
      danglingEndpointsCount++
      if (danglingEndpoints.length < maxExamples) {
        danglingEndpoints.push({
          verbId: verb.id,
          from: verb.sourceId,
          to: verb.targetId,
          type: verb.type,
          missingEnd: fromMissing && toMissing ? 'both' : fromMissing ? 'from' : 'to'
        })
      }
    }
  })

  // 3. Per-source read-path comparison. A verb must be returned by the read
  //    path of ITS OWN source — the exact consult a readdir/traversal makes.
  let missingFromReadsCount = 0
  const missingFromReads: GraphAuditDiscrepancy[] = []
  let readOnlyCount = 0
  const readOnlyVerbIds: string[] = []
  const readOnlySeen = new Set<string>()

  for (const [sourceId, verbs] of bySource) {
    const readIds = new Set((await deps.readRelationsFrom(sourceId)).map((r) => r.id))

    for (const verb of verbs) {
      if (!readIds.has(verb.id)) {
        missingFromReadsCount++
        if (missingFromReads.length < maxExamples) {
          missingFromReads.push({
            verbId: verb.id,
            from: verb.sourceId,
            to: verb.targetId,
            type: verb.type
          })
        }
      }
    }

    for (const readId of readIds) {
      if (!canonicalVerbIds.has(readId) && !readOnlySeen.has(readId)) {
        readOnlySeen.add(readId)
        readOnlyCount++
        if (readOnlyVerbIds.length < maxExamples) {
          readOnlyVerbIds.push(readId)
        }
      }
    }
  }

  const coherent =
    missingFromReadsCount === 0 && danglingEndpointsCount === 0 && readOnlyCount === 0

  const report: GraphAuditReport = {
    coherent,
    verbsInCanonical,
    entitiesInCanonical: entityIds.size,
    sourcesChecked: bySource.size,
    missingFromReadsCount,
    missingFromReads,
    danglingEndpointsCount,
    danglingEndpoints,
    readOnlyCount,
    readOnlyVerbIds,
    visibilityHiddenCount,
    truncatedExamples:
      missingFromReadsCount > missingFromReads.length ||
      danglingEndpointsCount > danglingEndpoints.length ||
      readOnlyCount > readOnlyVerbIds.length,
    durationMs: Date.now() - started
  }

  if (coherent) {
    prodLog.info(
      `[GraphAudit] coherent: ${verbsInCanonical} verbs across ${bySource.size} sources — ` +
        `the read path returns canonical truth (${report.durationMs}ms)`
    )
  } else {
    prodLog.warn(
      `[GraphAudit] INCOHERENT: ${missingFromReadsCount} present-but-invisible, ` +
        `${danglingEndpointsCount} dangling-endpoint, ${readOnlyCount} ghost ` +
        `(of ${verbsInCanonical} canonical verbs, ${bySource.size} sources, ` +
        `${visibilityHiddenCount} visibility-hidden by design) — ${report.durationMs}ms`
    )
  }

  return report
}