Merge branches 'fix/connected-find-order', 'fix/pending-embed-low-water' and 'fix/related-verb-array' into rel/10.4.9-candidate

tests/integration/pending-embed-low-water.test.ts

@@ -0,0 +1,145 @@
+/**
+ * @module tests/integration/pending-embed-low-water
+ * @description The pending-embed recovery fold is bounded and background (10.4.9).
+ *
+ * The fold used to scan the generation log from generation 1 at EVERY open,
+ * on the open's foreground — O(whole history) per open on long-lived brains.
+ * Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`)
+ * records the committed generation whenever the pending set drains to empty,
+ * recovery scans from `mark + 1`, and the fold runs behind the doors as a
+ * latched background task the worker, `awaitPendingEmbeds()` and `close()`
+ * wait on. The mark is advisory: stale-low costs a longer scan, never a
+ * marker — a pending embed enqueued before a crash is still recovered.
+ */
+import { describe, it, expect, afterEach, vi } from 'vitest'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { Brainy } from '../../src/brainy'
+import { NounType } from '../../src/types/graphTypes'
+
+const LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
+
+describe('pending-embed recovery: bounded by the low-water mark, behind the doors', () => {
+  const roots: string[] = []
+  const dir = (): string => {
+    const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-'))
+    roots.push(d)
+    return d
+  }
+  const open = async (root: string): Promise<Brainy<any>> => {
+    const brain = new Brainy<any>({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: root }
+    })
+    await brain.init()
+    return brain
+  }
+
+  afterEach(() => {
+    for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true })
+  })
+
+  it('drain-to-empty writes the mark, and the next open scans from mark + 1', async () => {
+    const root = dir()
+    const brain = await open(root)
+    // Hold the worker so the pending state is observable, then release it.
+    const realKick = (brain as any).kickEmbedWorker.bind(brain)
+    ;(brain as any).kickEmbedWorker = () => {}
+    await brain.add({
+      id: 'row-1',
+      data: 'the first deferred row',
+      type: NounType.Thing,
+      deferEmbedding: true
+    })
+    expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
+    ;(brain as any).kickEmbedWorker = realKick
+    await brain.awaitPendingEmbeds()
+    // The drain wrote the advisory mark (fire-and-forget: settle the microtask).
+    await new Promise((r) => setTimeout(r, 50))
+    const mark = (await (brain as any).storage.readRawObject(LOWWATER_PATH)) as {
+      generation: number
+    } | null
+    expect(mark).not.toBeNull()
+    expect(mark!.generation).toBeGreaterThan(0)
+    await brain.close()
+
+    const brain2 = await open(root)
+    const log = (brain2 as any).generationStore.getFactLog()
+    const scanSpy = vi.spyOn(log, 'scanFacts')
+    try {
+      await (brain2 as any).recoverPendingEmbedsFromLog()
+      expect(scanSpy).toHaveBeenCalledTimes(1)
+      const opts = scanSpy.mock.calls[0][0] as { fromGeneration?: number }
+      expect(opts.fromGeneration).toBeGreaterThanOrEqual(mark!.generation + 1)
+    } finally {
+      scanSpy.mockRestore()
+      await brain2.close()
+    }
+  })
+
+  it('a pending embed enqueued after the mark survives an unclean stop', async () => {
+    const root = dir()
+    const brain = await open(root)
+    await brain.add({ id: 'settled', data: 'lands before the mark', type: NounType.Thing })
+    await brain.awaitPendingEmbeds()
+    await new Promise((r) => setTimeout(r, 50))
+
+    // A deferred write whose embed never lands: block the worker, then drop
+    // the instance without close() — the unclean-stop shape.
+    ;(brain as any).kickEmbedWorker = () => {}
+    await brain.add({
+      id: 'orphan',
+      data: 'enqueued then abandoned',
+      type: NounType.Thing,
+      deferEmbedding: true
+    })
+    expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
+    // No close(): simulate the crash by releasing only the writer lock so the
+    // next open can proceed.
+    await (brain as any).storage.releaseWriterLock()
+
+    const brain2 = await open(root)
+    await (brain2 as any)._pendingEmbedRecovery
+    expect(brain2.pendingEmbedCount()).toBeGreaterThan(0)
+    await brain2.awaitPendingEmbeds()
+    expect(brain2.pendingEmbedCount()).toBe(0)
+    await brain2.close()
+    // Reap the crashed instance: its fence is gone, so close() fails loudly —
+    // swallow that here; the point is clearing its watchers and registry entry.
+    await brain.close().catch(() => undefined)
+  })
+
+  it('open arms the fold as a background latch; awaitPendingEmbeds waits on it', async () => {
+    const root = dir()
+    const brain = await open(root)
+    await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing })
+    await brain.awaitPendingEmbeds()
+    await brain.close()
+
+    const brain2 = await open(root)
+    // The latch exists the moment init() returns (writable filesystem brain)…
+    expect((brain2 as any)._pendingEmbedRecovery).not.toBeNull()
+    // …and the barrier settles it before answering.
+    await brain2.awaitPendingEmbeds()
+    expect(brain2.pendingEmbedCount()).toBe(0)
+    await brain2.close()
+  })
+
+  it('a clean close with an empty set writes the mark even if no drain happened', async () => {
+    const root = dir()
+    const brain = await open(root)
+    await brain.add({ id: 'r1', data: 'row one', type: NounType.Thing })
+    await brain.awaitPendingEmbeds()
+    await brain.close()
+    // Read the mark back through the storage door (the adapter owns the
+    // on-disk encoding), on a fresh instance.
+    const brain2 = await open(root)
+    const mark = (await (brain2 as any).storage.readRawObject(LOWWATER_PATH)) as {
+      generation: number
+    } | null
+    expect(mark).not.toBeNull()
+    expect(mark!.generation).toBeGreaterThan(0)
+    await brain2.close()
+  })
+})

tests/integration/related-verb-array.test.ts

@@ -0,0 +1,89 @@
+/**
+ * @module tests/integration/related-verb-array
+ * @description related() honours EVERY verb type in an array (10.4.9).
+ *
+ * The storage fast paths for `sourceId + verbType` and `verbType` collapsed a
+ * verb-type ARRAY to its first element — `related({ from, type: [a, b] })`
+ * silently returned only `a` edges, whichever order the array came in. The
+ * same quiet-loss class as the graph-first paging defect, one seam over.
+ * These pins seed a store where the SECOND requested type's edge must come
+ * back, on every path the collapse lived in.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import { Brainy } from '../../src/brainy'
+import { NounType, VerbType } from '../../src/types/graphTypes'
+import { v5 } from '../../src/universal/uuid'
+
+describe('related() with a verb-type array returns every requested type', () => {
+  let brain: Brainy<any>
+
+  beforeAll(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    for (const id of ['a', 'b', 'c', 'd']) {
+      await brain.add({ id, data: `node ${id}`, type: NounType.Person })
+    }
+    await brain.relate({ from: 'a', to: 'b', type: VerbType.Supports })
+    await brain.relate({ from: 'a', to: 'c', type: VerbType.RelatedTo })
+    await brain.relate({ from: 'a', to: 'd', type: VerbType.Knows })
+    await brain.relate({ from: 'b', to: 'c', type: VerbType.RelatedTo })
+  })
+
+  afterAll(async () => {
+    brain = null as any
+  })
+
+  it('from + type array: the second type\'s edge comes back, both orders', async () => {
+    for (const types of [
+      [VerbType.Supports, VerbType.RelatedTo],
+      [VerbType.RelatedTo, VerbType.Supports]
+    ]) {
+      const edges = await brain.related({ from: 'a', type: types })
+      const targets = new Set(edges.map((e) => e.to))
+      expect(targets.has(v5('b')), `types [${types}] missing Supports edge`).toBe(true)
+      expect(targets.has(v5('c')), `types [${types}] missing RelatedTo edge`).toBe(true)
+      expect(targets.has(v5('d'))).toBe(false)
+      expect(edges).toHaveLength(2)
+    }
+  })
+
+  it('a single-element array behaves exactly like the scalar', async () => {
+    const scalar = await brain.related({ from: 'a', type: VerbType.Supports })
+    const array = await brain.related({ from: 'a', type: [VerbType.Supports] })
+    expect(array.map((e) => e.id).sort()).toEqual(scalar.map((e) => e.id).sort())
+    expect(array).toHaveLength(1)
+  })
+
+  it('no duplicate edges when types overlap the same edge set', async () => {
+    const edges = await brain.related({
+      from: 'a',
+      type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows]
+    })
+    const ids = edges.map((e) => e.id)
+    expect(new Set(ids).size).toBe(ids.length)
+    expect(edges).toHaveLength(3)
+  })
+
+  it('type-only asks (no anchor) honour the whole array too', async () => {
+    const edges = await brain.related({ type: [VerbType.Supports, VerbType.Knows] })
+    const verbs = new Set(edges.map((e) => e.type))
+    expect(verbs.has(VerbType.Supports)).toBe(true)
+    expect(verbs.has(VerbType.Knows)).toBe(true)
+    expect(edges).toHaveLength(2)
+  })
+
+  it('to + type array: the target side honours every type too', async () => {
+    const edges = await brain.related({ to: 'c', type: [VerbType.RelatedTo, VerbType.Supports] })
+    const froms = new Set(edges.map((e) => e.from))
+    expect(froms.has(v5('a'))).toBe(true)
+    expect(froms.has(v5('b'))).toBe(true)
+    expect(edges).toHaveLength(2)
+  })
+
+  it('pagination stays consistent across the union', async () => {
+    const page1 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2 })
+    const page2 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2, offset: 2 })
+    const all = [...page1, ...page2].map((e) => e.id)
+    expect(new Set(all).size).toBe(3)
+  })
+})