perf(vfs): the old-root sweep runs once per store, not once per open

MEASURED on a 14,056-noun / 72,679-verb production-shaped store, measured
solo under an exclusive lock: the vfs-bootstrap phase cost 43,021 ms of a cold open and 52,696 ms of
a WARM REOPEN. What dominates it is a migration sweep — a filtered find() over
the whole store hunting for root directories created before the fixed root id
existed. A store either carries such duplicates or never will, and the sweep
ran on every open, forever, in the foreground.

It is now caused by the store's state instead of by the open count: a durable
marker under _system/ records that the sweep has run, and a store carrying it
never sweeps again. A store without one sweeps in the BACKGROUND — the sweep
only removes duplicate roots, nothing serves from them, and it was already
declared non-critical — narrated at both ends, with whenRootSweepSettled() for
anyone who needs to observe rather than race it. An adapter with no raw-object
door keeps the old behaviour: correctness over cost, never a silent skip.

Pins: tests/integration/vfs-root-sweep-once.test.ts — the sweep runs on the
first open and never on the second or third; a sweep slowed to 4s does not
delay the open.

src/vfs/VirtualFileSystem.ts

@@ -6,6 +6,7 @@
  */
 
 import { Readable, Writable } from 'stream'
+import { prodLog } from '../utils/logger.js'
 import crypto from 'crypto'
 import { v4 as uuidv4 } from '../universal/uuid.js'
 import { Brainy } from '../brainy.js'
@@ -66,6 +67,15 @@ export class VirtualFileSystem implements IVirtualFileSystem {
   private config: Required<Omit<VFSConfig, 'rootEntityId'>> & { rootEntityId?: string }
   private rootEntityId?: string
   private initialized = false
+  /**
+   * The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}.
+   */
+  private rootSweep?: Promise<void>
+  /**
+   * Where the completed old-root sweep is recorded. Engine plumbing under
+   * `_system/`, like every other marker there — never enumerated as data.
+   */
+  private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json'
   private currentUser: string = 'system'  // Track current user for collaboration
 
   // Knowledge Layer features available via augmentation (brain.use('knowledge'))
@@ -143,8 +153,17 @@ export class VirtualFileSystem implements IVirtualFileSystem {
     // Create or find root entity
     this.rootEntityId = await this.initializeRoot()
 
-    // Clean up old UUID-based roots (one-time migration)
+    // Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS.
-    await this.cleanupOldRoots()
+    // This is a migration sweep for roots created before the fixed root id
+    // existed. It ran on EVERY open, forever: a filtered find over the whole
+    // store hunting for duplicates that a store has either always had or
+    // never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it
+    // dominates cost 43-53 SECONDS of every open, warm reopens included.
+    // Now: a durable marker records that the sweep has run, and a store
+    // carrying it never sweeps again; a store without one sweeps in the
+    // BACKGROUND (the sweep only removes duplicate roots — nothing serves
+    // from them — and it was always declared non-critical).
+    this.rootSweep = this.sweepOldRootsIfNeeded()
 
     // Initialize projection registry with auto-discovery of built-in projections
     this.projectionRegistry = new ProjectionRegistry()
@@ -394,6 +413,88 @@ export class VirtualFileSystem implements IVirtualFileSystem {
    *
    * This is a one-time migration helper that can be removed in future versions.
    */
+  /**
+   * @description Run the old-root sweep at most once per store, in the
+   * background, and record that it ran. See the call site in {@link init} for
+   * the measurement that made this necessary.
+   * @returns A promise that settles when the sweep has finished (or was
+   *   skipped); nothing in the read path awaits it.
+   */
+  private async sweepOldRootsIfNeeded(): Promise<void> {
+    const store = this.rawObjectStore()
+    if (store === null) {
+      // A storage adapter with no raw-object door cannot carry the marker.
+      // Sweep every open, as before — correctness over cost.
+      await this.cleanupOldRoots()
+      return
+    }
+    try {
+      const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH)
+      if (marker !== null && marker !== undefined) return
+    } catch {
+      // Unreadable marker: sweep, and rewrite it below.
+    }
+    prodLog.narrate(
+      '[VFS] one-time sweep for pre-fixed-id root directories running in the background — ' +
+        'the open does not wait for it, and once it has run this store never sweeps again.'
+    )
+    const startedAt = Date.now()
+    await this.cleanupOldRoots()
+    try {
+      await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, {
+        sweptAt: new Date().toISOString(),
+        durationMs: Date.now() - startedAt
+      })
+      prodLog.narrate(
+        `[VFS] old-root sweep complete in ${Date.now() - startedAt}ms and recorded — ` +
+          'no future open pays for it.'
+      )
+    } catch (error) {
+      // Unrecorded sweep = the next open sweeps again. Conservative, and said
+      // out loud rather than quietly repeated forever.
+      prodLog.narrate(
+        `[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` +
+          `recorded (${(error as Error).message}) — the next open will sweep again.`
+      )
+    }
+  }
+
+  /**
+   * @description Settle once the background old-root sweep has finished.
+   * Resolves immediately when the store already carried the marker. Exists so
+   * tests and operators can observe the sweep instead of racing it; no read
+   * path waits on it.
+   * @returns A promise that settles with the sweep.
+   */
+  public async whenRootSweepSettled(): Promise<void> {
+    await this.rootSweep
+  }
+
+  /**
+   * @description The brain's storage adapter, narrowed to the raw-object door
+   * this migration marker needs. Boundary: `Brainy.storage` is private, and
+   * this is the same reach-in the engine uses elsewhere for exactly this kind
+   * of engine-internal artifact. Returns null when the adapter has no
+   * raw-object door.
+   */
+  private rawObjectStore(): {
+    readRawObject: (key: string) => Promise<unknown>
+    writeRawObject: (key: string, value: unknown) => Promise<void>
+  } | null {
+    const storage = (this.brain as unknown as { storage?: Record<string, unknown> }).storage
+    if (
+      storage &&
+      typeof storage.readRawObject === 'function' &&
+      typeof storage.writeRawObject === 'function'
+    ) {
+      return storage as unknown as {
+        readRawObject: (key: string) => Promise<unknown>
+        writeRawObject: (key: string, value: unknown) => Promise<void>
+      }
+    }
+    return null
+  }
+
   private async cleanupOldRoots(): Promise<void> {
     try {
       // Find any old VFS roots with UUID-based IDs (not our fixed ID)

tests/integration/vfs-root-sweep-once.test.ts

@@ -0,0 +1,106 @@
+/**
+ * @module tests/integration/vfs-root-sweep-once
+ * @description THE OLD-ROOT SWEEP RUNS ONCE PER STORE, NOT ONCE PER OPEN.
+ *
+ * The VFS bootstrap ran a filtered `find()` over the whole store on EVERY
+ * open, hunting for root directories created before the fixed root id existed
+ * — duplicates a store has either always had or never will. MEASURED on a
+ * 14,056-noun / 72,679-verb store: the phase it dominates cost 43–53 SECONDS
+ * of every open, warm reopens included.
+ *
+ * The law: a migration sweep is caused by the store's state, not by the clock
+ * or the open count. It runs behind the doors, records that it ran, and a
+ * store carrying that record never sweeps again.
+ */
+
+import { describe, it, expect, afterEach, vi } from 'vitest'
+import { mkdtempSync, rmSync, existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js'
+
+describe('the VFS old-root sweep', () => {
+  const dirs: string[] = []
+  const brains: Brainy[] = []
+
+  afterEach(async () => {
+    for (const b of brains.splice(0)) {
+      try { await b.close() } catch { /* already closed */ }
+    }
+    for (const d of dirs.splice(0)) {
+      try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
+    }
+    vi.restoreAllMocks()
+  })
+
+  async function open(dir: string): Promise<Brainy> {
+    const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
+    brains.push(brain)
+    await brain.init()
+    return brain
+  }
+
+  it('sweeps on the first open, records it, and never sweeps again', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-'))
+    dirs.push(dir)
+
+    const sweepSpy = vi.spyOn(
+      VirtualFileSystem.prototype as unknown as { cleanupOldRoots: () => Promise<void> },
+      'cleanupOldRoots'
+    )
+
+    const first = await open(dir)
+    await (first.vfs as unknown as { whenRootSweepSettled: () => Promise<void> }).whenRootSweepSettled()
+    expect(sweepSpy).toHaveBeenCalledTimes(1)
+    // The record is durable engine plumbing under _system/, like every other marker.
+    expect(
+      existsSync(join(dir, '_system', 'vfs-root-sweep.json')) ||
+        existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz'))
+    ).toBe(true)
+
+    await first.add({ data: 'a row so the store is not trivially empty', type: NounType.Concept })
+    await first.flush()
+    await first.close()
+    brains.splice(brains.indexOf(first), 1)
+
+    sweepSpy.mockClear()
+    const second = await open(dir)
+    await (second.vfs as unknown as { whenRootSweepSettled: () => Promise<void> }).whenRootSweepSettled()
+    expect(sweepSpy).not.toHaveBeenCalled()
+
+    await second.close()
+    brains.splice(brains.indexOf(second), 1)
+
+    // ...and a third open, to prove it is the record and not a one-off.
+    sweepSpy.mockClear()
+    const third = await open(dir)
+    await (third.vfs as unknown as { whenRootSweepSettled: () => Promise<void> }).whenRootSweepSettled()
+    expect(sweepSpy).not.toHaveBeenCalled()
+  }, 180_000)
+
+  it('the open does not wait for the sweep', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-'))
+    dirs.push(dir)
+
+    const proto = VirtualFileSystem.prototype as unknown as Record<
+      string,
+      (...args: unknown[]) => Promise<unknown>
+    >
+    const real = proto.cleanupOldRoots
+    proto.cleanupOldRoots = async function slow(this: unknown, ...args: unknown[]) {
+      await new Promise((r) => setTimeout(r, 4_000))
+      return real.apply(this, args)
+    }
+    try {
+      const startedAt = Date.now()
+      const brain = await open(dir)
+      const openMs = Date.now() - startedAt
+      expect(openMs).toBeLessThan(3_000)
+      await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise<void> }).whenRootSweepSettled()
+    } finally {
+      proto.cleanupOldRoots = real
+    }
+  }, 180_000)
+})