perf(generations): discover generations by directory name, not by walking the log

MEASURED on a copy of a real store (14,056 nouns / 72,679 verbs, an 11 GB
generation history): the generation-store phase cost 55,538 ms of a WARM
REOPEN after a clean close — with the fold correctly skipped, so nothing in
that phase's name explained it.

This is what it was doing. Discovering which generations exist on disk called
listRawObjects('_generations'), which RECURSES the whole tree and returns
every file in every generation directory — to extract a set of integers that
the top-level directory NAMES already spell out. The cost scales with the
entire history, is paid on every open, warm or cold, and grows for the life of
the store.

A one-level door — listRawPrefixes(prefix), the immediate child directory
names — is added to the storage seam. The filesystem adapter answers it with a
single readdir; BaseStorage derives it from the recursive listing, so an
adapter without a cheap implementation is never wrong, only never faster; and
the generation store falls back to the old listing when the door is absent.

One behavioural difference, stated: an EMPTY generation directory is now
discovered where the file listing could not see it. Above the committed
watermark that is a crash scar, and recovery already has an explicit branch
for it ("indeterminate partial dir" — dropped, narrated). Below it, it becomes
a resolvable generation holding no records, which is what an empty generation
means.

Suites: the durability kill matrix (15), db-mvcc (30), history repacking (4),
rollback trapdoor (3), entity-tree stamp (4) and the full unit suite (2,105)
all green.

src/db/generationStore.ts

@@ -537,12 +537,29 @@ export class GenerationStore {
     this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon')
     this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed)
 
-    // Discover existing generation record directories.
+    // Discover existing generation record directories — BY DIRECTORY NAME.
-    const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
+    // This used to call listRawObjects(), which recurses the whole
+    // `_generations/` tree and returns every file in every generation, to
+    // extract a set of integers the top-level directory names already spell.
+    // MEASURED on a real store with an 11 GB generation history: the phase
+    // this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with
+    // no fold to blame — this walk is what it was doing. An adapter without
+    // the one-level door falls back to the recursive listing, unchanged.
     const seenGens = new Set<number>()
-    for (const p of recordPaths) {
+    const oneLevel = (
-      const gen = parseGenerationFromPath(p)
+      this.storage as { listRawPrefixes?: (prefix: string) => Promise<string[]> }
-      if (gen !== null) seenGens.add(gen)
+    ).listRawPrefixes
+    if (typeof oneLevel === 'function') {
+      for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) {
+        const gen = Number(name)
+        if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen)
+      }
+    } else {
+      const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
+      for (const p of recordPaths) {
+        const gen = parseGenerationFromPath(p)
+        if (gen !== null) seenGens.add(gen)
+      }
     }
 
     let rolledBack = 0

src/db/types.ts

@@ -450,6 +450,21 @@ export interface GenerationStorage {
   deleteRawObject(path: string): Promise<void>
   /** List raw object paths under a prefix (normalized, `.gz`-stripped). */
   listRawObjects(prefix: string): Promise<string[]>
+  /**
+   * OPTIONAL: the IMMEDIATE child directory names under a prefix — one level,
+   * no recursion, no file paths.
+   *
+   * Why it exists: discovering which generations are on disk needs only the
+   * top-level directory NAMES under `_generations/`, but the only door for it
+   * was `listRawObjects`, which recurses the whole tree and returns every file
+   * in every generation. On a store with a long history that is a full walk of
+   * the entire generation log, paid on EVERY open, to learn a set of integers
+   * the directory names already spell out.
+   *
+   * An adapter without this door keeps working — the caller falls back to the
+   * recursive listing.
+   */
+  listRawPrefixes?(prefix: string): Promise<string[]>
   /** Remove every object under a prefix (and the directory itself on disk). */
   removeRawPrefix(prefix: string): Promise<void>
   /** Durability barrier: fsync the given object paths (no-op in memory). */

src/storage/adapters/fileSystemStorage.ts

@@ -686,6 +686,30 @@ export class FileSystemStorage extends BaseStorage {
     return pruned
   }
 
+  /**
+   * @description The IMMEDIATE child directory names under a prefix — ONE
+   * `readdir`, no recursion, no file paths. See the seam's JSDoc
+   * (`src/db/types.ts`) for what this replaced: discovering the generations on
+   * disk walked the entire generation log on every open, reading out every
+   * file in every generation, to learn the set of integers the top-level
+   * directory names already spell.
+   * @param prefix - Storage-root-relative directory prefix.
+   * @returns The child directory names (not paths); empty when the prefix does
+   *   not exist.
+   */
+  public override async listRawPrefixes(prefix: string): Promise<string[]> {
+    await this.ensureInitialized()
+    const fullPath = path.join(this.rootDir, prefix)
+    try {
+      const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
+      return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory())
+        .map((e: { name: string }) => e.name)
+    } catch (error: any) {
+      if (error?.code === 'ENOENT') return []
+      throw error
+    }
+  }
+
   /**
    * Primitive operation: List objects under path prefix
    * All metadata operations use this internally via base class routing

src/storage/baseStorage.ts

@@ -1437,6 +1437,29 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     return this.listObjectsUnderPath(prefix)
   }
 
+  /**
+   * @description The IMMEDIATE child directory names under a prefix — one
+   * level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a
+   * separate door exists. This default derives them from the recursive
+   * listing, so it is never WRONG, only never faster; the filesystem adapter
+   * overrides it with a single directory read.
+   * @param prefix - Storage-root-relative directory prefix.
+   * @returns The child directory names (not paths), in listing order.
+   */
+  public async listRawPrefixes(prefix: string): Promise<string[]> {
+    await this.ensureInitialized()
+    const paths = await this.listObjectsUnderPath(prefix)
+    const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/`
+    const names = new Set<string>()
+    for (const p of paths) {
+      const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null
+      if (rest === null) continue
+      const slash = rest.search(/[/\\]/)
+      if (slash > 0) names.add(rest.slice(0, slash))
+    }
+    return [...names]
+  }
+
   /**
    * Remove every object under a storage-root-relative prefix. The filesystem
    * adapter overrides this with a recursive directory removal; this default