diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index da5887f6..5e93cd96 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -5,10 +5,6 @@ name: CI # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # tag's publish-source run and starve every release (observed on 8.10.3 and # 9.0.0: the publish sat behind the tag's own redundant CI). -concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true - on: push: branches: ['**'] diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index 6bd42b2a..58cb1d30 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -12,11 +12,6 @@ on: push: tags: - 'v*' - workflow_dispatch: - inputs: - ref_reason: - description: 'why this manual run (e.g. tag event dropped)' - required: false jobs: publish: diff --git a/CHANGELOG.md b/CHANGELOG.md index a54d609e..7154d5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31) + +- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d) + + +### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31) + +- build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f) +- fix(generations): a sealed segment may only declare the generations it holds (a963a744) +- fix(recovery): a torn generation-log tail is a terminal verdict, never a wait (c9930871) + + ### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) - fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) diff --git a/RELEASES.md b/RELEASES.md index c875cb26..e8833b80 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,12 +1,5 @@ # @soulcraft/brainy — Release Notes for Consumers -Machine-readable release notes are published at -https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json -(this engine) and -https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json -(the product engine) — read by HQ's `/hq/releases` door, and the source of -truth ahead of this file. - This file is the **quick reference for downstream sessions** tracking Brainy changes. Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases diff --git a/package-lock.json b/package-lock.json index c4f66561..9e573da3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.6", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 06ce0253..51322998 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.6", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", diff --git a/scripts/release.sh b/scripts/release.sh index 142fa06f..1a4fe575 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,8 +154,7 @@ else fi # Create new changelog entry -RELEASE_DATE=$(date +%Y-%m-%d) -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE}) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -175,19 +174,6 @@ if [ -f "CHANGELOG.md" ]; then fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" -# Step 6b: Update the releases wall entry — mechanical, derived from the -# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json -# from the one shared releases repo, soulcraftlabs/releases on The Source — -# this used to be hand-written after every release (David: never again — -# make it a step of the rail, landed in the one shared home; this repo no -# longer hosts its own copy). This step clones/fetches that repo into a -# local cache, prepends the entry, and pushes it directly — a real -# cross-repo push, refusing loudly (never skipping) on any -# clone/validation/commit/push failure. -echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" -node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md -echo -e "${GREEN}✅ Releases wall updated${NC}\n" - # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" git add package.json package-lock.json CHANGELOG.md @@ -251,7 +237,7 @@ fi # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then - if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then echo -e "${GREEN}✅ Release page created on The Source${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs deleted file mode 100644 index d4ec7ba5..00000000 --- a/scripts/wall-entry.mjs +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env node -/** - * @module scripts/wall-entry - * @description The releases-wall entry, made mechanical. The fleet's HQ page - * reads one public JSON per product from the ONE releases repo on The Source - * (soulcraftlabs/releases, files .json at its root — shape - * {product, entries:[{version, date, headline, items, url, thumb?}]}), at - * https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/.json. - * Those entries were hand-written after every release, then briefly written - * into this repo's own releases/.json; this script is the one door - * that composes an entry and lands it in the shared repo, so it is never - * hand-written and never forked across repos again. - * - * Two modes: - * - * 1. Generate + publish (default): - * node wall-entry.mjs --product

--version --date \ - * --from-changelog - * Derives an entry from the CHANGELOG.md entry for (headline = the - * entry's first bullet, items = every bullet, trimmed of its trailing - * commit hash), then: - * - clones (or, if a cached clone already exists, fetches and resets) - * the releases repo into a local cache directory, - * - prepends the entry to /

.json, newest first — replacing - * any existing entry for the same version so a re-run is idempotent, - * - validates the file's shape before and after, - * - commits the change as "chore(wall):

" and pushes main. - * A failure at any step (clone, validation, commit, push, a - * non-fast-forward remote) exits non-zero naming the cure. Nothing is - * ever skipped — the wall either lands correctly or the release fails. - * - * 2. Dry run: - * node wall-entry.mjs --dry-run --product

--version \ - * --date --from-changelog - * Derives the entry exactly as above and prints it, along with the file - * it would be written to, but touches no clone and no remote — usable - * from a fresh checkout with no cache and no network. - * - * 3. Validate only (--check): - * node wall-entry.mjs --check --file - * Validates an arbitrary wall file's exact key set (top-level and - * per-entry), field types, and strict-descending semver ordering with - * no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 = - * named violations printed to stderr. - * - * The remote and the local cache directory are each overridable - * (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE / - * WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local - * bare repo and a throwaway cache directory — never the real remote or the - * real developer cache. - * - * No dependencies beyond the system `git` binary — CHANGELOG parsing, - * semver comparison, and JSON shape checking are all hand-rolled below. - */ - -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' -import { execFileSync } from 'node:child_process' -import { homedir } from 'node:os' -import { dirname, join } from 'node:path' - -const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git' - -/** @returns {string} */ -function defaultCacheDir() { - const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache') - return join(base, 'soulcraft-releases') -} - -// Required on every entry; "thumb" is optional (may be absent, or present as -// string | null) — matching the HQ contract's {..., thumb?}. -const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url'] -const ENTRY_OPTIONAL_KEYS = ['thumb'] -const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] -const FILE_KEYS = ['product', 'entries'] - -// The public permalink pattern, by product. Every entry MUST carry an https -// permalink: HQ's parser rejects a wall whose entries carry url: null (the -// whole feed became unreadable on 2026-09-02). A product whose forge repo is -// private links its PUBLIC package page on The Source instead of a release -// page that would 404 for HQ's readers. -const RELEASE_URL_PATTERNS = { - 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, - 'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`, -} - -/** - * Parse argv into a flag map. `--flag value` sets a string; `--flag` alone - * (end of argv, or followed by another `--flag`) sets boolean true. - * @param {string[]} argv - * @returns {Record} - */ -function parseArgs(argv) { - /** @type {Record} */ - const args = {} - for (let i = 0; i < argv.length; i++) { - const a = argv[i] - if (!a.startsWith('--')) continue - const key = a.slice(2) - const next = argv[i + 1] - if (next === undefined || next.startsWith('--')) { - args[key] = true - } else { - args[key] = next - i++ - } - } - return args -} - -/** - * Print a loud, named error and exit 1. Every refusal in this script goes - * through here so the failure mode is always the same shape: "wall-entry: ". - * @param {string} message - * @returns {never} - */ -function fail(message) { - console.error(`wall-entry: ${message}`) - process.exit(1) -} - -/** - * @param {string} version - * @returns {{major: number, minor: number, patch: number, pre: string | null} | null} - */ -function parseSemver(version) { - const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version) - if (!m) return null - return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null } -} - -/** - * @param {string} a - * @param {string} b - * @returns {number} positive if a > b, negative if a < b, 0 if equal. - */ -function compareSemver(a, b) { - const pa = parseSemver(a) - const pb = parseSemver(b) - if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`) - if (pa.major !== pb.major) return pa.major - pb.major - if (pa.minor !== pb.minor) return pa.minor - pb.minor - if (pa.patch !== pb.patch) return pa.patch - pb.patch - if (pa.pre === pb.pre) return 0 - if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version - if (pb.pre === null) return -1 - return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0 -} - -/** - * Validate a wall file's full shape: top-level keys ("product", "entries" — - * no more, no less), per-entry keys and field types ("thumb" optional), and - * strict-descending semver ordering with no duplicates. Collects every - * violation instead of failing on the first, so a caller reports the whole - * picture in one pass. - * @param {unknown} data - * @returns {string[]} Violation messages; empty means the file is clean. - */ -function validateShape(data) { - /** @type {string[]} */ - const errors = [] - - if (typeof data !== 'object' || data === null || Array.isArray(data)) { - return ['top level: expected a JSON object'] - } - const obj = /** @type {Record} */ (data) - - const topKeys = Object.keys(obj) - const missingTop = FILE_KEYS.filter((k) => !(k in obj)) - const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k)) - if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`) - if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`) - - if (typeof obj.product !== 'string' || obj.product.trim() === '') { - errors.push('top level: "product" must be a non-empty string') - } - if (!Array.isArray(obj.entries)) { - errors.push('top level: "entries" must be an array') - return errors // nothing further to check without an array - } - - const entries = /** @type {unknown[]} */ (obj.entries) - entries.forEach((rawEntry, i) => { - const label = `entries[${i}]` - if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) { - errors.push(`${label}: expected an object`) - return - } - const entry = /** @type {Record} */ (rawEntry) - const keys = Object.keys(entry) - const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry)) - const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k)) - if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) - if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) - - if (typeof entry.version !== 'string' || !parseSemver(entry.version)) { - errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`) - } - if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) { - errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`) - } - if (typeof entry.headline !== 'string' || entry.headline.trim() === '') { - errors.push(`${label}: "headline" must be a non-empty string`) - } - if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { - errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) - } - if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) { - errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`) - } - if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { - errors.push(`${label}: "thumb" must be a string or null when present`) - } - }) - - // Ordering: newest first, strictly descending, no duplicate versions — - // checked only over entries whose version parsed (a bad version is - // already reported above; comparing it too would just be noise). - const versioned = entries - .map((e, i) => ({ i, version: /** @type {any} */ (e)?.version })) - .filter((e) => typeof e.version === 'string' && parseSemver(e.version)) - for (let i = 0; i < versioned.length - 1; i++) { - const a = versioned[i] - const b = versioned[i + 1] - const cmp = compareSemver(a.version, b.version) - if (cmp === 0) { - errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`) - } else if (cmp < 0) { - errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`) - } - } - - return errors -} - -/** - * Extract one version's entry body from a standard-version-style CHANGELOG.md - * (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines - * until the next heading or EOF). - * @param {string} changelog - * @param {string} version - * @returns {string[]} Bullet lines, trimmed of their leading "- " and - * trailing " (hash)". - */ -function extractChangelogBullets(changelog, version) { - const lines = changelog.split('\n') - const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/ - let start = -1 - for (let i = 0; i < lines.length; i++) { - const m = headingRe.exec(lines[i]) - if (m && m[1] === version) { - start = i + 1 - break - } - } - if (start === -1) { - fail( - `version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`, - ) - } - /** @type {string[]} */ - const bullets = [] - for (let i = start; i < lines.length; i++) { - if (headingRe.test(lines[i])) break // next entry starts - const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim()) - if (lines[i].trim().startsWith('- ') && bulletMatch) { - const text = bulletMatch[1].trim() - if (text) bullets.push(text) - } - } - if (bullets.length === 0) { - fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`) - } - return bullets -} - -/** - * Derive a wall entry from a CHANGELOG.md. - * @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts - * @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} - */ -function deriveEntry({ product, version, date, changelogPath, url, thumb }) { - if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) - if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) { - fail(`--date "${date}" is not a YYYY-MM-DD date`) - } - if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`) - - const changelog = readFileSync(changelogPath, 'utf8') - const items = extractChangelogBullets(changelog, version) - const headline = items[0] - - const pattern = RELEASE_URL_PATTERNS[product] - if (url === undefined && pattern === undefined) { - throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`) - } - const resolvedUrl = url !== undefined ? url : pattern(version) - const resolvedThumb = thumb !== undefined ? thumb : null - - return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } -} - -/** - * Load and shape-validate a wall file. - * @param {string} filePath - * @returns {Record} - */ -function loadWallFile(filePath) { - if (!existsSync(filePath)) fail(`"${filePath}" does not exist`) - /** @type {unknown} */ - let data - try { - data = JSON.parse(readFileSync(filePath, 'utf8')) - } catch (err) { - fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) - } - const errors = validateShape(data) - if (errors.length) { - fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`) - } - return /** @type {Record} */ (data) -} - -/** - * Run a git command, throwing an Error whose message is git's own stderr - * (trimmed) on failure — every caller wraps this to name the cure. - * @param {string[]} args - * @param {string} cwd - * @returns {string} stdout, trimmed. - */ -function git(args, cwd) { - try { - return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim() - } catch (err) { - const stderr = /** @type {any} */ (err).stderr - const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message - throw new Error(message) - } -} - -/** - * Ensure a clean, up-to-date local clone of the releases repo at - * `cacheDir`, checked out on `main` — cloning fresh if `cacheDir` has no - * `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a - * stray local commit or edit left by a previous failed run can never leak - * into the next one). - * @param {string} remote - * @param {string} cacheDir - */ -function ensureReleasesClone(remote, cacheDir) { - if (existsSync(join(cacheDir, '.git'))) { - try { - git(['remote', 'set-url', 'origin', remote], cacheDir) - git(['fetch', '--prune', 'origin'], cacheDir) - git(['checkout', 'main'], cacheDir) - git(['reset', '--hard', 'origin/main'], cacheDir) - git(['clean', '-fd'], cacheDir) - } catch (err) { - fail( - `cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` + - ` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`, - ) - } - return - } - - mkdirSync(dirname(cacheDir), { recursive: true }) - try { - git(['clone', remote, cacheDir], dirname(cacheDir)) - } catch (err) { - fail( - `cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` + - ` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`, - ) - } - try { - git(['checkout', 'main'], cacheDir) - } catch (err) { - fail( - `cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` + - ` cure: confirm the releases repo's default branch is named "main"`, - ) - } -} - -/** - * Prepend `entry` to the wall at `/.json`, replacing any - * existing entry for the same version (idempotent re-runs), validating - * before and after, committing, and pushing — or refusing loudly, naming - * the cure, at whichever step fails. - * @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry - * @param {string} product - * @param {string} remote - * @param {string} cacheDir - */ -function publishEntry(entry, product, remote, cacheDir) { - ensureReleasesClone(remote, cacheDir) - - const filePath = join(cacheDir, `${product}.json`) - if (!existsSync(filePath)) { - fail( - `"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`, - ) - } - const wall = loadWallFile(filePath) - - if (wall.product !== product) { - fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`) - } - - const replacing = wall.entries.some((e) => e.version === entry.version) - wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)] - - const postErrors = validateShape(wall) - if (postErrors.length) { - fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`) - } - - writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') - - const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir) - if (status === '') { - console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`) - return - } - - try { - git(['add', `${product}.json`], cacheDir) - git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir) - } catch (err) { - fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`) - } - - try { - git(['push', 'origin', 'main'], cacheDir) - } catch (err) { - fail( - `push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` + - ` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`, - ) - } - - const sha = git(['rev-parse', 'HEAD'], cacheDir) - console.log( - `wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`, - ) -} - -function main() { - const args = parseArgs(process.argv.slice(2)) - - if (args.check) { - const filePath = /** @type {string | undefined} */ (args.file) - if (!filePath) fail('--check needs --file ') - const wall = loadWallFile(/** @type {string} */ (filePath)) - console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) - process.exit(0) - } - - // Generate mode (default, also covers --dry-run): --product, --version, - // --date, --from-changelog required. - const product = /** @type {string | undefined} */ (args.product) - const version = /** @type {string | undefined} */ (args.version) - const date = /** @type {string | undefined} */ (args.date) - const fromChangelog = /** @type {string | undefined} */ (args['from-changelog']) - - const missing = [] - if (!product) missing.push('--product') - if (!version) missing.push('--version') - if (!date) missing.push('--date') - if (!fromChangelog) missing.push('--from-changelog') - if (missing.length) { - fail( - `missing required flag(s): ${missing.join(', ')}\n` + - 'Usage:\n' + - ' wall-entry.mjs --product

--version --date --from-changelog [--dry-run]\n' + - ' wall-entry.mjs --check --file ', - ) - } - - const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url) - const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb) - - const entry = deriveEntry({ - product: /** @type {string} */ (product), - version: /** @type {string} */ (version), - date: /** @type {string} */ (date), - changelogPath: /** @type {string} */ (fromChangelog), - url: urlArg, - thumb: thumbArg, - }) - - const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE) - const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir()) - - if (args['dry-run']) { - console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`) - console.log(JSON.stringify(entry, null, 2)) - process.exit(0) - } - - publishEntry(entry, /** @type {string} */ (product), remote, cacheDir) -} - -main() diff --git a/src/brainy.ts b/src/brainy.ts index da04577e..464f689b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -15,6 +15,7 @@ import { JsHnswVectorIndex } from './hnsw/hnswIndex.js' import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js' import type { StorageOptions } from './storage/storageFactory.js' import { rebuildCounts } from './utils/rebuildCounts.js' +import { jsonSafeIndexMetadata } from './utils/jsonSafeIndexMetadata.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' import { @@ -1819,31 +1820,31 @@ export class Brainy implements BrainyInterface { // a deferred write's ack and its background embed DELAYED a vector; // this is where it lands. if (!this.isReadOnly) { - try { - await step( - 'bridge-pending-embed-sidecars', - 'migrating any pre-log deferred-embed marker files into the generation log', - () => this.bridgeLegacyPendingEmbedSidecars() - ) - await step( - 'recover-pending-embeds', - 'folding the generation log\'s deferred-embed markers back into the pending set', - () => this.recoverPendingEmbedsFromLog() - ) - if (this._pendingEmbedIds.size > 0) { - prodLog.info( - `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + - `session — resuming in the background` + // BEHIND THE DOORS (the open pays nothing here): the bridge + the + // recovery fold run as one latched background task; the embed worker + // starts when it settles. A pending embed's outcome was always + // eventual — moving its recovery off the open's foreground changes + // when the worker starts, never whether a marker is honored. + // awaitPendingEmbeds() and close() wait on the latch first. + this._pendingEmbedRecovery = (async () => { + try { + await this.bridgeLegacyPendingEmbedSidecars() + await this.recoverPendingEmbedsFromLog() + if (this._pendingEmbedIds.size > 0) { + prodLog.info( + `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + + `session — resuming in the background` + ) + const t = setTimeout(() => this.kickEmbedWorker(), 0) + ;(t as { unref?: () => void }).unref?.() + } + } catch (err) { + prodLog.warn( + `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + + `the log's markers remain durable; recovery retries next open` ) - const t = setTimeout(() => this.kickEmbedWorker(), 0) - ;(t as { unref?: () => void }).unref?.() } - } catch (err) { - prodLog.warn( - `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + - `the log's markers remain durable; recovery retries next open` - ) - } + })() } // PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob @@ -2407,6 +2408,19 @@ export class Brainy implements BrainyInterface { */ private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/' + /** + * Storage-root-relative path of the ADVISORY pending-embed low-water mark: + * `{ generation, writtenAt }`, written whenever the pending set drains to + * empty (and at clean close when empty). Every marker in facts at or below + * `generation` is consumed, so recovery scans from `generation + 1`. The + * mark is advisory and monotone-safe: stale-low costs a longer scan, never + * a lost marker; it is never required for correctness. + */ + private static readonly PENDING_EMBED_LOWWATER_PATH = '_system/pending_embeds_lowwater.json' + + /** Resolves when the background pending-embed recovery fold has settled (open arms it). */ + private _pendingEmbedRecovery: Promise | null = null + /** * @description Mark a deferred embed pending (MT5): the id joins the * in-memory fast-path set and the returned `embed.pending` record is @@ -2434,6 +2448,40 @@ export class Brainy implements BrainyInterface { */ private clearPendingEmbed(id: string): void { this._pendingEmbedIds.delete(id) + if (this._pendingEmbedIds.size === 0) this.maybeWriteEmbedLowWater() + } + + /** + * @description Advance the advisory low-water mark: called at drain-to-empty + * (and at clean close when empty), it records the fact log's CURRENT head — + * with the set empty, every marker at or below the head has been consumed, + * so the next open's recovery fold scans only what comes after. Fire-and- + * forget at the drain (close() awaits the core); loud on failure: a missed + * write costs the next open a longer scan, never a marker. No-op without a + * fact log (no durable markers exist there) and on read-only opens. + */ + private maybeWriteEmbedLowWater(): void { + void this.writeEmbedLowWater() + } + + /** The awaitable core of {@link maybeWriteEmbedLowWater} — close() awaits it. */ + private async writeEmbedLowWater(): Promise { + if (this.isReadOnly) return + const log = this.generationStore ? this.generationStore.getFactLog() : null + if (!log) return + const generation = log.headGeneration() + if (!(generation > 0)) return + try { + await this.storage.writeRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH, { + generation, + writtenAt: Date.now() + }) + } catch (err) { + prodLog.warn( + `[Brainy] pending-embed low-water write failed at generation ${generation}: ` + + `${(err as Error).message} — the next open scans from the previous mark` + ) + } } /** @@ -2444,9 +2492,14 @@ export class Brainy implements BrainyInterface { * survives the fold is exactly the set of acknowledged deferred writes * whose vectors have not landed. * - * BOUND (honest): no durable low-water mark exists for the earliest - * unconsumed pending, so the fold scans the log's committed facts from - * generation 1 — a sequential read of the log at open, O(log bytes). + * BOUND: the scan starts at the advisory low-water mark + * ({@link Brainy.PENDING_EMBED_LOWWATER_PATH}) — the log head at which the + * pending set last drained to empty — so a settled brain reads only the + * facts since then, not its whole history. Without a mark (first open + * after upgrade) it scans from generation 1, once; a stale-low mark costs + * a longer scan, never a marker. The fold runs BEHIND the doors (open + * arms it as a background task and the embed worker starts when it + * settles); {@link awaitPendingEmbeds} and close() wait for it first. * It is SKIPPED WHOLESALE when the log has never had a v2 tail * ({@link FactLog.hasV2History} — v1 facts cannot carry marker records), * so pre-cutover brains pay nothing; on a mixed log the scan still reads @@ -2459,7 +2512,18 @@ export class Brainy implements BrainyInterface { private async recoverPendingEmbedsFromLog(): Promise { const log = this.generationStore.getFactLog() if (!log || !log.hasV2History()) return - const scan = log.scanFacts({ fromGeneration: 1 }) + let fromGeneration = 1 + try { + const mark = (await this.storage.readRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH)) as { + generation?: number + } | null + if (mark && typeof mark.generation === 'number' && mark.generation > 0) { + fromGeneration = mark.generation + 1 + } + } catch { + // No mark (or unreadable): scan from 1 — correctness over cost. + } + const scan = log.scanFacts({ fromGeneration }) for await (const batch of scan.batches()) { for (const fact of batch.facts) { for (const record of fact.records ?? []) { @@ -2646,6 +2710,7 @@ export class Brainy implements BrainyInterface { * before I proceed" callers use this; nothing else ever needs to wait. */ public async awaitPendingEmbeds(): Promise { + if (this._pendingEmbedRecovery) await this._pendingEmbedRecovery while (this._pendingEmbedIds.size > 0 || this._embedWorkerFlight) { this.kickEmbedWorker() await (this._embedWorkerFlight ?? Promise.resolve()) @@ -4203,32 +4268,19 @@ export class Brainy implements BrainyInterface { */ /** * @description A JSON-safe view of a record bound for the metadata-index - * crossing. The seam's metadata is JSON-safe BY CONTRACT (a native provider - * serializes it; u64 ints as Number corrupt above 2^53) — but - * {@link resolveVerbEndpointInts} MIRRORS the resolved endpoint ints onto - * the verb object itself as BigInt (`verb.sourceInt`/`targetInt`), so a - * verb object reused as index metadata carried BigInts into - * JSON.stringify, which throws, aborting the whole transaction (found by - * the first joint pair gate). Endpoint ints ride their OWN op params on the - * graph legs — the metadata crossing drops every BigInt-valued top-level - * key instead of guessing at a lossy numeric encoding. + * crossing — delegates to the shared {@link jsonSafeIndexMetadata} leaf, + * which the metadata-index transaction operations ALSO apply at execute + * and rollback time. This plan-time wrap alone proved insufficient: it + * returns the same reference when the record is clean, and `transact()`'s + * delete legs share that reference with a graph-retraction op whose + * execute-time endpoint resolution mirrors BigInt ints onto it (the full + * aliasing story lives on the leaf module's doc). * @param metadata - The candidate index-metadata record. * @returns The same object when already JSON-safe, else a shallow copy * without the BigInt-valued keys. */ private static jsonSafeIndexMetadata(metadata: unknown): unknown { - if (metadata === null || typeof metadata !== 'object') return metadata - const rec = metadata as Record - let hasBigint = false - for (const k in rec) { - if (typeof rec[k] === 'bigint') { hasBigint = true; break } - } - if (!hasBigint) return metadata - const out: Record = {} - for (const k in rec) { - if (typeof rec[k] !== 'bigint') out[k] = rec[k] - } - return out + return jsonSafeIndexMetadata(metadata) } private metadataIndexRetractionOp( @@ -7292,6 +7344,47 @@ export class Brainy implements BrainyInterface { await this.verifyMetadataLive() } + // PLANNED FIND (optional provider door, `MetadataIndexProvider.planFindPage`). + // + // The stage doors below each serve one stage, so a find that consults + // three of them crosses into the index three times and marshals a result + // set at every crossing — a filter matching a hundred thousand rows + // builds a hundred thousand id strings to return a page of twenty-five. + // An index that can decide the stage order itself answers the page in one + // call and materializes ids only for the page. + // + // The hook sits ABOVE the branch selection because the branches are what + // decide stage order per call site; an index that plans has to be asked + // before that choice is made, not inside one of its arms. + // + // Optional and additive: a provider without the door, and any shape the + // door hands back, take exactly the path they always took. `null` is a + // routing decision the door must make BEFORE doing any work — never a + // partial answer. Every guard above still ran (readiness, the migration + // gate, the where-clause validation, the metadata cold-read guard), and + // the serving law is applied here on the way out: an empty answer is + // re-verified against the index that produced it before it is believed. + const planningIndex = this.metadataIndex as unknown as MetadataIndexProvider + if (typeof planningIndex.planFindPage === 'function') { + const planned = await planningIndex.planFindPage(params, [...hiddenIds], this.graphIndex) + if (planned !== null && planned !== undefined) { + if (planned.ids.length === 0) { + // A cold adjacency can report a size yet hold no edges, so an empty + // graph answer is not truth until the adjacency verifies live. A + // genuinely edgeless anchor verifies and the empty result stands. + if (planned.emptyAt === 'graph') await this.verifyGraphAdjacencyLive() + return [] + } + const plannedEntities = await this.batchGet(planned.ids) + const plannedResults: Result[] = [] + for (const id of planned.ids) { + const entity = plannedEntities.get(id) + if (entity) plannedResults.push(this.createResult(id, 1.0, entity)) + } + return plannedResults + } + } + // Handle metadata-only queries (no vector search needed) if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) { // Build filter for metadata index @@ -7482,7 +7575,37 @@ export class Brainy implements BrainyInterface { // JS path — there the materialized `candidateIds` restricts the walk instead. let preResolvedAllowedIds: OpaqueIdSet | undefined - if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { + // Graph-first law (10.4.8, BRAINY-PROD-LATENCY-TRIAD rounds 44/45): with + // `connected` present the NEIGHBOUR SET is the candidate universe. It is + // resolved first from the adjacency (O(neighbours)), the metadata filter + // is evaluated over those ids only, and paging happens LAST. The earlier + // order materialized the whole-store filtered id list, paged it, hydrated + // the page, and only then intersected with the neighbours — O(store) per + // call, and a neighbour outside the first page was silently dropped. + let graphFirstIds: string[] | null = null + if (hasGraphCriteria) { + graphFirstIds = await this.resolveConnectedIds(params) + if (hiddenIds.size > 0) { + graphFirstIds = graphFirstIds.filter((id) => !hiddenIds.has(id)) + } + if ( + graphFirstIds.length > 0 && + (params.where || params.type || params.subtype || params.service || params.excludeVFS) + ) { + preResolvedFilter = this.buildMetadataFilter(params) + graphFirstIds = await this.filterIdsWithinBelted(preResolvedFilter, graphFirstIds) + } + if (graphFirstIds.length === 0) { + return [] + } + if (!hasVectorSearchCriteria) { + return await this.pageConnectedIds(params, graphFirstIds) + } + // The vector leg walks ONLY the neighbours (its candidate walk). The + // filter is already applied above, so no opaque universe is produced — + // it would describe the whole store, not the neighbour set. + preResolvedMetadataIds = graphFirstIds + } else if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { preResolvedFilter = this.buildMetadataFilter(params) preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter) @@ -7671,9 +7794,11 @@ export class Brainy implements BrainyInterface { } } - // Graph search component with O(1) traversal - if (params.connected) { - results = await this.executeGraphSearch(params, results) + // The text leg of a hybrid find has no candidate door, so its hits are + // held to the neighbour set here; the vector leg walked only the neighbours. + if (graphFirstIds !== null && results.length > 0) { + const neighbourSet = new Set(graphFirstIds) + results = results.filter((r) => neighbourSet.has(r.id)) } // Apply fusion scoring if requested @@ -12788,6 +12913,29 @@ export class Brainy implements BrainyInterface { } } + /** + * The id-scoped twin of {@link filterIdsBelted}: evaluate `filter` over `ids` + * only, through the provider's own evaluation so the answer can never drift + * from `getIdsForFilter`'s. A provider without the door is served by its + * whole-store answer intersected here (the reference index implements the + * door itself). Same belt: field refusals cross as `BrainyFieldRefusal`. + */ + private async filterIdsWithinBelted(filter: unknown, ids: readonly string[]): Promise { + this.ensureIndexesLoaded(['metadata']) + const mip = this.metadataIndex as unknown as MetadataIndexProvider + try { + if (typeof mip.filterIdsWithin === 'function') { + return await mip.filterIdsWithin(filter, ids) + } + const matched = new Set(await this.metadataIndex.getIdsForFilter(filter)) + return ids.filter((id) => matched.has(id)) + } catch (err) { + const normalized = asBrainyFieldRefusal(err) + if (normalized) throw normalized + throw err + } + } + async getIndexStatus(): Promise<{ initialized: boolean /** `true` once open()'s index-build-if-needed step has run. Named for API @@ -15771,16 +15919,16 @@ export class Brainy implements BrainyInterface { } /** - * Execute graph search component. + * Resolve `params.connected` to the neighbour id set — the graph-first + * find's candidate universe (deterministic traversal order, anchors excluded). * * Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via - * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read - * only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via` - * were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the - * immediate neighbour at every depth. + * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. An empty set + * is re-verified against the adjacency before it is believed — a not-serving + * adjacency throws rather than answering `[]` as truth. */ - private async executeGraphSearch(params: FindParams, existingResults: Result[]): Promise[]> { - if (!params.connected) return existingResults + private async resolveConnectedIds(params: FindParams): Promise { + if (!params.connected) return [] const { from, to, depth, direction = 'both' } = params.connected const via = params.connected.via ?? params.connected.type @@ -15834,8 +15982,8 @@ export class Brainy implements BrainyInterface { if (anchorInt === undefined) return new Set() // unmapped → no relations const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType) - // No limit: match the JS BFS exactly — overall result limiting happens - // downstream against existingResults. + // No limit: match the JS BFS exactly — the page is cut downstream, + // after the metadata filter, by pageConnectedIds / the candidate walk. const reachedInts = await provider.findConnectedSubtype( anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null ) @@ -15920,22 +16068,44 @@ export class Brainy implements BrainyInterface { await this.verifyGraphAdjacencyLive() } - // Filter existing results to only connected entities - if (existingResults.length > 0) { - return existingResults.filter(r => connectedIds.has(r.id)) - } + return [...connectedIds] + } - // Batch-load connected entities for fast cloud-storage performance + /** + * Page and hydrate an already-filtered neighbour set — the pure graph (and + * graph + metadata) find's tail. `orderBy` sorts the WHOLE set by field value + * before the page is cut (never the page after), null values last on `asc` + * and first on `desc`; without `orderBy` the traversal order stands. + */ + private async pageConnectedIds(params: FindParams, ids: string[]): Promise[]> { + const limit = params.limit || 10 + const offset = params.offset || 0 + let ordered = ids + if (params.orderBy) { + const field = params.orderBy + const asc = (params.order || 'asc') === 'asc' + const valued = await Promise.all( + ids.map(async (id) => ({ id, value: await this.metadataIndex.getFieldValueForEntity(id, field) })) + ) + valued.sort((a, b) => { + if (a.value == null && b.value == null) return 0 + if (a.value == null) return asc ? 1 : -1 + if (b.value == null) return asc ? -1 : 1 + if (a.value === b.value) return 0 + const comparison = a.value < b.value ? -1 : 1 + return asc ? comparison : -comparison + }) + ordered = valued.map((v) => v.id) + } + const pageIds = ordered.slice(offset, offset + limit) + const entitiesMap = await this.batchGet(pageIds) const results: Result[] = [] - const ids = [...connectedIds] - const entitiesMap = await this.batchGet(ids) - for (const id of ids) { + for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } - return results } @@ -19455,6 +19625,19 @@ export class Brainy implements BrainyInterface { * terminal releases have run. */ async close(): Promise { + if (this._pendingEmbedRecovery) { + // Settle the background marker fold before the durable steps — its scan + // is bounded by the low-water mark (a full scan happens at most once, + // on the first open after upgrade). + const settleStart = Date.now() + await this._pendingEmbedRecovery + const settleMs = Date.now() - settleStart + if (settleMs >= 1000) { + prodLog.info(`[Brainy] close: pending-embed recovery settled in ${settleMs}ms`) + } + this._pendingEmbedRecovery = null + } + if (this._pendingEmbedIds.size === 0) await this.writeEmbedLowWater() let closeFailure: unknown = null try { await this.closeDurableSteps() diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index 5b10116c..f4cdd632 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-29T10:04:19-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-06-29T10:04:19-07:00", + generatedAt: "2026-08-27T09:18:45-07:00", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/plugin.ts b/src/plugin.ts index b1aef8e0..fdad42f0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -411,6 +411,65 @@ export interface MetadataIndexProvider { * @returns The matching id universe as an opaque set. */ getIdSetForFilter?(filter: any): Promise + /** + * @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the + * survivors in the caller's order — the door a graph-first + * `find({ connected, where })` walks. The neighbour set is the universe there, + * so the filter must cost O(|ids|) membership checks, never a whole-store + * materialization. A native index answers from its roaring filter result + * (membership by entity int); the reference index answers from its own + * `getIdsForFilter`, so the two doors can never disagree. Absent → Brainy + * intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)). + * @param filter - The same filter shape accepted by `getIdsForFilter`. + * @param ids - The candidate ids (canonical). The answer is a subsequence. + */ + filterIdsWithin?(filter: any, ids: readonly string[]): Promise + /** + * @description OPTIONAL: plan and execute a WHOLE `find()` — the graph + * traversal, the metadata filter, the ordering and the page — and answer the + * page's ids, or `null` for a shape this index does not plan. + * + * The doors above each serve one stage, so a `find()` that consults three of + * them crosses into the index three times and marshals a result set at every + * crossing. An index that can decide the stage ORDER itself does the whole + * thing in one call and materializes ids only for the page — a filter + * matching a hundred thousand rows then builds twenty-five id strings instead + * of a hundred thousand. + * + * The contract this door must keep, because Brainy cannot check it: + * + * - **The same answer.** Identical rows, in identical order, to what the + * stage doors would have produced for the same params. This door changes + * which code runs, never what the answer is. + * - **The law of the stages** (`find({ connected })` is graph-first): the + * neighbour set is the candidate universe, the filter is evaluated over + * those ids only, `orderBy` sorts the whole candidate set, and the page is + * cut LAST. + * - **`null` before work, not instead of an answer.** A shape the index does + * not plan must be handed back BEFORE any evaluation, so Brainy serves it + * through the stage doors exactly as it always has. Returning `null` after + * partial work, or an empty page for a shape it could not evaluate, is a + * silent wrong answer. + * - **`emptyAt` names the stage** that produced an empty page — `'graph'`, + * `'filter'`, `'visibility'` or `'none'` — so Brainy can apply its serving + * law to the right index. An empty answer from an index that is not + * serving must refuse loudly, and Brainy can only re-verify what it is told. + * + * Absent → every `find()` is served by the stage doors, which is Brainy's + * own behaviour and the ordering oracle for any implementation of this one. + * @param params - The find params, already normalized by `find()` + * (natural-language parsed, `connected` anchors resolved to canonical ids, + * an empty `where` dropped). + * @param hiddenIds - Ids this read must not return; apply BEFORE paging so + * `limit` stays exact. + * @param graphIndex - The active graph provider, for a `connected` plan. + * @returns The page's ids plus the stage that emptied it, or `null`. + */ + planFindPage?( + params: any, + hiddenIds: readonly string[], + graphIndex: unknown + ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null> getIdsForTextQuery(query: string): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index cabe2e30..a90adb93 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1089,6 +1089,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter { // Counts changed since the last persist? Drives the write-through flush. protected pendingCountPersist = false + /** The one persist running right now, if any (single-flight law — see flushCounts). */ + private countPersistInFlight: Promise | null = null + /** The one trailing persist a burst has queued behind the in-flight one. */ + private countPersistTrailing: Promise | null = null /** * Get total noun count - O(1) operation @@ -1341,15 +1345,46 @@ export abstract class BaseStorageAdapter implements StorageAdapter { return } - try { - // Persist to storage (implemented by subclass) - await this.persistCounts() - this.pendingCountPersist = false - } catch (error) { - console.error('CRITICAL: Failed to flush counts to storage:', error) - // Keep pending flag set so we retry on next operation - throw error + // SINGLE-FLIGHT, COALESCED. Counts are write-through on every change, so + // a burst of writes used to launch one persist per change, all in flight + // together. Two of them inside the same millisecond shared the atomic + // writer's temp path (`.tmp--`): both wrote it, the first rename + // consumed it, the second rename found nothing — ENOENT, ~1,500 times a + // day on a busy production brain, with a full ledger write per change + // behind it. Now exactly one persist runs at a time; requests that arrive + // while it runs collapse into ONE trailing persist that carries the final + // state. A burst of N changes costs at most two writes and never races + // itself. + if (this.countPersistInFlight) { + // The in-flight write may have already serialised a stale snapshot — + // ask for one more pass after it, and let every caller in this burst + // await that same pass. + if (!this.countPersistTrailing) { + this.countPersistTrailing = this.countPersistInFlight + .catch(() => undefined) + .then(() => { + this.countPersistTrailing = null + return this.flushCounts() + }) + } + return this.countPersistTrailing } + + this.countPersistInFlight = (async () => { + try { + // Persist to storage (implemented by subclass) + this.pendingCountPersist = false + await this.persistCounts() + } catch (error) { + // Keep the flag set so the next operation retries. + this.pendingCountPersist = true + console.error('CRITICAL: Failed to flush counts to storage:', error) + throw error + } finally { + this.countPersistInFlight = null + } + })() + return this.countPersistInFlight } /** diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 5ec1d88e..87b6406f 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2400,8 +2400,15 @@ export class FileSystemStorage extends BaseStorage { * Atomic write via temp-file-then-rename so concurrent readers never see a * half-written lock JSON. Reused by writer-lock writes + heartbeat. */ + /** Monotonic per-process sequence so two atomic writes never share a temp path. */ + private static atomicWriteSeq = 0 + private async writeFileAtomic(filePath: string, contents: string): Promise { - const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}` + // pid + timestamp alone collided: two writers of the same target inside + // one millisecond shared this path, and the loser's rename found the + // winner had already moved it (ENOENT). The sequence makes every call's + // temp path its own. + const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}-${++FileSystemStorage.atomicWriteSeq}` await fs.promises.writeFile(tmp, contents) await fs.promises.rename(tmp, filePath) } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d8bcb780..a1cc2e35 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -2942,19 +2942,33 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const sourceId = Array.isArray(options.filter.sourceId) - ? options.filter.sourceId[0] - : options.filter.sourceId + const sourceIds = Array.isArray(options.filter.sourceId) + ? options.filter.sourceId + : [options.filter.sourceId] - const verbType = Array.isArray(options.filter.verbType) - ? options.filter.verbType[0] - : options.filter.verbType + // EVERY requested verb type is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const verbTypes = new Set( + Array.isArray(options.filter.verbType) + ? options.filter.verbType + : [options.filter.verbType] + ) - // Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter), - // then apply the subtype / visibility metadata filters on the candidate set. - const verbsBySource = await this.getVerbsBySource_internal(sourceId) + // Get verbs by source (union over every requested source), filter by the + // requested type SET (O(1) graph lookup + O(n) type filter), then apply + // the subtype / visibility metadata filters on the candidate set. + const bySource: HNSWVerbWithMetadata[] = [] + const seenVerbIds = new Set() + for (const oneSource of sourceIds) { + for (const v of await this.getVerbsBySource_internal(oneSource)) { + if (!seenVerbIds.has(v.id)) { + seenVerbIds.add(v.id) + bySource.push(v) + } + } + } const filteredVerbs = this.applyVerbMetadataFilters( - verbsBySource.filter(v => v.verb === verbType), + bySource.filter(v => verbTypes.has(v.verb)), options.filter ) @@ -2985,16 +2999,22 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const sourceId = Array.isArray(options.filter.sourceId) - ? options.filter.sourceId[0] - : options.filter.sourceId - - // Get verbs by source directly (hydrated with metadata), then apply the - // subtype / visibility metadata filters on the O(degree) candidate set. - const verbsBySource = this.applyVerbMetadataFilters( - await this.getVerbsBySource_internal(sourceId), - options.filter - ) + // EVERY requested source is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const onlySourceIds = Array.isArray(options.filter.sourceId) + ? options.filter.sourceId + : [options.filter.sourceId] + const sourceUnion: HNSWVerbWithMetadata[] = [] + const seenSourceVerbIds = new Set() + for (const oneSource of onlySourceIds) { + for (const v of await this.getVerbsBySource_internal(oneSource)) { + if (!seenSourceVerbIds.has(v.id)) { + seenSourceVerbIds.add(v.id) + sourceUnion.push(v) + } + } + } + const verbsBySource = this.applyVerbMetadataFilters(sourceUnion, options.filter) // Apply pagination const paginatedVerbs = verbsBySource.slice(offset, offset + limit) @@ -3023,16 +3043,22 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const targetId = Array.isArray(options.filter.targetId) - ? options.filter.targetId[0] - : options.filter.targetId - - // Get verbs by target directly (hydrated with metadata), then apply the - // subtype / visibility metadata filters on the O(degree) candidate set. - const verbsByTarget = this.applyVerbMetadataFilters( - await this.getVerbsByTarget_internal(targetId), - options.filter - ) + // EVERY requested target is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const onlyTargetIds = Array.isArray(options.filter.targetId) + ? options.filter.targetId + : [options.filter.targetId] + const targetUnion: HNSWVerbWithMetadata[] = [] + const seenTargetVerbIds = new Set() + for (const oneTarget of onlyTargetIds) { + for (const v of await this.getVerbsByTarget_internal(oneTarget)) { + if (!seenTargetVerbIds.has(v.id)) { + seenTargetVerbIds.add(v.id) + targetUnion.push(v) + } + } + } + const verbsByTarget = this.applyVerbMetadataFilters(targetUnion, options.filter) // Apply pagination const paginatedVerbs = verbsByTarget.slice(offset, offset + limit) @@ -3061,16 +3087,25 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const verbType = Array.isArray(options.filter.verbType) - ? options.filter.verbType[0] - : options.filter.verbType + // EVERY requested verb type is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const verbTypes = Array.isArray(options.filter.verbType) + ? options.filter.verbType + : [options.filter.verbType] - // Get verbs by type directly (hydrated with metadata), then apply the - // subtype / visibility metadata filters on the candidate set. - const verbsByType = this.applyVerbMetadataFilters( - await this.getVerbsByType_internal(verbType), - options.filter - ) + // Get verbs by each requested type (hydrated with metadata), deduped by + // id, then apply the subtype / visibility metadata filters on the set. + const byType: HNSWVerbWithMetadata[] = [] + const seenTypeVerbIds = new Set() + for (const oneType of verbTypes) { + for (const v of await this.getVerbsByType_internal(oneType)) { + if (!seenTypeVerbIds.has(v.id)) { + seenTypeVerbIds.add(v.id) + byType.push(v) + } + } + } + const verbsByType = this.applyVerbMetadataFilters(byType, options.filter) // Apply pagination const paginatedVerbs = verbsByType.slice(offset, offset + limit) diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 1bbbca88..0142dc54 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -14,6 +14,7 @@ import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' import { isZeroNormVector } from '../../utils/distance.js' +import { jsonSafeIndexMetadata } from '../../utils/jsonSafeIndexMetadata.js' import { prodLog } from '../../utils/logger.js' /** @@ -390,13 +391,21 @@ export class AddToMetadataIndexOperation implements Operation { // rollback so add + undo reference the same watermark. const generation = this.generationFn?.() - // Add to metadata index (skipFlush=true for transaction atomicity) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + // The JSON-safe view is taken HERE, per crossing, never at construction: + // the entity reference this op holds can be mutated between plan and + // execute (a graph op's execute-time endpoint-int resolution mirrors + // BigInts onto a shared verb object) — see jsonSafeIndexMetadata's + // module doc. + await this.index.addToIndex( + this.id, jsonSafeIndexMetadata(this.entity), true, false, generation + ) // Return rollback action return async () => { // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + await this.index.removeFromIndex( + this.id, jsonSafeIndexMetadata(this.entity), generation + ) } } } @@ -432,13 +441,21 @@ export class RemoveFromMetadataIndexOperation implements Operation { // Resolve the removal generation once; reuse it for the rollback re-add. const generation = this.generationFn?.() - // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + // Sanitized per crossing, never at construction — transact()'s delete + // legs hand this op the SAME verb object the graph-retraction op's + // execute-time endpoint resolution mutates (BigInt sourceInt/targetInt), + // so a plan-time view aliases the pollution. See jsonSafeIndexMetadata's + // module doc. + await this.index.removeFromIndex( + this.id, jsonSafeIndexMetadata(this.entity), generation + ) // Return rollback action return async () => { // Re-add with original metadata (skipFlush=true) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + await this.index.addToIndex( + this.id, jsonSafeIndexMetadata(this.entity), true, false, generation + ) } } } diff --git a/src/utils/jsonSafeIndexMetadata.ts b/src/utils/jsonSafeIndexMetadata.ts new file mode 100644 index 00000000..d3b1be5f --- /dev/null +++ b/src/utils/jsonSafeIndexMetadata.ts @@ -0,0 +1,47 @@ +/** + * @module utils/jsonSafeIndexMetadata + * @description The metadata-index crossing's JSON-safety law, as a leaf + * function both the coordinator and the transaction operations share. + * + * The seam's metadata is JSON-safe BY CONTRACT (a native provider serializes + * it; u64 ints as Number corrupt above 2^53) — but `resolveVerbEndpointInts` + * MIRRORS the resolved endpoint ints onto the verb object itself as BigInt + * (`verb.sourceInt`/`targetInt`), so a verb object reused as index metadata + * carries BigInts into JSON.stringify, which throws, aborting the whole + * transaction. Endpoint ints ride their OWN op params on the graph legs — the + * metadata crossing drops every BigInt-valued top-level key instead of + * guessing at a lossy numeric encoding. + * + * WHY THIS IS A LEAF MODULE, ENFORCED AT THE CROSSING: sanitizing only at + * operation-construction time is not enough. `transact()`'s delete legs pass + * the SAME verb object to both the graph-retraction op (whose endpoint-int + * thunk deliberately resolves at EXECUTE time, for same-batch forward refs) + * and the metadata-retraction op. At plan time the verb is still clean, so a + * plan-time sanitize returns the same reference — then the graph op executes + * first, mirrors the BigInt ints onto the shared object, and the metadata op + * crosses the seam with them (found by the first fleet adoption of the native + * pair: every transact-wrapped edge delete aborted). The crossing itself is + * the only place ordering cannot bypass. + */ + +/** + * A JSON-safe view of a record bound for the metadata-index crossing. + * + * @param metadata - The candidate index-metadata record. + * @returns The same object when already JSON-safe, else a shallow copy + * without the BigInt-valued keys. + */ +export function jsonSafeIndexMetadata(metadata: unknown): unknown { + if (metadata === null || typeof metadata !== 'object') return metadata + const rec = metadata as Record + let hasBigint = false + for (const k in rec) { + if (typeof rec[k] === 'bigint') { hasBigint = true; break } + } + if (!hasBigint) return metadata + const out: Record = {} + for (const k in rec) { + if (typeof rec[k] !== 'bigint') out[k] = rec[k] + } + return out +} diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3e0e3d17..0fd312e2 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2575,6 +2575,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { /** Once-per-field flag for the fallback-degradation announcement. */ private static announcedFallbackSorts = new Set() + /** + * Evaluate `filter` over `ids` only — the graph-first find's door (the + * neighbour set filtered by id, never the store filtered and then + * intersected). This index answers from its own `getIdsForFilter`, so the + * two doors cannot disagree; the cost is that of the filter over this + * in-memory index, and the answer keeps the caller's order. + */ + async filterIdsWithin(filter: any, ids: readonly string[]): Promise { + if (ids.length === 0) return [] + const matched = new Set(await this.getIdsForFilter(filter)) + return ids.filter((id) => matched.has(id)) + } + async getSortedIdsForFilter( filter: any, orderBy: string, diff --git a/tests/integration/counts-persist-single-flight.test.ts b/tests/integration/counts-persist-single-flight.test.ts new file mode 100644 index 00000000..5acbdcc3 --- /dev/null +++ b/tests/integration/counts-persist-single-flight.test.ts @@ -0,0 +1,111 @@ +/** + * @module tests/integration/counts-persist-single-flight + * @description Regression for a production race in FileSystemStorage's + * counts ledger: `persistCounts()` was write-through on every count change + * with no serialization, and the atomic writer named its temp file with + * millisecond granularity (`.tmp--`). Two persists inside one + * millisecond shared the temp path — both wrote it, the first rename + * consumed it, the second rename found nothing: ENOENT, ~1,500 times a day + * on a busy production brain, with a full ledger write per change behind it. + * + * Under pin: persists are single-flight and coalesced — one in flight, at + * most one trailing pass carrying the burst's final state — and every atomic + * write owns a unique temp path. A burst of N count changes costs at most + * two ledger writes, never errors, and leaves a ledger equal to memory. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +describe('counts persistence is single-flight, coalesced, and never races its own temp file', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-counts-race-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + vi.restoreAllMocks() + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('a burst of concurrent count changes → at most two ledger writes, zero errors, ledger == memory', async () => { + const storage = brain.storage + const countsPath: string = storage.countsFilePath + expect(countsPath, 'the filesystem adapter persists a counts ledger').toBeTruthy() + + // Let init's own persists settle so the burst is measured alone. + await storage.flushCounts?.() + + const renameSpy = vi.spyOn(fs.promises, 'rename') + const errorSpy = vi.spyOn(console, 'error') + + // Twenty-five concurrent count changes — the shape of a write burst; each + // used to launch its own persist. + const BURST = 25 + await Promise.all( + Array.from({ length: BURST }, () => storage.scheduleCountPersist()) + ) + + const ledgerRenames = renameSpy.mock.calls.filter(([, to]) => String(to) === countsPath) + expect(ledgerRenames.length, 'single-flight + one trailing pass').toBeLessThanOrEqual(2) + expect(ledgerRenames.length, 'the burst was persisted at all').toBeGreaterThanOrEqual(1) + + const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts')) + expect(persistErrors).toEqual([]) + + const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) + expect(ledger.totalNounCount).toBe(storage.totalNounCount) + expect(ledger.totalVerbCount).toBe(storage.totalVerbCount) + }) + + it('real writes in parallel: the ledger lands complete and no persist error is logged', async () => { + const storage = brain.storage + const countsPath: string = storage.countsFilePath + const errorSpy = vi.spyOn(console, 'error') + + await Promise.all( + Array.from({ length: 12 }, (_, i) => + brain.add({ data: `burst row ${i}`, type: NounType.Thing }) + ) + ) + await storage.flushCounts?.() + + const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts')) + expect(persistErrors).toEqual([]) + const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) + expect(ledger.totalNounCount).toBe(storage.totalNounCount) + expect(await brain.getNounCount()).toBe(ledger.totalNounCount) + }) + + it('every atomic write owns its own temp path — two writes in one millisecond never collide', async () => { + const storage = brain.storage + const tmpNames: string[] = [] + vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (p: any) => { + tmpNames.push(String(p)) + }) + vi.spyOn(fs.promises, 'rename').mockImplementation(async () => undefined) + const target = path.join(dir, 'probe.json') + await Promise.all([ + storage.writeFileAtomic(target, '{"a":1}'), + storage.writeFileAtomic(target, '{"a":2}'), + storage.writeFileAtomic(target, '{"a":3}') + ]) + const probeTmps = tmpNames.filter((n) => n.startsWith(`${target}.tmp-`)) + expect(probeTmps.length).toBe(3) + expect(new Set(probeTmps).size, 'no two writes shared a temp path').toBe(3) + }) +}) diff --git a/tests/integration/find-connected-order.test.ts b/tests/integration/find-connected-order.test.ts new file mode 100644 index 00000000..b04e7f99 --- /dev/null +++ b/tests/integration/find-connected-order.test.ts @@ -0,0 +1,165 @@ +/** + * @module tests/integration/find-connected-order + * @description The graph-first law for `find({ connected })` (10.4.8). + * + * With `connected` present the neighbour set is the candidate universe: it is + * resolved from the adjacency first, the metadata filter is evaluated over + * those ids only, and the page is cut last. The earlier order materialized the + * whole-store filtered id list, paged it, hydrated the page, and only then + * intersected with the neighbours — so a neighbour outside the first page of + * the filtered STORE was silently dropped, and every call paid O(store). + * + * These pins hold both halves. The answer: every matching neighbour is + * reachable by paging, a non-neighbour never appears, a negation (`missing`) + * is evaluated over the neighbours, `orderBy` sorts the whole neighbour set + * before the page is cut, and the vector leg walks the neighbours only. The + * cost shape: the metadata index is asked about the neighbour ids only, and + * hydration is one page — never the store. + */ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { v5 } from '../../src/universal/uuid' +import { generateTestVector } from '../helpers/test-factory' + +/** Matching rows that are NOT neighbours — added FIRST, so the whole-store filtered list leads with them. */ +const NOISE = 120 +/** Matching rows that ARE neighbours of the anchor. */ +const NEIGHBOURS = 30 +/** Neighbours carrying `retracted: true` — excluded by the `missing` negation. */ +const RETRACTED = 4 + +describe('find({ connected }) is graph-first: neighbours → filter → page', () => { + let brain: Brainy + const anchor = 'anchor' + const sharedVector = generateTestVector() + const neighbourIds = new Set(Array.from({ length: NEIGHBOURS }, (_, i) => v5(`nb-${i}`))) + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ + id: anchor, + data: 'the anchor', + type: NounType.Person, + metadata: { kind: 'anchor' }, + vector: generateTestVector() + }) + for (let i = 0; i < NOISE; i++) { + await brain.add({ + id: `noise-${i}`, + data: `noise ${i}`, + type: NounType.Person, + metadata: { kind: 'note', rank: 1000 + i }, + vector: sharedVector + }) + } + for (let i = 0; i < NEIGHBOURS; i++) { + await brain.add({ + id: `nb-${i}`, + data: `neighbour ${i}`, + type: NounType.Person, + metadata: { kind: 'note', rank: i + 1, ...(i < RETRACTED ? { retracted: true } : {}) }, + vector: sharedVector + }) + await brain.relate({ from: anchor, to: `nb-${i}`, type: VerbType.Knows }) + } + }) + + afterAll(async () => { + brain = null as any + }) + + it('returns the matching neighbours page by page — none dropped, never a non-neighbour', async () => { + const seen = new Set() + for (let offset = 0; offset <= NEIGHBOURS; offset += 10) { + const page = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 10, + offset + }) + expect(page).toHaveLength(offset < NEIGHBOURS ? 10 : 0) + for (const r of page) { + expect(neighbourIds.has(r.entity.id)).toBe(true) + expect(seen.has(r.entity.id)).toBe(false) + seen.add(r.entity.id) + } + } + expect(seen.size).toBe(NEIGHBOURS) + }) + + it('evaluates a negation (`missing`) over the neighbour set, not the store', async () => { + const results = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note', retracted: { missing: true } }, + limit: 100 + }) + expect(results).toHaveLength(NEIGHBOURS - RETRACTED) + for (const r of results) { + expect(neighbourIds.has(r.entity.id)).toBe(true) + expect(r.entity.metadata.retracted).toBeUndefined() + } + }) + + it('asks the metadata index about the neighbour ids only, and hydrates one page', async () => { + const index = (brain as any).metadataIndex + const within = vi.spyOn(index, 'filterIdsWithin') + const hydrate = vi.spyOn(brain as any, 'batchGet') + try { + const results = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 10 + }) + expect(results).toHaveLength(10) + expect(within).toHaveBeenCalledTimes(1) + const askedIds = within.mock.calls[0][1] as string[] + expect(askedIds).toHaveLength(NEIGHBOURS) + for (const id of askedIds) expect(neighbourIds.has(id)).toBe(true) + expect(hydrate).toHaveBeenCalledTimes(1) + expect(hydrate.mock.calls[0][0]).toHaveLength(10) + } finally { + within.mockRestore() + hydrate.mockRestore() + } + }) + + it('orders the WHOLE neighbour set before cutting the page', async () => { + const results = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + orderBy: 'rank', + order: 'desc', + limit: 5 + }) + expect(results.map((r) => r.entity.metadata.rank)).toEqual([30, 29, 28, 27, 26]) + }) + + it('walks the vector leg over the neighbours only', async () => { + const results = await brain.find({ + vector: sharedVector, + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 5 + }) + expect(results).toHaveLength(5) + for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true) + }) + + it('an anchor without neighbours answers [] before the filter is asked', async () => { + const index = (brain as any).metadataIndex + const within = vi.spyOn(index, 'filterIdsWithin') + try { + const results = await brain.find({ + connected: { from: 'noise-0', direction: 'out' }, + where: { kind: 'note' }, + limit: 10 + }) + expect(results).toEqual([]) + expect(within).not.toHaveBeenCalled() + } finally { + within.mockRestore() + } + }) +}) diff --git a/tests/integration/find-planner-door.test.ts b/tests/integration/find-planner-door.test.ts new file mode 100644 index 00000000..964b13f9 --- /dev/null +++ b/tests/integration/find-planner-door.test.ts @@ -0,0 +1,137 @@ +/** + * @module tests/integration/find-planner-door + * @description The optional `MetadataIndexProvider.planFindPage` door. + * + * The stage doors each serve one stage, so a `find()` that consults three of + * them crosses into the index three times and marshals a result set at every + * crossing — a filter matching a hundred thousand rows builds a hundred + * thousand id strings to return a page of twenty-five. An index that can decide + * the stage order itself answers the page in one call. + * + * These pins hold the three properties that make such a door safe to add: + * + * 1. **Absent, nothing changes.** The reference index has no planner, and every + * find is served by the stage doors exactly as before. That is also what + * makes this engine the ordering oracle for any index that implements one. + * 2. **Present, it is asked first and its answer is used** — above the branch + * selection, with the params already normalized, the hidden ids passed, and + * the graph provider handed over. + * 3. **`null` is routing, not an answer.** A door that declines a shape leaves + * it to the path that always served it, and the result is unchanged. + * + * Plus the serving law: an empty page stamped `emptyAt: 'graph'` is re-verified + * against the adjacency before it is believed, so a not-serving graph refuses + * loudly instead of answering `[]` as truth. + */ +import { describe, it, expect, beforeAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { generateTestVector } from '../helpers/test-factory' + +describe('find(): the optional planner door', () => { + let brain: Brainy + const anchor = 'planner-anchor' + let neighbourId = '' + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ + id: anchor, + data: 'anchor', + type: NounType.Person, + metadata: { kind: 'anchor' }, + vector: generateTestVector() + }) + for (let i = 0; i < 12; i++) { + const id = await brain.add({ + id: `row-${i}`, + data: `row ${i}`, + type: NounType.Person, + metadata: { kind: 'note', rank: i }, + vector: generateTestVector() + }) + if (i === 0) neighbourId = id + await brain.relate({ from: anchor, to: id, type: VerbType.Knows }) + } + }) + + /** Install a planner door for one call, then remove it. */ + const withDoor = async ( + door: (...a: any[]) => Promise, + body: () => Promise + ): Promise => { + const index = (brain as any).metadataIndex + index.planFindPage = door + try { + return await body() + } finally { + delete index.planFindPage + } + } + + it('is absent on the reference index — every find is served by the stage doors', async () => { + expect((brain as any).metadataIndex.planFindPage).toBeUndefined() + const results = await brain.find({ where: { kind: 'note' }, limit: 5 }) + expect(results).toHaveLength(5) + }) + + it('is asked before the branches, with normalized params and the graph provider', async () => { + const door = vi.fn(async () => null) + await withDoor(door, async () => { + await brain.find({ where: { kind: 'note' }, limit: 5 }) + }) + expect(door).toHaveBeenCalledTimes(1) + const [params, hidden, graph] = door.mock.calls[0] as any[] + expect(params.where).toEqual({ kind: 'note' }) + expect(Array.isArray(hidden)).toBe(true) + expect(graph).toBe((brain as any).graphIndex) + }) + + it('uses the page it answers, hydrated and in the door\'s order', async () => { + const results = await withDoor( + async () => ({ ids: [neighbourId], emptyAt: 'none' as const }), + async () => brain.find({ where: { kind: 'note' }, limit: 5 }) + ) + expect(results).toHaveLength(1) + expect(results[0].entity.id).toBe(neighbourId) + }) + + it('a declining door changes nothing — the shape is served as it always was', async () => { + const withoutDoor = await brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 }) + const declined = await withDoor( + async () => null, + async () => brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 }) + ) + expect(declined.map((r) => r.entity.id)).toEqual(withoutDoor.map((r) => r.entity.id)) + }) + + it('re-verifies the adjacency before believing an empty graph answer', async () => { + const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive') + try { + const results = await withDoor( + async () => ({ ids: [], emptyAt: 'graph' as const }), + async () => brain.find({ connected: { from: anchor }, where: { kind: 'note' }, limit: 5 }) + ) + expect(results).toEqual([]) + expect(verify).toHaveBeenCalled() + } finally { + verify.mockRestore() + } + }) + + it('does not re-verify the adjacency for an empty the FILTER produced', async () => { + const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive') + verify.mockClear() + try { + const results = await withDoor( + async () => ({ ids: [], emptyAt: 'filter' as const }), + async () => brain.find({ where: { kind: 'note' }, limit: 5 }) + ) + expect(results).toEqual([]) + expect(verify).not.toHaveBeenCalled() + } finally { + verify.mockRestore() + } + }) +}) diff --git a/tests/integration/pending-embed-low-water.test.ts b/tests/integration/pending-embed-low-water.test.ts new file mode 100644 index 00000000..ff01b349 --- /dev/null +++ b/tests/integration/pending-embed-low-water.test.ts @@ -0,0 +1,145 @@ +/** + * @module tests/integration/pending-embed-low-water + * @description The pending-embed recovery fold is bounded and background (10.4.9). + * + * The fold used to scan the generation log from generation 1 at EVERY open, + * on the open's foreground — O(whole history) per open on long-lived brains. + * Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`) + * records the committed generation whenever the pending set drains to empty, + * recovery scans from `mark + 1`, and the fold runs behind the doors as a + * latched background task the worker, `awaitPendingEmbeds()` and `close()` + * wait on. The mark is advisory: stale-low costs a longer scan, never a + * marker — a pending embed enqueued before a crash is still recovered. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' + +const LOWWATER_PATH = '_system/pending_embeds_lowwater.json' + +describe('pending-embed recovery: bounded by the low-water mark, behind the doors', () => { + const roots: string[] = [] + const dir = (): string => { + const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-')) + roots.push(d) + return d + } + const open = async (root: string): Promise> => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: root } + }) + await brain.init() + return brain + } + + afterEach(() => { + for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true }) + }) + + it('drain-to-empty writes the mark, and the next open scans from mark + 1', async () => { + const root = dir() + const brain = await open(root) + // Hold the worker so the pending state is observable, then release it. + const realKick = (brain as any).kickEmbedWorker.bind(brain) + ;(brain as any).kickEmbedWorker = () => {} + await brain.add({ + id: 'row-1', + data: 'the first deferred row', + type: NounType.Thing, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBeGreaterThan(0) + ;(brain as any).kickEmbedWorker = realKick + await brain.awaitPendingEmbeds() + // The drain wrote the advisory mark (fire-and-forget: settle the microtask). + await new Promise((r) => setTimeout(r, 50)) + const mark = (await (brain as any).storage.readRawObject(LOWWATER_PATH)) as { + generation: number + } | null + expect(mark).not.toBeNull() + expect(mark!.generation).toBeGreaterThan(0) + await brain.close() + + const brain2 = await open(root) + const log = (brain2 as any).generationStore.getFactLog() + const scanSpy = vi.spyOn(log, 'scanFacts') + try { + await (brain2 as any).recoverPendingEmbedsFromLog() + expect(scanSpy).toHaveBeenCalledTimes(1) + const opts = scanSpy.mock.calls[0][0] as { fromGeneration?: number } + expect(opts.fromGeneration).toBeGreaterThanOrEqual(mark!.generation + 1) + } finally { + scanSpy.mockRestore() + await brain2.close() + } + }) + + it('a pending embed enqueued after the mark survives an unclean stop', async () => { + const root = dir() + const brain = await open(root) + await brain.add({ id: 'settled', data: 'lands before the mark', type: NounType.Thing }) + await brain.awaitPendingEmbeds() + await new Promise((r) => setTimeout(r, 50)) + + // A deferred write whose embed never lands: block the worker, then drop + // the instance without close() — the unclean-stop shape. + ;(brain as any).kickEmbedWorker = () => {} + await brain.add({ + id: 'orphan', + data: 'enqueued then abandoned', + type: NounType.Thing, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBeGreaterThan(0) + // No close(): simulate the crash by releasing only the writer lock so the + // next open can proceed. + await (brain as any).storage.releaseWriterLock() + + const brain2 = await open(root) + await (brain2 as any)._pendingEmbedRecovery + expect(brain2.pendingEmbedCount()).toBeGreaterThan(0) + await brain2.awaitPendingEmbeds() + expect(brain2.pendingEmbedCount()).toBe(0) + await brain2.close() + // Reap the crashed instance: its fence is gone, so close() fails loudly — + // swallow that here; the point is clearing its watchers and registry entry. + await brain.close().catch(() => undefined) + }) + + it('open arms the fold as a background latch; awaitPendingEmbeds waits on it', async () => { + const root = dir() + const brain = await open(root) + await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing }) + await brain.awaitPendingEmbeds() + await brain.close() + + const brain2 = await open(root) + // The latch exists the moment init() returns (writable filesystem brain)… + expect((brain2 as any)._pendingEmbedRecovery).not.toBeNull() + // …and the barrier settles it before answering. + await brain2.awaitPendingEmbeds() + expect(brain2.pendingEmbedCount()).toBe(0) + await brain2.close() + }) + + it('a clean close with an empty set writes the mark even if no drain happened', async () => { + const root = dir() + const brain = await open(root) + await brain.add({ id: 'r1', data: 'row one', type: NounType.Thing }) + await brain.awaitPendingEmbeds() + await brain.close() + // Read the mark back through the storage door (the adapter owns the + // on-disk encoding), on a fresh instance. + const brain2 = await open(root) + const mark = (await (brain2 as any).storage.readRawObject(LOWWATER_PATH)) as { + generation: number + } | null + expect(mark).not.toBeNull() + expect(mark!.generation).toBeGreaterThan(0) + await brain2.close() + }) +}) diff --git a/tests/integration/related-verb-array.test.ts b/tests/integration/related-verb-array.test.ts new file mode 100644 index 00000000..36a49850 --- /dev/null +++ b/tests/integration/related-verb-array.test.ts @@ -0,0 +1,89 @@ +/** + * @module tests/integration/related-verb-array + * @description related() honours EVERY verb type in an array (10.4.9). + * + * The storage fast paths for `sourceId + verbType` and `verbType` collapsed a + * verb-type ARRAY to its first element — `related({ from, type: [a, b] })` + * silently returned only `a` edges, whichever order the array came in. The + * same quiet-loss class as the graph-first paging defect, one seam over. + * These pins seed a store where the SECOND requested type's edge must come + * back, on every path the collapse lived in. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { v5 } from '../../src/universal/uuid' + +describe('related() with a verb-type array returns every requested type', () => { + let brain: Brainy + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + for (const id of ['a', 'b', 'c', 'd']) { + await brain.add({ id, data: `node ${id}`, type: NounType.Person }) + } + await brain.relate({ from: 'a', to: 'b', type: VerbType.Supports }) + await brain.relate({ from: 'a', to: 'c', type: VerbType.RelatedTo }) + await brain.relate({ from: 'a', to: 'd', type: VerbType.Knows }) + await brain.relate({ from: 'b', to: 'c', type: VerbType.RelatedTo }) + }) + + afterAll(async () => { + brain = null as any + }) + + it('from + type array: the second type\'s edge comes back, both orders', async () => { + for (const types of [ + [VerbType.Supports, VerbType.RelatedTo], + [VerbType.RelatedTo, VerbType.Supports] + ]) { + const edges = await brain.related({ from: 'a', type: types }) + const targets = new Set(edges.map((e) => e.to)) + expect(targets.has(v5('b')), `types [${types}] missing Supports edge`).toBe(true) + expect(targets.has(v5('c')), `types [${types}] missing RelatedTo edge`).toBe(true) + expect(targets.has(v5('d'))).toBe(false) + expect(edges).toHaveLength(2) + } + }) + + it('a single-element array behaves exactly like the scalar', async () => { + const scalar = await brain.related({ from: 'a', type: VerbType.Supports }) + const array = await brain.related({ from: 'a', type: [VerbType.Supports] }) + expect(array.map((e) => e.id).sort()).toEqual(scalar.map((e) => e.id).sort()) + expect(array).toHaveLength(1) + }) + + it('no duplicate edges when types overlap the same edge set', async () => { + const edges = await brain.related({ + from: 'a', + type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows] + }) + const ids = edges.map((e) => e.id) + expect(new Set(ids).size).toBe(ids.length) + expect(edges).toHaveLength(3) + }) + + it('type-only asks (no anchor) honour the whole array too', async () => { + const edges = await brain.related({ type: [VerbType.Supports, VerbType.Knows] }) + const verbs = new Set(edges.map((e) => e.type)) + expect(verbs.has(VerbType.Supports)).toBe(true) + expect(verbs.has(VerbType.Knows)).toBe(true) + expect(edges).toHaveLength(2) + }) + + it('to + type array: the target side honours every type too', async () => { + const edges = await brain.related({ to: 'c', type: [VerbType.RelatedTo, VerbType.Supports] }) + const froms = new Set(edges.map((e) => e.from)) + expect(froms.has(v5('a'))).toBe(true) + expect(froms.has(v5('b'))).toBe(true) + expect(edges).toHaveLength(2) + }) + + it('pagination stays consistent across the union', async () => { + const page1 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2 }) + const page2 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2, offset: 2 }) + const all = [...page1, ...page2].map((e) => e.id) + expect(new Set(all).size).toBe(3) + }) +}) diff --git a/tests/integration/transact-edge-delete-bigint-aliasing.test.ts b/tests/integration/transact-edge-delete-bigint-aliasing.test.ts new file mode 100644 index 00000000..b3902571 --- /dev/null +++ b/tests/integration/transact-edge-delete-bigint-aliasing.test.ts @@ -0,0 +1,184 @@ +/** + * @module tests/integration/transact-edge-delete-bigint-aliasing + * @description Regression for a fleet-adoption blocker: ANY edge delete + * inside `transact()` — a direct unrelate or a noun-remove's cascade — + * aborted with the metadata seam's BigInt JSON-guard error on a strict + * (native) metadata provider. + * + * The aliasing chain: `planTxUnrelate`/the remove-cascade pass the SAME verb + * object to the graph-retraction op and the metadata-retraction op. The + * metadata leg's JSON-safe wrap ran at PLAN time, when the verb was still + * clean — so it returned the same reference. At EXECUTE time the graph op + * runs first and `resolveVerbEndpointInts` mirrors BigInt + * `sourceInt`/`targetInt` onto the shared object (deliberately deferred for + * same-batch forward refs — see transact-forward-ref-graph.test.ts); the + * metadata op then crossed the seam with the polluted object. Direct + * `unrelate()` resolves ints at BUILD time, before its sanitize, which is why + * only the transact() shapes ever hit it. + * + * Fix under pin: the JSON-safe view is taken AT THE CROSSING — inside the + * metadata-index operations' execute/rollback — so no plan-vs-execute + * ordering can bypass it. The JS baseline index tolerates BigInts (it would + * mask the bug), so these pins SPY on the seam and assert what actually + * crossed, exactly as a strict native provider would judge it. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { + AddToMetadataIndexOperation, + RemoveFromMetadataIndexOperation +} from '../../src/transaction/operations/index.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +/** Top-level BigInt-valued keys of a candidate seam crossing (the guard's law). */ +const bigintKeys = (metadata: unknown): string[] => { + if (metadata === null || typeof metadata !== 'object') return [] + return Object.entries(metadata as Record) + .filter(([, v]) => typeof v === 'bigint') + .map(([k]) => k) +} + +describe('transact() edge deletes never carry BigInt across the metadata seam', () => { + let dir: string + let brain: any + let crossings: Array<{ door: string; id: string; keys: string[] }> + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-tx-bigint-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + + // Spy on the seam the way a strict native provider judges it: record the + // BigInt-valued top-level keys of every metadata argument that crosses. + // The JS baseline index tolerates BigInts, so without this the baseline + // run would green a shape the native pair aborts on. + crossings = [] + const index = brain.metadataIndex + for (const door of ['addToIndex', 'removeFromIndex'] as const) { + const real = index[door].bind(index) + index[door] = (id: string, metadata: unknown, ...rest: unknown[]) => { + crossings.push({ door, id, keys: bigintKeys(metadata) }) + return real(id, metadata, ...rest) + } + } + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('CASE 1 (the fleet repro): relate, then transact([{op: unrelate}])', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const verbId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + crossings.length = 0 + await brain.transact([{ op: 'unrelate', id: verbId }]) + + const polluted = crossings.filter((c) => c.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.storage.getVerb(verbId)).toBeFalsy() + }) + + it('CASE 2 (the cascade shape): transact([{op: remove}]) cascading edge deletes', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const c = await brain.add({ id: freshId(), data: 'c', type: NounType.Thing }) + const ab = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + const ca = await brain.relate({ from: c, to: a, type: VerbType.RelatedTo }) + + crossings.length = 0 + await brain.transact([{ op: 'remove', id: a }]) + + const polluted = crossings.filter((c2) => c2.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.get(a)).toBeFalsy() + expect(await brain.storage.getVerb(ab)).toBeFalsy() + expect(await brain.storage.getVerb(ca)).toBeFalsy() + }) + + it('CASE 3 (one batch, both legs): adds + relate + unrelate of a pre-existing edge', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const old = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + const x = freshId() + crossings.length = 0 + await brain.transact([ + { op: 'add', id: x, data: 'x', type: NounType.Thing }, + { op: 'relate', from: a, to: x, type: VerbType.RelatedTo }, + { op: 'unrelate', id: old } + ]) + + const polluted = crossings.filter((c) => c.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.storage.getVerb(old)).toBeFalsy() + const edges = await brain.related({ from: a }) + expect(edges.length).toBe(1) + expect(edges[0].id).not.toBe(old) + }) +}) + +describe('the metadata-index operations sanitize at the crossing, not at construction', () => { + /** A strict seam: refuses BigInts exactly as the native provider does. */ + const strictIndex = () => { + const seen: Array<{ door: string; keys: string[] }> = [] + const judge = (door: string, metadata: unknown) => { + const keys = bigintKeys(metadata) + seen.push({ door, keys }) + if (keys.length > 0) { + throw new Error( + `${door}: the metadata object violates the provider seam's JSON ` + + `contract — BigInt at ${keys.join(', ')}.` + ) + } + } + return { + seen, + addToIndex: async (_id: string, metadata: unknown) => judge('addToIndex', metadata), + removeFromIndex: async (_id: string, metadata: unknown) => judge('removeFromIndex', metadata) + } + } + + it('RemoveFromMetadataIndexOperation: entity mutated AFTER construction still crosses clean', async () => { + const index = strictIndex() + const verb: Record = { id: 'v1', sourceId: 'a', targetId: 'b' } + const op = new RemoveFromMetadataIndexOperation(index as any, 'v1', verb, () => 7n) + + // The graph leg's execute-time endpoint resolution, simulated: the shared + // object is polluted between plan and execute. + verb.sourceInt = 800_000n + verb.targetInt = 800_001n + + const rollback = await op.execute() + await rollback() + expect(index.seen.map((s) => s.keys)).toEqual([[], []]) + }) + + it('AddToMetadataIndexOperation: same law on the add leg and its rollback', async () => { + const index = strictIndex() + const verb: Record = { id: 'v2', sourceId: 'a', targetId: 'b' } + const op = new AddToMetadataIndexOperation(index as any, 'v2', verb, () => 7n) + + verb.sourceInt = 800_000n + verb.targetInt = 800_001n + + const rollback = await op.execute() + await rollback() + expect(index.seen.map((s) => s.keys)).toEqual([[], []]) + }) +}) diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts deleted file mode 100644 index 8bf9d357..00000000 --- a/tests/unit/release/wall-entry.test.ts +++ /dev/null @@ -1,395 +0,0 @@ -/** - * scripts/wall-entry.mjs — the mechanical releases-wall entry. - * - * The script's only real interface is its CLI (it has no importable - * exports by design — one door, no parallel API to drift from it), so - * these tests spawn it exactly as scripts/release.sh does: as a child - * process, against a fixture CHANGELOG and a throwaway local bare repo - * standing in for git@source.soulcraft.com:soulcraftlabs/releases.git - * (--remote) plus a throwaway cache directory (--cache-dir) standing in - * for ~/.cache/soulcraft-releases — never the real remote, never the - * real developer cache. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { execFileSync } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' - -const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs') - -/** Run the script and capture the outcome without throwing on a non-zero exit. */ -function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } { - try { - const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' }) - return { status: 0, stdout, stderr: '' } - } catch (err: any) { - return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } - } -} - -function git(args: string[], cwd: string): string { - return execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim() -} - -const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' - -/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ -function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string { - const body = entries - .map( - (e) => - `### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` + - e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') + - '\n', - ) - .join('\n') - return CHANGELOG_HEADER + '\n' + body -} - -function wallFile(product: string, entries: unknown[]): string { - return JSON.stringify({ product, entries }, null, 2) + '\n' -} - -const BASE_ENTRY = { - version: '10.4.11', - date: '2026-09-02', - headline: 'A faster open', - items: ['A faster open.'], - url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11', - thumb: null, -} - -/** A throwaway bare repo standing in for the real soulcraftlabs/releases remote. */ -function initBareRemote(): string { - const remoteDir = mkdtempSync(join(tmpdir(), 'wall-remote-')) - execFileSync('git', ['init', '--bare', '-b', 'main', remoteDir]) - return remoteDir -} - -/** Seed the bare remote with an initial .json, via a throwaway clone. */ -function seedRemote(remoteDir: string, product: string, entries: unknown[]): void { - const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-')) - execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) - git(['config', 'user.email', 'seed@example.com'], seedDir) - git(['config', 'user.name', 'Seed'], seedDir) - writeFileSync(join(seedDir, `${product}.json`), wallFile(product, entries)) - git(['add', `${product}.json`], seedDir) - git(['commit', '-m', 'seed'], seedDir) - git(['push', 'origin', 'main'], seedDir) - rmSync(seedDir, { recursive: true, force: true }) -} - -/** Read .json back out of the bare remote's main tip, via a throwaway clone. */ -function readRemote(remoteDir: string, product: string): any { - const readDir = mkdtempSync(join(tmpdir(), 'wall-read-')) - execFileSync('git', ['clone', remoteDir, readDir], { stdio: 'ignore' }) - const data = JSON.parse(readFileSync(join(readDir, `${product}.json`), 'utf8')) - rmSync(readDir, { recursive: true, force: true }) - return data -} - -/** Reject every push — stands in for any push failure (including a genuine - * non-fast-forward raced by a concurrent release rail), which this script - * treats identically: refuse loudly, name the cure, touch nothing further. */ -function makeRemoteRejectPushes(remoteDir: string): void { - const hookPath = join(remoteDir, 'hooks', 'pre-receive') - writeFileSync(hookPath, '#!/bin/sh\necho "remote: simulated push rejection" >&2\nexit 1\n') - chmodSync(hookPath, 0o755) -} - -let dir: string -let remoteDir: string -let cacheDir: string - -beforeEach(() => { - dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) - remoteDir = initBareRemote() - cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases') -}) - -afterEach(() => { - rmSync(dir, { recursive: true, force: true }) - rmSync(remoteDir, { recursive: true, force: true }) - rmSync(cacheDir, { recursive: true, force: true }) -}) - -describe('wall-entry.mjs — generate + publish', () => { - it('derives headline from the first bullet and items from every bullet, hashes stripped, and pushes it to the remote', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - writeFileSync( - join(dir, 'CHANGELOG.md'), - buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), - ) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/wrote v10\.4\.12.*pushed/i) - - const wall = readRemote(remoteDir, 'open-brainy') - expect(wall.entries).toHaveLength(2) - expect(wall.entries[0]).toEqual({ - version: '10.4.12', - date: '2026-09-03', - headline: 'fix(wall): mechanize the entry', - items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'], - url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12', - thumb: null, - }) - // the older entry stays put, still second - expect(wall.entries[1].version).toBe('10.4.11') - }) - - it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }])) - - run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - - const wall = readRemote(remoteDir, 'open-brainy') - expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) - }) - - it('replaces an entry with the same version instead of duplicating it — idempotent re-runs', () => { - seedRemote(remoteDir, 'open-brainy', [ - { ...BASE_ENTRY, headline: 'stale headline, pre-fix' }, - { ...BASE_ENTRY, version: '10.4.10' }, - ]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: the corrected headline'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/replaced v10\.4\.11/i) - - const wall = readRemote(remoteDir, 'open-brainy') - expect(wall.entries).toHaveLength(2) // not 3 — replaced, not duplicated - expect(wall.entries[0].version).toBe('10.4.11') - expect(wall.entries[0].headline).toBe('fix: the corrected headline') - expect(wall.entries[1].version).toBe('10.4.10') - }) - - it('a re-run with byte-identical content commits nothing and still succeeds', () => { - // headline always equals items[0] for a derived entry, so this fixture - // (unlike BASE_ENTRY, whose headline/items intentionally diverge for the - // shape-only tests below) has to keep the two in lockstep to ever roundtrip. - const stableEntry = { ...BASE_ENTRY, headline: 'A faster open.', items: ['A faster open.'] } - seedRemote(remoteDir, 'open-brainy', [stableEntry]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['A faster open.'] }])) - const before = readRemote(remoteDir, 'open-brainy') - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/nothing to commit/i) - expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) - }) - - it('derives the public package-page permalink for the product engine (private repo, never null)', () => { - seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: 'https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.5' }]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) - - const result = run( - ['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - - const wall = readRemote(remoteDir, 'brainy') - expect(wall.entries[0].url).toBe('https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.6') - expect(wall.entries[0].thumb).toBeNull() - }) - - it('refuses a product with no permalink pattern, naming the cure', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['feat: first'] }])) - - const result = run(['--product', 'mystery', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - expect(result.status).not.toBe(0) - expect(result.stderr).toMatch(/no permalink pattern for product "mystery"/) - expect(result.stderr).toMatch(/never carry url: null/) - }) - - it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { - seedRemote(remoteDir, 'open-brainy', []) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - const beforeSha = git(['rev-parse', 'main'], remoteDir) - - const result = run( - ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) - expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) - }) - - it('refuses by naming the cure when the remote cannot be cloned', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) - const noSuchRemote = join(tmpdir(), 'wall-remote-does-not-exist-' + Date.now()) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', noSuchRemote, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/cannot clone/i) - expect(result.stderr).toMatch(/cure:/i) - }) - - it('refuses by naming the cure, and touches no remote, when the fetched wall fails shape validation', () => { - const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-broken-')) - execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) - git(['config', 'user.email', 'seed@example.com'], seedDir) - git(['config', 'user.name', 'Seed'], seedDir) - writeFileSync( - join(seedDir, 'open-brainy.json'), - JSON.stringify({ product: 'open-brainy', entries: [{ version: '10.4.11', date: '2026-09-02', items: ['x'], url: null }] }, null, 2), - ) - git(['add', 'open-brainy.json'], seedDir) - git(['commit', '-m', 'seed broken'], seedDir) - git(['push', 'origin', 'main'], seedDir) - rmSync(seedDir, { recursive: true, force: true }) - const beforeSha = git(['rev-parse', 'main'], remoteDir) - - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/fails shape validation/i) - expect(result.stderr).toMatch(/missing key\(s\) headline/i) - expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) - }) - - it('refuses by naming the cure when the remote rejects the push (stands in for a raced non-fast-forward)', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - makeRemoteRejectPushes(remoteDir) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/push to .* failed/i) - expect(result.stderr).toMatch(/cure:/i) - }) - - it('refuses a cross-product write when the file\'s "product" field does not match --product', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-mismatch-')) - execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) - git(['config', 'user.email', 'seed@example.com'], seedDir) - git(['config', 'user.name', 'Seed'], seedDir) - const corrupted = JSON.parse(readFileSync(join(seedDir, 'open-brainy.json'), 'utf8')) - corrupted.product = 'brainy' - writeFileSync(join(seedDir, 'open-brainy.json'), JSON.stringify(corrupted, null, 2) + '\n') - git(['add', 'open-brainy.json'], seedDir) - git(['commit', '-m', 'corrupt product field'], seedDir) - git(['push', 'origin', 'main'], seedDir) - rmSync(seedDir, { recursive: true, force: true }) - - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/product "brainy".*--product "open-brainy"/i) - }) -}) - -describe('wall-entry.mjs — --dry-run', () => { - it('prints the entry and the target path, and touches neither the cache dir nor the remote', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: a dry run'] }])) - const beforeSha = git(['rev-parse', 'main'], remoteDir) - - const result = run( - ['--dry-run', '--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/would write to/i) - expect(result.stdout).toMatch(/"version": "10\.4\.12"/) - expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) - }) -}) - -describe('wall-entry.mjs — --check', () => { - it('passes a well-formed, newest-first file with no duplicates', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/OK/) - }) - - it('passes a file where "thumb" is entirely absent (optional per the HQ contract)', () => { - const { thumb, ...noThumb } = BASE_ENTRY as any - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [noThumb])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(0) - }) - - it('catches a missing entry key', () => { - const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'] } // no "url" - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/missing key\(s\) url/) - }) - - it('catches an unexpected top-level key (e.g. the retired "history" field)', () => { - const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) - raw.history = 'retired field' - writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/unexpected key\(s\) history/) - }) - - it('catches entries that are not newest-first', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/not newest-first/) - }) - - it('catches a duplicate version even with identical entries', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/duplicate version 10\.4\.11/) - }) - - it('catches an empty items array', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/"items" must be a non-empty array/) - }) - - it('catches a malformed date', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/) - }) -})