From 8752f11f4d5e312a47dde521c267e9b395de0d21 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:13:40 -0700 Subject: [PATCH 01/21] chore(releases): the product engine's release wall leaves the reference repo Only the open engine's own wall (releases/open-brainy.json) belongs in the public reference project. The product's notes are served from the product's own repository. --- releases/brainy.json | 76 -------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 releases/brainy.json diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} From 85b1fa5c1a82ccad2f5ce9bd86b31fc18ab13357 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:15:39 -0700 Subject: [PATCH 02/21] =?UTF-8?q?ci(release):=20mechanize=20the=20releases?= =?UTF-8?q?-wall=20entry=20=E2=80=94=20never=20hand-written=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every release used to get its releases/open-brainy.json entry typed by hand after the fact. scripts/wall-entry.mjs derives it from the CHANGELOG entry release.sh just composed (headline = first bullet, items = every bullet, hash stripped) and prepends it, refusing by name on a duplicate version and validating the whole file's shape + newest-first ordering before and after it writes. release.sh now runs it as its own step, between the CHANGELOG update and the release commit, and stages releases/open-brainy.json into that commit. The product engine's rail runs this identical script against its own releases/brainy.json, unchanged — each repo's wall file lives beside the CHANGELOG it derives from; there is no cross-repo step. A --check mode validates a wall file's exact key set, field types, and newest-first ordering with no duplicates, read-only. tests/unit/release/wall-entry.test.ts covers derivation, prepend, duplicate refusal, and --check's shape/ordering checks over temp copies — never the real files. --check also runs green against both releases/open-brainy.json and releases/brainy.json as they stand today. --- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 364 ++++++++++++++++++++++++++ tests/unit/release/wall-entry.test.ts | 216 +++++++++++++++ 3 files changed, 591 insertions(+), 2 deletions(-) create mode 100644 scripts/wall-entry.mjs create mode 100644 tests/unit/release/wall-entry.test.ts diff --git a/scripts/release.sh b/scripts/release.sh index 5d434320..07d225ce 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,8 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +RELEASE_DATE=$(date +%Y-%m-%d) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE}) ${COMMITS} " @@ -174,9 +175,17 @@ if [ -f "CHANGELOG.md" ]; then fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" +# Step 6b: Update the releases wall entry — mechanical, derived from the +# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json +# directly; this used to be hand-written after every release (David: never +# again — make it a step of the rail). +echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" +node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md +echo -e "${GREEN}✅ Releases wall updated${NC}\n" + # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md +git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}✅ Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs new file mode 100644 index 00000000..998431da --- /dev/null +++ b/scripts/wall-entry.mjs @@ -0,0 +1,364 @@ +#!/usr/bin/env node +/** + * @module scripts/wall-entry + * @description The releases-wall entry, made mechanical. The fleet's HQ page + * reads one public JSON per product (releases/.json — shape + * {product, entries:[{version, date, headline, items, url, thumb}], history}). + * Those entries were hand-written after every release; this script is the + * one door that composes one, so it never has to be typed by hand again. + * + * Two modes: + * + * 1. Generate + write in place (default): + * node wall-entry.mjs --product

--version --date \ + * --from-changelog [--file releases/

.json] + * Derives an entry from the CHANGELOG.md entry for (headline = the + * entry's first bullet, items = every bullet, trimmed of its trailing + * commit hash), prepends it to --file (default releases/.json, + * newest first), refusing by name if is already present, and + * validates the whole file's shape + ordering before and after writing. + * Both engines run this identically, each against its own repo's + * releases/.json — the wall file always lives beside the + * CHANGELOG it is derived from, never in another repo. + * + * 2. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates the file's exact key set (top-level and per-entry), field + * types, and strict-descending semver ordering with no duplicates. + * Read-only; never writes. Exit 0 = clean, exit 1 = named violations + * printed to stderr. + * + * No dependencies — CHANGELOG parsing, semver comparison, and JSON shape + * checking are all hand-rolled below. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs' + +const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] +const FILE_KEYS = ['product', 'entries', 'history'] + +// The public release-page URL pattern, by product — only products with a +// PUBLIC forge repo get a derived link. A product without an entry here +// (e.g. "brainy", whose repo is private) gets url: null, matching every +// entry the fleet has shipped for it so far — a private link would 404 for +// anyone reading the public HQ page. +const RELEASE_URL_PATTERNS = { + 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, +} + +/** + * Parse argv into a flag map. `--flag value` sets a string; `--flag` alone + * (end of argv, or followed by another `--flag`) sets boolean true. + * @param {string[]} argv + * @returns {Record} + */ +function parseArgs(argv) { + /** @type {Record} */ + const args = {} + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (!a.startsWith('--')) continue + const key = a.slice(2) + const next = argv[i + 1] + if (next === undefined || next.startsWith('--')) { + args[key] = true + } else { + args[key] = next + i++ + } + } + return args +} + +/** + * Print a loud, named error and exit 1. Every refusal in this script goes + * through here so the failure mode is always the same shape: "wall-entry: ". + * @param {string} message + * @returns {never} + */ +function fail(message) { + console.error(`wall-entry: ${message}`) + process.exit(1) +} + +/** + * @param {string} version + * @returns {{major: number, minor: number, patch: number, pre: string | null} | null} + */ +function parseSemver(version) { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version) + if (!m) return null + return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null } +} + +/** + * @param {string} a + * @param {string} b + * @returns {number} positive if a > b, negative if a < b, 0 if equal. + */ +function compareSemver(a, b) { + const pa = parseSemver(a) + const pb = parseSemver(b) + if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`) + if (pa.major !== pb.major) return pa.major - pb.major + if (pa.minor !== pb.minor) return pa.minor - pb.minor + if (pa.patch !== pb.patch) return pa.patch - pb.patch + if (pa.pre === pb.pre) return 0 + if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version + if (pb.pre === null) return -1 + return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0 +} + +/** + * Validate a wall file's full shape: top-level keys, per-entry keys and + * field types, and strict-descending semver ordering with no duplicates. + * Collects every violation instead of failing on the first, so --check + * reports the whole picture in one pass. + * @param {unknown} data + * @returns {string[]} Violation messages; empty means the file is clean. + */ +function validateShape(data) { + /** @type {string[]} */ + const errors = [] + + if (typeof data !== 'object' || data === null || Array.isArray(data)) { + return ['top level: expected a JSON object'] + } + const obj = /** @type {Record} */ (data) + + const topKeys = Object.keys(obj) + const missingTop = FILE_KEYS.filter((k) => !(k in obj)) + const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k)) + if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`) + if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`) + + if (typeof obj.product !== 'string' || obj.product.trim() === '') { + errors.push('top level: "product" must be a non-empty string') + } + if (typeof obj.history !== 'string' || obj.history.trim() === '') { + errors.push('top level: "history" must be a non-empty string') + } + if (!Array.isArray(obj.entries)) { + errors.push('top level: "entries" must be an array') + return errors // nothing further to check without an array + } + + const entries = /** @type {unknown[]} */ (obj.entries) + entries.forEach((rawEntry, i) => { + const label = `entries[${i}]` + if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) { + errors.push(`${label}: expected an object`) + return + } + const entry = /** @type {Record} */ (rawEntry) + const keys = Object.keys(entry) + const missing = ENTRY_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) + if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) + + if (typeof entry.version !== 'string' || !parseSemver(entry.version)) { + errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`) + } + if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) { + errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`) + } + if (typeof entry.headline !== 'string' || entry.headline.trim() === '') { + errors.push(`${label}: "headline" must be a non-empty string`) + } + if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { + errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) + } + if (!(entry.url === null || typeof entry.url === 'string')) { + errors.push(`${label}: "url" must be a string or null`) + } + if (!(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null`) + } + }) + + // Ordering: newest first, strictly descending, no duplicate versions — + // checked only over entries whose version parsed (a bad version is + // already reported above; comparing it too would just be noise). + const versioned = entries + .map((e, i) => ({ i, version: /** @type {any} */ (e)?.version })) + .filter((e) => typeof e.version === 'string' && parseSemver(e.version)) + for (let i = 0; i < versioned.length - 1; i++) { + const a = versioned[i] + const b = versioned[i + 1] + const cmp = compareSemver(a.version, b.version) + if (cmp === 0) { + errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`) + } else if (cmp < 0) { + errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`) + } + } + + return errors +} + +/** + * Extract one version's entry body from a standard-version-style CHANGELOG.md + * (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines + * until the next heading or EOF). + * @param {string} changelog + * @param {string} version + * @returns {string[]} Bullet lines, trimmed of their leading "- " and + * trailing " (hash)". + */ +function extractChangelogBullets(changelog, version) { + const lines = changelog.split('\n') + const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/ + let start = -1 + for (let i = 0; i < lines.length; i++) { + const m = headingRe.exec(lines[i]) + if (m && m[1] === version) { + start = i + 1 + break + } + } + if (start === -1) { + fail( + `version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`, + ) + } + /** @type {string[]} */ + const bullets = [] + for (let i = start; i < lines.length; i++) { + if (headingRe.test(lines[i])) break // next entry starts + const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim()) + if (lines[i].trim().startsWith('- ') && bulletMatch) { + const text = bulletMatch[1].trim() + if (text) bullets.push(text) + } + } + if (bullets.length === 0) { + fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`) + } + return bullets +} + +/** + * Derive a wall entry from a CHANGELOG.md. + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + */ +function deriveEntry({ product, version, date, changelogPath, url, thumb }) { + if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) + if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) { + fail(`--date "${date}" is not a YYYY-MM-DD date`) + } + if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`) + + const changelog = readFileSync(changelogPath, 'utf8') + const items = extractChangelogBullets(changelog, version) + const headline = items[0] + + const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const resolvedThumb = thumb !== undefined ? thumb : null + + return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } +} + +/** + * Load and shape-validate a wall file. + * @param {string} filePath + * @returns {Record} + */ +function loadWallFile(filePath) { + if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + /** @type {unknown} */ + let data + try { + data = JSON.parse(readFileSync(filePath, 'utf8')) + } catch (err) { + fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + } + const errors = validateShape(data) + if (errors.length) { + fail(`--file "${filePath}" fails shape validation before any write —\n ${errors.join('\n ')}`) + } + return /** @type {Record} */ (data) +} + +/** + * Prepend `entry` to the wall file at `filePath`, refusing by name if the + * version is already present, validating before and after, and writing the + * file back with the repo's exact formatting (2-space JSON, trailing newline). + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} filePath + * @param {string | undefined} expectedProduct + */ +function applyEntry(entry, filePath, expectedProduct) { + const wall = loadWallFile(filePath) + + if (expectedProduct && wall.product !== expectedProduct) { + fail( + `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given — refusing a cross-product write`, + ) + } + + if (wall.entries.some((e) => e.version === entry.version)) { + fail(`refusing — version ${entry.version} is already present in "${filePath}"`) + } + + wall.entries = [entry, ...wall.entries] + + const postErrors = validateShape(wall) + if (postErrors.length) { + fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`) + } + + writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') + console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) +} + +function main() { + const args = parseArgs(process.argv.slice(2)) + + if (args.check) { + const filePath = /** @type {string | undefined} */ (args.file) ?? + (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) + if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const wall = loadWallFile(/** @type {string} */ (filePath)) + console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) + process.exit(0) + } + + // Generate mode (default): --product, --version, --date, --from-changelog required. + const product = /** @type {string | undefined} */ (args.product) + const version = /** @type {string | undefined} */ (args.version) + const date = /** @type {string | undefined} */ (args.date) + const fromChangelog = /** @type {string | undefined} */ (args['from-changelog']) + + const missing = [] + if (!product) missing.push('--product') + if (!version) missing.push('--version') + if (!date) missing.push('--date') + if (!fromChangelog) missing.push('--from-changelog') + if (missing.length) { + fail( + `missing required flag(s): ${missing.join(', ')}\n` + + 'Usage:\n' + + ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + + ' wall-entry.mjs --check --file ', + ) + } + + const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url) + const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb) + + const entry = deriveEntry({ + product: /** @type {string} */ (product), + version: /** @type {string} */ (version), + date: /** @type {string} */ (date), + changelogPath: /** @type {string} */ (fromChangelog), + url: urlArg, + thumb: thumbArg, + }) + + const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) + applyEntry(entry, filePath, /** @type {string} */ (product)) +} + +main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts new file mode 100644 index 00000000..fc41731c --- /dev/null +++ b/tests/unit/release/wall-entry.test.ts @@ -0,0 +1,216 @@ +/** + * scripts/wall-entry.mjs — the mechanical releases-wall entry. + * + * The script's only real interface is its CLI (it has no importable + * exports by design — one door, no parallel API to drift from it), so + * these tests spawn it exactly as scripts/release.sh does: as a child + * process, against a temp copy of a wall file and a fixture CHANGELOG, + * never against the repo's real releases/*.json. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs') + +/** Run the script and capture the outcome without throwing on a non-zero exit. */ +function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } { + try { + const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' }) + return { status: 0, stdout, stderr: '' } + } catch (err: any) { + return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } + } +} + +const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' + +/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ +function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string { + const body = entries + .map( + (e) => + `### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` + + e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') + + '\n', + ) + .join('\n') + return CHANGELOG_HEADER + '\n' + body +} + +function wallFile(product: string, entries: unknown[]): string { + return JSON.stringify( + { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, + null, + 2, + ) + '\n' +} + +const BASE_ENTRY = { + version: '10.4.11', + date: '2026-09-02', + headline: 'A faster open', + items: ['A faster open.'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11', + thumb: null, +} + +let dir: string + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) +}) + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }) +}) + +describe('wall-entry.mjs — generate + prepend', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + expect(result.status).toBe(0) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries).toHaveLength(2) + expect(wall.entries[0]).toEqual({ + version: '10.4.12', + date: '2026-09-03', + headline: 'fix(wall): mechanize the entry', + items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12', + thumb: null, + }) + // the older entry stays put, still second + expect(wall.entries[1].version).toBe('10.4.11') + }) + + it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) + }) + + it('derives no URL (null) for a product with no known public release-page pattern', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) + + run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].thumb).toBeNull() + }) + + it('refuses by name when the version is already present, and leaves the file untouched', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + const before = wallFile('open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'wall.json'), before) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) + expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched + }) + + it('refuses when the CHANGELOG has no entry yet for the target version', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) + + const result = run( + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + }) + + it('refuses a cross-product write when --product does not match the target file', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + }) +}) + +describe('wall-entry.mjs — --check', () => { + it('passes a well-formed, newest-first file with no duplicates', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/OK/) + }) + + it('catches a missing entry key', () => { + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/missing key\(s\) thumb/) + }) + + it('catches an unexpected top-level key', () => { + const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) + raw.extra = 'not allowed' + writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + }) + + it('catches entries that are not newest-first', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/not newest-first/) + }) + + it('catches a duplicate version even with identical entries', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/duplicate version 10\.4\.11/) + }) + + it('catches an empty items array', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"items" must be a non-empty array/) + }) + + it('catches a malformed date', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/) + }) +}) From adcb883e67ab82b749d37a510ab66323ae1da64e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:51:33 -0700 Subject: [PATCH 03/21] ci(release): publish the wall entry to the shared releases repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rail used to write releases/open-brainy.json (and, before that, also carried the product engine's releases/brainy.json) in this repo. It now clones (or refreshes a cached clone of) soulcraftlabs/releases on The Source, prepends the derived entry to open-brainy.json there (replacing any entry for the same version so a re-run is idempotent), and pushes main directly. Any failure — clone, shape validation, commit, or a rejected push — exits non-zero naming the cure; nothing is ever skipped. Both wall files are gone from this repo — the shared repo is the one home HQ reads. --dry-run derives and prints the entry without touching any clone or remote. Tests point --remote/--cache-dir at a throwaway local bare repo and cache dir, never the real ones. --- releases/brainy.json | 76 -------- releases/open-brainy.json | 136 ------------- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 253 ++++++++++++++++++------ tests/unit/release/wall-entry.test.ts | 271 +++++++++++++++++++++----- 5 files changed, 423 insertions(+), 326 deletions(-) delete mode 100644 releases/brainy.json delete mode 100644 releases/open-brainy.json diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 9f1cd239..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.11", - "date": "2026-09-02", - "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", - "items": [ - "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page — one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", - "A brain now has exactly one shutdown owner — a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", - "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright — both scoped and recursive:false searches were silently broken before this.", - "Open no longer rescans a brain's whole fact log on every open — sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", - "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn — two correctness invariants that were previously assumed, not enforced." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", - "thumb": null - }, - { - "version": "10.4.10", - "date": "2026-09-02", - "headline": "A planner door for indexes, batched containment repair, and a fixed near()", - "items": [ - "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", - "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", - "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", - "thumb": null - }, - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} diff --git a/scripts/release.sh b/scripts/release.sh index 07d225ce..142fa06f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -176,16 +176,21 @@ fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" # Step 6b: Update the releases wall entry — mechanical, derived from the -# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json -# directly; this used to be hand-written after every release (David: never -# again — make it a step of the rail). +# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json +# from the one shared releases repo, soulcraftlabs/releases on The Source — +# this used to be hand-written after every release (David: never again — +# make it a step of the rail, landed in the one shared home; this repo no +# longer hosts its own copy). This step clones/fetches that repo into a +# local cache, prepends the entry, and pushes it directly — a real +# cross-repo push, refusing loudly (never skipping) on any +# clone/validation/commit/push failure. echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md echo -e "${GREEN}✅ Releases wall updated${NC}\n" # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json +git add package.json package-lock.json CHANGELOG.md git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}✅ Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index 998431da..043341eb 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -2,40 +2,76 @@ /** * @module scripts/wall-entry * @description The releases-wall entry, made mechanical. The fleet's HQ page - * reads one public JSON per product (releases/.json — shape - * {product, entries:[{version, date, headline, items, url, thumb}], history}). - * Those entries were hand-written after every release; this script is the - * one door that composes one, so it never has to be typed by hand again. + * reads one public JSON per product from the ONE releases repo on The Source + * (soulcraftlabs/releases, files .json at its root — shape + * {product, entries:[{version, date, headline, items, url, thumb?}]}), at + * https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/.json. + * Those entries were hand-written after every release, then briefly written + * into this repo's own releases/.json; this script is the one door + * that composes an entry and lands it in the shared repo, so it is never + * hand-written and never forked across repos again. * * Two modes: * - * 1. Generate + write in place (default): + * 1. Generate + publish (default): * node wall-entry.mjs --product

--version --date \ - * --from-changelog [--file releases/

.json] + * --from-changelog * Derives an entry from the CHANGELOG.md entry for (headline = the * entry's first bullet, items = every bullet, trimmed of its trailing - * commit hash), prepends it to --file (default releases/.json, - * newest first), refusing by name if is already present, and - * validates the whole file's shape + ordering before and after writing. - * Both engines run this identically, each against its own repo's - * releases/.json — the wall file always lives beside the - * CHANGELOG it is derived from, never in another repo. + * commit hash), then: + * - clones (or, if a cached clone already exists, fetches and resets) + * the releases repo into a local cache directory, + * - prepends the entry to /

.json, newest first — replacing + * any existing entry for the same version so a re-run is idempotent, + * - validates the file's shape before and after, + * - commits the change as "chore(wall):

" and pushes main. + * A failure at any step (clone, validation, commit, push, a + * non-fast-forward remote) exits non-zero naming the cure. Nothing is + * ever skipped — the wall either lands correctly or the release fails. * - * 2. Validate only (--check): - * node wall-entry.mjs --check --file - * Validates the file's exact key set (top-level and per-entry), field - * types, and strict-descending semver ordering with no duplicates. - * Read-only; never writes. Exit 0 = clean, exit 1 = named violations - * printed to stderr. + * 2. Dry run: + * node wall-entry.mjs --dry-run --product

--version \ + * --date --from-changelog + * Derives the entry exactly as above and prints it, along with the file + * it would be written to, but touches no clone and no remote — usable + * from a fresh checkout with no cache and no network. * - * No dependencies — CHANGELOG parsing, semver comparison, and JSON shape - * checking are all hand-rolled below. + * 3. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates an arbitrary wall file's exact key set (top-level and + * per-entry), field types, and strict-descending semver ordering with + * no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 = + * named violations printed to stderr. + * + * The remote and the local cache directory are each overridable + * (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE / + * WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local + * bare repo and a throwaway cache directory — never the real remote or the + * real developer cache. + * + * No dependencies beyond the system `git` binary — CHANGELOG parsing, + * semver comparison, and JSON shape checking are all hand-rolled below. */ -import { readFileSync, writeFileSync, existsSync } from 'node:fs' +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' +import { execFileSync } from 'node:child_process' +import { homedir } from 'node:os' +import { dirname, join } from 'node:path' -const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] -const FILE_KEYS = ['product', 'entries', 'history'] +const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git' + +/** @returns {string} */ +function defaultCacheDir() { + const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache') + return join(base, 'soulcraft-releases') +} + +// Required on every entry; "thumb" is optional (may be absent, or present as +// string | null) — matching the HQ contract's {..., thumb?}. +const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url'] +const ENTRY_OPTIONAL_KEYS = ['thumb'] +const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] +const FILE_KEYS = ['product', 'entries'] // The public release-page URL pattern, by product — only products with a // PUBLIC forge repo get a derived link. A product without an entry here @@ -110,10 +146,11 @@ function compareSemver(a, b) { } /** - * Validate a wall file's full shape: top-level keys, per-entry keys and - * field types, and strict-descending semver ordering with no duplicates. - * Collects every violation instead of failing on the first, so --check - * reports the whole picture in one pass. + * Validate a wall file's full shape: top-level keys ("product", "entries" — + * no more, no less), per-entry keys and field types ("thumb" optional), and + * strict-descending semver ordering with no duplicates. Collects every + * violation instead of failing on the first, so a caller reports the whole + * picture in one pass. * @param {unknown} data * @returns {string[]} Violation messages; empty means the file is clean. */ @@ -135,9 +172,6 @@ function validateShape(data) { if (typeof obj.product !== 'string' || obj.product.trim() === '') { errors.push('top level: "product" must be a non-empty string') } - if (typeof obj.history !== 'string' || obj.history.trim() === '') { - errors.push('top level: "history" must be a non-empty string') - } if (!Array.isArray(obj.entries)) { errors.push('top level: "entries" must be an array') return errors // nothing further to check without an array @@ -152,8 +186,8 @@ function validateShape(data) { } const entry = /** @type {Record} */ (rawEntry) const keys = Object.keys(entry) - const missing = ENTRY_KEYS.filter((k) => !(k in entry)) - const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k)) if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) @@ -172,8 +206,8 @@ function validateShape(data) { if (!(entry.url === null || typeof entry.url === 'string')) { errors.push(`${label}: "url" must be a string or null`) } - if (!(entry.thumb === null || typeof entry.thumb === 'string')) { - errors.push(`${label}: "thumb" must be a string or null`) + if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null when present`) } }) @@ -266,43 +300,110 @@ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { * @returns {Record} */ function loadWallFile(filePath) { - if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + if (!existsSync(filePath)) fail(`"${filePath}" does not exist`) /** @type {unknown} */ let data try { data = JSON.parse(readFileSync(filePath, 'utf8')) } catch (err) { - fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) } const errors = validateShape(data) if (errors.length) { - fail(`--file "${filePath}" fails shape validation before any write —\n ${errors.join('\n ')}`) + fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`) } return /** @type {Record} */ (data) } /** - * Prepend `entry` to the wall file at `filePath`, refusing by name if the - * version is already present, validating before and after, and writing the - * file back with the repo's exact formatting (2-space JSON, trailing newline). - * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry - * @param {string} filePath - * @param {string | undefined} expectedProduct + * Run a git command, throwing an Error whose message is git's own stderr + * (trimmed) on failure — every caller wraps this to name the cure. + * @param {string[]} args + * @param {string} cwd + * @returns {string} stdout, trimmed. */ -function applyEntry(entry, filePath, expectedProduct) { - const wall = loadWallFile(filePath) +function git(args, cwd) { + try { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim() + } catch (err) { + const stderr = /** @type {any} */ (err).stderr + const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message + throw new Error(message) + } +} - if (expectedProduct && wall.product !== expectedProduct) { +/** + * Ensure a clean, up-to-date local clone of the releases repo at + * `cacheDir`, checked out on `main` — cloning fresh if `cacheDir` has no + * `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a + * stray local commit or edit left by a previous failed run can never leak + * into the next one). + * @param {string} remote + * @param {string} cacheDir + */ +function ensureReleasesClone(remote, cacheDir) { + if (existsSync(join(cacheDir, '.git'))) { + try { + git(['remote', 'set-url', 'origin', remote], cacheDir) + git(['fetch', '--prune', 'origin'], cacheDir) + git(['checkout', 'main'], cacheDir) + git(['reset', '--hard', 'origin/main'], cacheDir) + git(['clean', '-fd'], cacheDir) + } catch (err) { + fail( + `cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` + + ` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`, + ) + } + return + } + + mkdirSync(dirname(cacheDir), { recursive: true }) + try { + git(['clone', remote, cacheDir], dirname(cacheDir)) + } catch (err) { fail( - `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given — refusing a cross-product write`, + `cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` + + ` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`, ) } + try { + git(['checkout', 'main'], cacheDir) + } catch (err) { + fail( + `cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` + + ` cure: confirm the releases repo's default branch is named "main"`, + ) + } +} - if (wall.entries.some((e) => e.version === entry.version)) { - fail(`refusing — version ${entry.version} is already present in "${filePath}"`) +/** + * Prepend `entry` to the wall at `/.json`, replacing any + * existing entry for the same version (idempotent re-runs), validating + * before and after, committing, and pushing — or refusing loudly, naming + * the cure, at whichever step fails. + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} product + * @param {string} remote + * @param {string} cacheDir + */ +function publishEntry(entry, product, remote, cacheDir) { + ensureReleasesClone(remote, cacheDir) + + const filePath = join(cacheDir, `${product}.json`) + if (!existsSync(filePath)) { + fail( + `"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`, + ) + } + const wall = loadWallFile(filePath) + + if (wall.product !== product) { + fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`) } - wall.entries = [entry, ...wall.entries] + const replacing = wall.entries.some((e) => e.version === entry.version) + wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)] const postErrors = validateShape(wall) if (postErrors.length) { @@ -310,22 +411,48 @@ function applyEntry(entry, filePath, expectedProduct) { } writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') - console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) + + const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir) + if (status === '') { + console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`) + return + } + + try { + git(['add', `${product}.json`], cacheDir) + git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir) + } catch (err) { + fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`) + } + + try { + git(['push', 'origin', 'main'], cacheDir) + } catch (err) { + fail( + `push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` + + ` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`, + ) + } + + const sha = git(['rev-parse', 'HEAD'], cacheDir) + console.log( + `wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`, + ) } function main() { const args = parseArgs(process.argv.slice(2)) if (args.check) { - const filePath = /** @type {string | undefined} */ (args.file) ?? - (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) - if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const filePath = /** @type {string | undefined} */ (args.file) + if (!filePath) fail('--check needs --file ') const wall = loadWallFile(/** @type {string} */ (filePath)) console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) process.exit(0) } - // Generate mode (default): --product, --version, --date, --from-changelog required. + // Generate mode (default, also covers --dry-run): --product, --version, + // --date, --from-changelog required. const product = /** @type {string | undefined} */ (args.product) const version = /** @type {string | undefined} */ (args.version) const date = /** @type {string | undefined} */ (args.date) @@ -340,8 +467,8 @@ function main() { fail( `missing required flag(s): ${missing.join(', ')}\n` + 'Usage:\n' + - ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + - ' wall-entry.mjs --check --file ', + ' wall-entry.mjs --product

--version --date --from-changelog [--dry-run]\n' + + ' wall-entry.mjs --check --file ', ) } @@ -357,8 +484,16 @@ function main() { thumb: thumbArg, }) - const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) - applyEntry(entry, filePath, /** @type {string} */ (product)) + const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE) + const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir()) + + if (args['dry-run']) { + console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`) + console.log(JSON.stringify(entry, null, 2)) + process.exit(0) + } + + publishEntry(entry, /** @type {string} */ (product), remote, cacheDir) } main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index fc41731c..7f96da25 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -4,12 +4,15 @@ * The script's only real interface is its CLI (it has no importable * exports by design — one door, no parallel API to drift from it), so * these tests spawn it exactly as scripts/release.sh does: as a child - * process, against a temp copy of a wall file and a fixture CHANGELOG, - * never against the repo's real releases/*.json. + * process, against a fixture CHANGELOG and a throwaway local bare repo + * standing in for git@source.soulcraft.com:soulcraftlabs/releases.git + * (--remote) plus a throwaway cache directory (--cache-dir) standing in + * for ~/.cache/soulcraft-releases — never the real remote, never the + * real developer cache. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { execFileSync } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -25,6 +28,10 @@ function run(args: string[], cwd: string): { status: number; stdout: string; std } } +function git(args: string[], cwd: string): string { + return execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim() +} + const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' /** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ @@ -41,11 +48,7 @@ function buildChangelog(entries: Array<{ version: string; date: string; bullets: } function wallFile(product: string, entries: unknown[]): string { - return JSON.stringify( - { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, - null, - 2, - ) + '\n' + return JSON.stringify({ product, entries }, null, 2) + '\n' } const BASE_ENTRY = { @@ -57,31 +60,76 @@ const BASE_ENTRY = { thumb: null, } +/** A throwaway bare repo standing in for the real soulcraftlabs/releases remote. */ +function initBareRemote(): string { + const remoteDir = mkdtempSync(join(tmpdir(), 'wall-remote-')) + execFileSync('git', ['init', '--bare', '-b', 'main', remoteDir]) + return remoteDir +} + +/** Seed the bare remote with an initial .json, via a throwaway clone. */ +function seedRemote(remoteDir: string, product: string, entries: unknown[]): void { + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + writeFileSync(join(seedDir, `${product}.json`), wallFile(product, entries)) + git(['add', `${product}.json`], seedDir) + git(['commit', '-m', 'seed'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) +} + +/** Read .json back out of the bare remote's main tip, via a throwaway clone. */ +function readRemote(remoteDir: string, product: string): any { + const readDir = mkdtempSync(join(tmpdir(), 'wall-read-')) + execFileSync('git', ['clone', remoteDir, readDir], { stdio: 'ignore' }) + const data = JSON.parse(readFileSync(join(readDir, `${product}.json`), 'utf8')) + rmSync(readDir, { recursive: true, force: true }) + return data +} + +/** Reject every push — stands in for any push failure (including a genuine + * non-fast-forward raced by a concurrent release rail), which this script + * treats identically: refuse loudly, name the cure, touch nothing further. */ +function makeRemoteRejectPushes(remoteDir: string): void { + const hookPath = join(remoteDir, 'hooks', 'pre-receive') + writeFileSync(hookPath, '#!/bin/sh\necho "remote: simulated push rejection" >&2\nexit 1\n') + chmodSync(hookPath, 0o755) +} + let dir: string +let remoteDir: string +let cacheDir: string beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) + remoteDir = initBareRemote() + cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases') }) afterEach(() => { rmSync(dir, { recursive: true, force: true }) + rmSync(remoteDir, { recursive: true, force: true }) + rmSync(cacheDir, { recursive: true, force: true }) }) -describe('wall-entry.mjs — generate + prepend', () => { - it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { +describe('wall-entry.mjs — generate + publish', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped, and pushes it to the remote', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) writeFileSync( join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), ) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(0) + expect(result.stdout).toMatch(/wrote v10\.4\.12.*pushed/i) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'open-brainy') expect(wall.entries).toHaveLength(2) expect(wall.entries[0]).toEqual({ version: '10.4.12', @@ -96,68 +144,182 @@ describe('wall-entry.mjs — generate + prepend', () => { }) it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { - writeFileSync( - join(dir, 'CHANGELOG.md'), - buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), - ) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }])) - run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'open-brainy') expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) }) + it('replaces an entry with the same version instead of duplicating it — idempotent re-runs', () => { + seedRemote(remoteDir, 'open-brainy', [ + { ...BASE_ENTRY, headline: 'stale headline, pre-fix' }, + { ...BASE_ENTRY, version: '10.4.10' }, + ]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: the corrected headline'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/replaced v10\.4\.11/i) + + const wall = readRemote(remoteDir, 'open-brainy') + expect(wall.entries).toHaveLength(2) // not 3 — replaced, not duplicated + expect(wall.entries[0].version).toBe('10.4.11') + expect(wall.entries[0].headline).toBe('fix: the corrected headline') + expect(wall.entries[1].version).toBe('10.4.10') + }) + + it('a re-run with byte-identical content commits nothing and still succeeds', () => { + // headline always equals items[0] for a derived entry, so this fixture + // (unlike BASE_ENTRY, whose headline/items intentionally diverge for the + // shape-only tests below) has to keep the two in lockstep to ever roundtrip. + const stableEntry = { ...BASE_ENTRY, headline: 'A faster open.', items: ['A faster open.'] } + seedRemote(remoteDir, 'open-brainy', [stableEntry]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['A faster open.'] }])) + const before = readRemote(remoteDir, 'open-brainy') + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/nothing to commit/i) + expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) + }) + it('derives no URL (null) for a product with no known public release-page pattern', () => { + seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }]) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) - run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + const result = run( + ['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'brainy') expect(wall.entries[0].url).toBeNull() expect(wall.entries[0].thumb).toBeNull() }) - it('refuses by name when the version is already present, and leaves the file untouched', () => { + it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { + seedRemote(remoteDir, 'open-brainy', []) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - const before = wallFile('open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'wall.json'), before) + const beforeSha = git(['rev-parse', 'main'], remoteDir) const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) - expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched - }) - - it('refuses when the CHANGELOG has no entry yet for the target version', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) - - const result = run( - ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(1) expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) }) - it('refuses a cross-product write when --product does not match the target file', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + it('refuses by naming the cure when the remote cannot be cloned', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + const noSuchRemote = join(tmpdir(), 'wall-remote-does-not-exist-' + Date.now()) const result = run( - ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', noSuchRemote, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + expect(result.stderr).toMatch(/cannot clone/i) + expect(result.stderr).toMatch(/cure:/i) + }) + + it('refuses by naming the cure, and touches no remote, when the fetched wall fails shape validation', () => { + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-broken-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + writeFileSync( + join(seedDir, 'open-brainy.json'), + JSON.stringify({ product: 'open-brainy', entries: [{ version: '10.4.11', date: '2026-09-02', items: ['x'], url: null }] }, null, 2), + ) + git(['add', 'open-brainy.json'], seedDir) + git(['commit', '-m', 'seed broken'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) + const beforeSha = git(['rev-parse', 'main'], remoteDir) + + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/fails shape validation/i) + expect(result.stderr).toMatch(/missing key\(s\) headline/i) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) + }) + + it('refuses by naming the cure when the remote rejects the push (stands in for a raced non-fast-forward)', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + makeRemoteRejectPushes(remoteDir) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/push to .* failed/i) + expect(result.stderr).toMatch(/cure:/i) + }) + + it('refuses a cross-product write when the file\'s "product" field does not match --product', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-mismatch-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + const corrupted = JSON.parse(readFileSync(join(seedDir, 'open-brainy.json'), 'utf8')) + corrupted.product = 'brainy' + writeFileSync(join(seedDir, 'open-brainy.json'), JSON.stringify(corrupted, null, 2) + '\n') + git(['add', 'open-brainy.json'], seedDir) + git(['commit', '-m', 'corrupt product field'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) + + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "brainy".*--product "open-brainy"/i) + }) +}) + +describe('wall-entry.mjs — --dry-run', () => { + it('prints the entry and the target path, and touches neither the cache dir nor the remote', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: a dry run'] }])) + const beforeSha = git(['rev-parse', 'main'], remoteDir) + + const result = run( + ['--dry-run', '--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/would write to/i) + expect(result.stdout).toMatch(/"version": "10\.4\.12"/) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) }) }) @@ -169,21 +331,28 @@ describe('wall-entry.mjs — --check', () => { expect(result.stdout).toMatch(/OK/) }) + it('passes a file where "thumb" is entirely absent (optional per the HQ contract)', () => { + const { thumb, ...noThumb } = BASE_ENTRY as any + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [noThumb])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + }) + it('catches a missing entry key', () => { - const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'] } // no "url" writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) const result = run(['--check', '--file', 'wall.json'], dir) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/missing key\(s\) thumb/) + expect(result.stderr).toMatch(/missing key\(s\) url/) }) - it('catches an unexpected top-level key', () => { + it('catches an unexpected top-level key (e.g. the retired "history" field)', () => { const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) - raw.extra = 'not allowed' + raw.history = 'retired field' writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) const result = run(['--check', '--file', 'wall.json'], dir) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + expect(result.stderr).toMatch(/unexpected key\(s\) history/) }) it('catches entries that are not newest-first', () => { From aa457d715937142607245f93437f987ba00248f0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:52:05 -0700 Subject: [PATCH 04/21] chore(releases): both walls leave the reference repo, RELEASES.md points home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit releases/open-brainy.json follows brainy.json out — the shared repo (soulcraftlabs/releases on The Source) is now the one home for both products' release notes; this repo hosts neither. The releases/ directory is gone. RELEASES.md gains a pointer, under the heading, to the two raw URLs HQ's /hq/releases door reads (this file stays as the human-readable quick reference; those files are the source of truth). --- RELEASES.md | 7 ++ releases/open-brainy.json | 136 -------------------------------------- 2 files changed, 7 insertions(+), 136 deletions(-) delete mode 100644 releases/open-brainy.json diff --git a/RELEASES.md b/RELEASES.md index e8833b80..c875cb26 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,12 @@ # @soulcraft/brainy — Release Notes for Consumers +Machine-readable release notes are published at +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json +(this engine) and +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json +(the product engine) — read by HQ's `/hq/releases` door, and the source of +truth ahead of this file. + This file is the **quick reference for downstream sessions** tracking Brainy changes. Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 9f1cd239..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.11", - "date": "2026-09-02", - "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", - "items": [ - "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page — one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", - "A brain now has exactly one shutdown owner — a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", - "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright — both scoped and recursive:false searches were silently broken before this.", - "Open no longer rescans a brain's whole fact log on every open — sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", - "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn — two correctness invariants that were previously assumed, not enforced." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", - "thumb": null - }, - { - "version": "10.4.10", - "date": "2026-09-02", - "headline": "A planner door for indexes, batched containment repair, and a fixed near()", - "items": [ - "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", - "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", - "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", - "thumb": null - }, - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} From 97b5ea2d5ddb739f1d1d0ff4e31664b5ef551df4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:59:49 -0700 Subject: [PATCH 05/21] =?UTF-8?q?fix(wall):=20every=20entry=20carries=20an?= =?UTF-8?q?=20https=20permalink=20=E2=80=94=20the=20product=20engine=20lin?= =?UTF-8?q?ks=20its=20public=20package=20page;=20null=20refused,=20an=20un?= =?UTF-8?q?known=20product=20refuses=20by=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/wall-entry.mjs | 27 ++++++++++++++++----------- tests/unit/release/wall-entry.test.ts | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index 043341eb..d4ec7ba5 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -73,13 +73,14 @@ const ENTRY_OPTIONAL_KEYS = ['thumb'] const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] const FILE_KEYS = ['product', 'entries'] -// The public release-page URL pattern, by product — only products with a -// PUBLIC forge repo get a derived link. A product without an entry here -// (e.g. "brainy", whose repo is private) gets url: null, matching every -// entry the fleet has shipped for it so far — a private link would 404 for -// anyone reading the public HQ page. +// The public permalink pattern, by product. Every entry MUST carry an https +// permalink: HQ's parser rejects a wall whose entries carry url: null (the +// whole feed became unreadable on 2026-09-02). A product whose forge repo is +// private links its PUBLIC package page on The Source instead of a release +// page that would 404 for HQ's readers. const RELEASE_URL_PATTERNS = { 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, + 'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`, } /** @@ -203,8 +204,8 @@ function validateShape(data) { if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) } - if (!(entry.url === null || typeof entry.url === 'string')) { - errors.push(`${label}: "url" must be a string or null`) + if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) { + errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`) } if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { errors.push(`${label}: "thumb" must be a string or null when present`) @@ -274,8 +275,8 @@ function extractChangelogBullets(changelog, version) { /** * Derive a wall entry from a CHANGELOG.md. - * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts - * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} */ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) @@ -288,7 +289,11 @@ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { const items = extractChangelogBullets(changelog, version) const headline = items[0] - const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const pattern = RELEASE_URL_PATTERNS[product] + if (url === undefined && pattern === undefined) { + throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`) + } + const resolvedUrl = url !== undefined ? url : pattern(version) const resolvedThumb = thumb !== undefined ? thumb : null return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } @@ -382,7 +387,7 @@ function ensureReleasesClone(remote, cacheDir) { * existing entry for the same version (idempotent re-runs), validating * before and after, committing, and pushing — or refusing loudly, naming * the cure, at whichever step fails. - * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry * @param {string} product * @param {string} remote * @param {string} cacheDir diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index 7f96da25..8bf9d357 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -192,8 +192,8 @@ describe('wall-entry.mjs — generate + publish', () => { expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) }) - it('derives no URL (null) for a product with no known public release-page pattern', () => { - seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }]) + it('derives the public package-page permalink for the product engine (private repo, never null)', () => { + seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: 'https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.5' }]) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) const result = run( @@ -203,10 +203,20 @@ describe('wall-entry.mjs — generate + publish', () => { expect(result.status).toBe(0) const wall = readRemote(remoteDir, 'brainy') - expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].url).toBe('https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.6') expect(wall.entries[0].thumb).toBeNull() }) + it('refuses a product with no permalink pattern, naming the cure', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['feat: first'] }])) + + const result = run(['--product', 'mystery', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) + expect(result.status).not.toBe(0) + expect(result.stderr).toMatch(/no permalink pattern for product "mystery"/) + expect(result.stderr).toMatch(/never carry url: null/) + }) + it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { seedRemote(remoteDir, 'open-brainy', []) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) From a128f0eda5b450ebf9caeae8e78ecaceff04feeb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 08:52:58 -0700 Subject: [PATCH 06/21] fix(index): a field holds every value kind it was written with, not the first one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metadata index fixed a field's value type from the first value it saw. Every later value of another kind was coerced to that type, and when coercion failed — `Number('electronics')` is NaN — the value was dropped from the index with no error at all. The row stayed readable by id and by vector search and vanished only from equality filters on that one field, which is what made it so quiet: writing `category: 'electronics'` rows and then `category: 5` rows left `where { category: 5 }` returning nothing, while the same rows in a numbers-only corpus answered correctly. The column store now keeps one posting column per (field, kind), where a kind is a JavaScript typeof class. The first kind a field sees keeps the historical `_column_index//` layout, so a single-kind field is byte-identical to what earlier versions wrote and an index written before this opens unchanged; each later kind takes its own column at `_column_index//k//`. Equality reads the column matching the query value's own kind, so `{c: 5}` and `{c: '5'}` match different rows and neither is coerced into the other. Ranges route by the kind of their bounds, and an unbounded range — the "has any value" probe behind `exists` — reads every kind. A mixed field orders by kind first, then by value, because a number and a string have no order between them. A value that cannot be encoded for the column its own kind selected now raises instead of being skipped: that path is unreachable by construction, and if it is ever reached it is the silent drop this change exists to end. Two neighbours fell out of the same routing. A boolean query value is now encoded to the 1/0 the column stores, so boolean equality matches at all. And an integer column widens to f64 the first time a non-integer arrives, so 4.5 is stored as itself rather than rounded to 5 and answering the wrong query. Field type inference reports every kind a field holds beside its dominant reading, rather than leaving callers to treat one type as the whole answer. Pins: mixed-kind equality in both write orders, `5` vs `'5'`, booleans mixed in, a numeric range over a mixed field's numbers, close/reopen keeping every typed posting, and an index in the pre-existing on-disk shape still reading. `tests/critical-neural-validation.test.ts` — which writes `category` as strings in one test and as numbers in another against one shared brain — passes whole for the first time. --- .../architecture/data-storage-architecture.md | 34 ++ src/indexes/columnStore/ColumnStore.ts | 578 ++++++++++++++---- src/indexes/columnStore/ColumnTailBuffer.ts | 40 +- src/indexes/columnStore/types.ts | 60 ++ src/utils/fieldTypeInference.ts | 83 ++- .../metadata-field-typing.unit.test.ts | 122 ++++ .../column-store-mixed-kind.test.ts | 241 ++++++++ 7 files changed, 1025 insertions(+), 133 deletions(-) create mode 100644 tests/regression/metadata-field-typing.unit.test.ts create mode 100644 tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 83b9e23a..12398747 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -217,6 +217,40 @@ membership queries at scale: `__words__` for tokenized text…). - `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run segments, stored through the shared `_blobs/.bin` binary convention. +- `_column_index/{field}/k/{kind}/…` — the same two files again, for a + **second value kind** on the same field (see below). Absent for a field that + holds one kind, which is nearly all of them. + +### One posting column per (field, kind) + +A field is not obliged to hold one type of value. `category` may carry +`'electronics'` on some rows and `5` on others, and both are real values of +that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean +— so a field that holds several kinds gets **one column per kind**: + +- The first kind a field ever sees owns the plain `_column_index/{field}/` + layout above. A single-kind field is therefore byte-identical to what earlier + versions wrote, and an index written before typed postings opens unchanged. +- Every later kind gets its own column beside it at + `_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or + `boolean`. + +What that buys at query time: + +| | | +|---|---| +| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. | +| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. | +| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. | +| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. | +| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. | + +`null` and `undefined` are not kinds and are never posted; their absence is +what the `exists` / `missing` operators read. + +Older readers are unaffected by the additional columns: they see the field's +primary column exactly where it has always been, and a `k/{kind}` directory is +simply a name they never query. Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments additionally live as bucketed keys under `_system/idx/` (see §3). Which path diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 48f4a963..6bff86d4 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -23,7 +23,10 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js' import { ValueType, DEFAULT_FLUSH_THRESHOLD, - FLAG_MULTI_VALUE + FLAG_MULTI_VALUE, + POSTING_KINDS, + KIND_PATH_SEGMENT, + type PostingKind } from './types.js' import { ColumnTailBuffer } from './ColumnTailBuffer.js' import { ColumnManifest } from './ColumnManifest.js' @@ -52,10 +55,89 @@ interface HeapEntry { value: number | string entityIntId: number cursorIndex: number + /** + * Rank of the posting kind this entry came from, from {@link POSTING_KINDS}. + * A mixed-kind field has no natural total order, so the merge orders by kind + * first and by value within a kind. + */ + kindRank: number /** Iterator for the cursor — call next() to advance */ iterator: Generator } +/** + * One physical posting column: a (field, kind) pair and the key every internal + * map and every storage path uses for it. + */ +interface KindColumn { + /** The field as the query language names it. */ + field: string + /** The kind of value this column holds. */ + kind: PostingKind + /** + * Internal map / storage key. The field's PRIMARY kind uses the bare field + * name — the historical layout — and every other kind uses + * `//`. + */ + key: string +} + +/** + * The KIND a value indexes under — its JavaScript `typeof` class, not its + * storage encoding. + * + * Anything that is not a number, string or boolean indexes as a string, which + * is the `String(value)` treatment those values already received. `null` and + * `undefined` never reach here: `addEntity` skips them, and their absence is + * what the `exists` / `missing` operators read. + * + * @param value - The value about to be indexed or queried + * @returns The posting kind that owns this value + */ +function kindOfValue(value: unknown): PostingKind { + const t = typeof value + if (t === 'number') return 'number' + if (t === 'boolean') return 'boolean' + return 'string' +} + +/** + * The segment encoding a fresh column of this kind starts with. + * + * Only the number kind has a choice: an integer column starts as i64 and + * widens to f64 the first time a non-integer arrives + * ({@link ColumnTailBuffer.promoteToFloat}). + */ +function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType { + switch (kind) { + case 'boolean': + return ValueType.Boolean + case 'string': + return ValueType.String + case 'number': + return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float + } +} + +/** + * The kind a column of this encoding holds — the inverse of + * {@link initialValueTypeFor}, used to read a kind back off a manifest written + * before typed postings existed. + */ +function kindOfValueType(valueType: ValueType): PostingKind { + switch (valueType) { + case ValueType.Boolean: + return 'boolean' + case ValueType.String: + return 'string' + case ValueType.Number: + case ValueType.Float: + return 'number' + default: + throw new Error(`Unknown ValueType: ${valueType}`) + } +} + /** * Unified column store coordinator. * @@ -121,9 +203,19 @@ export class ColumnStore implements ColumnStoreProvider { */ private deletedEntities: Map = new Map() - /** Known field value types (inferred from first write). */ + /** Segment encoding per COLUMN key (not per field — a field has one per kind). */ private fieldTypes: Map = new Map() + /** + * Every posting column a field owns: field → kind → column key. + * + * This is the map that ends the first-writer type freeze. A field's first + * kind takes the bare field name as its column key, keeping the historical + * on-disk layout; each later kind takes its own column beside it. Nothing is + * coerced across kinds and nothing is dropped for being the wrong type. + */ + private fieldColumns: Map> = new Map() + /** Whether init() has completed. */ private initialized = false @@ -140,6 +232,128 @@ export class ColumnStore implements ColumnStoreProvider { this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4 } + // ========================================================================= + // Posting columns: (field, kind) → one physical column + // ========================================================================= + + /** + * Storage / map key for a (field, kind) column. + * + * `primary` is the kind that owns the bare field name. It is whichever kind + * the field saw first, which for an index written before typed postings is + * simply the kind of its single manifest — so the historical layout is + * preserved rather than migrated. + */ + private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string { + return primary === null || kind === primary + ? field + : `${field}/${KIND_PATH_SEGMENT}/${kind}` + } + + /** + * Split a discovered manifest path back into its (field, kind) column, or + * `null` when the path names a field's primary column rather than a kind + * column. `/k/` is the only shape that reads as a kind column, + * and only for a `` this version knows. + */ + private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null { + const marker = `/${KIND_PATH_SEGMENT}/` + const at = key.lastIndexOf(marker) + if (at <= 0) return null + const kind = key.slice(at + marker.length) + if (!POSTING_KINDS.includes(kind as PostingKind)) return null + return { field: key.slice(0, at), kind: kind as PostingKind } + } + + /** Record a discovered or freshly created column against its field. */ + private registerColumn(field: string, kind: PostingKind, key: string): void { + let byKind = this.fieldColumns.get(field) + if (!byKind) { + byKind = new Map() + this.fieldColumns.set(field, byKind) + } + const existing = byKind.get(kind) + if (existing !== undefined && existing !== key) { + // Two columns claiming one (field, kind) means the layout on disk is not + // one this writer could have produced. Serving it would silently answer + // from half the postings, so say which two and stop. + throw new Error( + `ColumnStore: field '${field}' has two '${kind}' posting columns on ` + + `disk ('${existing}' and '${key}'). The column index layout is ` + + `inconsistent — rebuild/repair the metadata index rather than ` + + `serving from one half of it.` + ) + } + byKind.set(kind, key) + } + + /** The column key for this (field, kind), or `null` if the field has no such kind. */ + private columnKey(field: string, kind: PostingKind): string | null { + return this.fieldColumns.get(field)?.get(kind) ?? null + } + + /** + * The column key for this (field, kind), creating the registration if the + * field has not seen this kind before. Write path only. + */ + private ensureColumnKey(field: string, kind: PostingKind): string { + const byKind = this.fieldColumns.get(field) + const existing = byKind?.get(kind) + if (existing !== undefined) return existing + + // The primary kind is the one already holding the bare field name, if any. + let primary: PostingKind | null = null + if (byKind) { + for (const [k, key] of byKind) { + if (key === field) { primary = k; break } + } + } + const key = ColumnStore.columnKeyFor(field, kind, primary) + this.registerColumn(field, kind, key) + return key + } + + /** + * Every posting column this field owns, in {@link POSTING_KINDS} order. + * + * Read doors that are not about one particular value — an unbounded range + * used as an "any value present" probe, distinct values, sorting — fan out + * over all of them. + */ + private columnsForField(field: string): KindColumn[] { + const byKind = this.fieldColumns.get(field) + if (!byKind) return [] + const out: KindColumn[] = [] + for (const kind of POSTING_KINDS) { + const key = byKind.get(kind) + if (key !== undefined) out.push({ field, kind, key }) + } + return out + } + + /** + * Which value kinds this field actually holds, in {@link POSTING_KINDS} + * order — the honest answer to "what type is this field?". + * + * A field that carries both `'electronics'` and `5` reports + * `['number', 'string']`, not whichever of them was written first. + * + * @param field - Field name + * @returns Every kind with at least one posting, or `[]` for an unknown field + */ + getFieldKinds(field: string): PostingKind[] { + return this.columnsForField(field) + .filter((c) => this.columnHasData(c.key)) + .map((c) => c.kind) + } + + /** Does this physical column hold any postings (persisted or buffered)? */ + private columnHasData(key: string): boolean { + const manifest = this.manifests.get(key) + const buffer = this.tailBuffers.get(key) + return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0) + } + /** * Initialize the column store: discover existing field manifests. */ @@ -157,11 +371,23 @@ export class ColumnStore implements ColumnStoreProvider { }).listObjectsUnderPath(this.basePath + '/') for (const path of paths) { if (path.endsWith('/MANIFEST.json')) { - const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '') - const manifest = new ColumnManifest(fieldName, this.basePath) + // The discovered name is a COLUMN key: either a bare field (that + // field's primary kind, which is every column an index written + // before typed postings has) or `/k/` for a second + // kind that arrived on a field later. + const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '') + const manifest = new ColumnManifest(columnKey, this.basePath) await manifest.load(storage) - this.manifests.set(fieldName, manifest) - this.fieldTypes.set(fieldName, manifest.valueType) + this.manifests.set(columnKey, manifest) + this.fieldTypes.set(columnKey, manifest.valueType) + + const parsed = ColumnStore.parseKindColumnKey(columnKey) + if (parsed) { + this.registerColumn(parsed.field, parsed.kind, columnKey) + } else { + this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey) + } + const fieldName = columnKey // Load global deleted bitmap if it exists. Raw blob preferred // (2.4.0 #4 cortex-shared format); legacy envelope fallback for @@ -264,26 +490,43 @@ export class ColumnStore implements ColumnStoreProvider { /** * Point filter: find entities where field equals value. * - * Searches all segments + tail buffer, returns union as roaring bitmap. - * Excludes globally deleted entities. + * The QUERY VALUE'S OWN KIND picks the posting column, and only that column + * is read. `where {category: 5}` answers from the number postings and + * `where {category: '5'}` from the string postings — neither borrows the + * other's rows, because a row written with the number `5` is not a row whose + * category is the text `'5'`. + * + * A field that has never seen this kind matches nothing, which is the true + * answer rather than a coerced one. + * + * Searches all segments + tail buffer of that column, returns the union as a + * roaring bitmap. Excludes globally deleted entities. */ async filter(field: string, value: unknown): Promise { const result = new RoaringBitmap32() - const deleted = this.deletedEntities.get(field) + const columnKey = this.columnKey(field, kindOfValue(value)) + if (columnKey === null) return result + + // The query value takes the column's encoding — a boolean queried against + // a boolean column has to become the 1/0 the column stores. + const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String) + if (encoded === undefined) return result + + const deleted = this.deletedEntities.get(columnKey) // Search segments - const cursors = await this.getSegmentCursors(field) + const cursors = await this.getSegmentCursors(columnKey) for (const cursor of cursors) { - const ids = cursor.getEntityIdsForValue(value as number | string) + const ids = cursor.getEntityIdsForValue(encoded) for (const id of ids) { if (!deleted || !deleted.has(id)) result.add(id) } } // Search tail buffer - const tailCursor = this.getTailBufferCursor(field) + const tailCursor = this.getTailBufferCursor(columnKey) if (tailCursor) { - const ids = tailCursor.getEntityIdsForValue(value as number | string) + const ids = tailCursor.getEntityIdsForValue(encoded) for (const id of ids) { if (!deleted || !deleted.has(id)) result.add(id) } @@ -324,22 +567,26 @@ export class ColumnStore implements ColumnStoreProvider { const out = new Map() if (wanted.size === 0 || !this.hasField(field)) return out - const deleted = this.deletedEntities.get(field) - const take = (entry: { value: number | string; entityIntId: number }): void => { - if (!wanted.has(entry.entityIntId)) return - if (deleted && deleted.has(entry.entityIntId)) return - out.set(entry.entityIntId, entry.value) - } + // Every kind the field holds is read, in POSTING_KINDS order — a value an + // entity wrote as a string is still that entity's value for this field. + for (const column of this.columnsForField(field)) { + const deleted = this.deletedEntities.get(column.key) + const take = (entry: { value: number | string; entityIntId: number }): void => { + if (!wanted.has(entry.entityIntId)) return + if (deleted && deleted.has(entry.entityIntId)) return + out.set(entry.entityIntId, entry.value) + } - // Segments oldest -> newest, then the tail: a later write overwrites an - // earlier one for the same id. - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) take(entry) - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) take(entry) + // Segments oldest -> newest, then the tail: a later write overwrites an + // earlier one for the same id. + const cursors = await this.getSegmentCursors(column.key) + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) take(entry) + } + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) take(entry) + } } return out } @@ -363,41 +610,59 @@ export class ColumnStore implements ColumnStoreProvider { includeMax: boolean = true ): Promise { const result = new RoaringBitmap32() - const cursors = await this.getSegmentCursors(field) const hasMin = min !== undefined && min !== null const hasMax = max !== undefined && max !== null - for (const cursor of cursors) { - const lo = hasMin ? min as number | string : cursor.minValue - const hi = hasMax ? max as number | string : cursor.maxValue - if (lo === undefined || hi === undefined) continue - // Exclusivity applies only to an explicitly provided bound. A bound taken - // from the segment's own min/max is a real stored value and must stay - // inclusive, or the segment's boundary entities would be wrongly dropped. - const ids = cursor.getEntityIdsInRange( - lo, - hi, - hasMin ? includeMin : true, - hasMax ? includeMax : true - ) - for (const id of ids) result.add(id) - } + // The BOUNDS pick the column: numeric bounds read the numeric postings, + // string bounds the string postings. An unbounded call is not a range at + // all — it is the "has any value here" probe behind `exists` — so it fans + // out over every kind the field holds. + const columns: KindColumn[] = hasMin + ? this.columnsForKind(field, kindOfValue(min)) + : hasMax + ? this.columnsForKind(field, kindOfValue(max)) + : this.columnsForField(field) - // Tail buffer range: linear scan (tail is small) - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) { - const v = entry.value as any - const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any)) - const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any)) - if (loOk && hiOk) result.add(entry.entityIntId) + for (const column of columns) { + const cursors = await this.getSegmentCursors(column.key) + for (const cursor of cursors) { + const lo = hasMin ? min as number | string : cursor.minValue + const hi = hasMax ? max as number | string : cursor.maxValue + if (lo === undefined || hi === undefined) continue + // Exclusivity applies only to an explicitly provided bound. A bound taken + // from the segment's own min/max is a real stored value and must stay + // inclusive, or the segment's boundary entities would be wrongly dropped. + const ids = cursor.getEntityIdsInRange( + lo, + hi, + hasMin ? includeMin : true, + hasMax ? includeMax : true + ) + for (const id of ids) result.add(id) + } + + // Tail buffer range: linear scan (tail is small) + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) { + const v = entry.value as any + const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any)) + const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any)) + if (loOk && hiOk) result.add(entry.entityIntId) + } } } return result } + /** The single column for this (field, kind), as a list, or empty if absent. */ + private columnsForKind(field: string, kind: PostingKind): KindColumn[] { + const key = this.columnKey(field, kind) + return key === null ? [] : [{ field, kind, key }] + } + /** * Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt). * @@ -428,18 +693,21 @@ export class ColumnStore implements ColumnStoreProvider { */ async getFilterValues(field: string): Promise { const valueSet = new Set() - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) { - valueSet.add(String(entry.value)) + for (const column of this.columnsForField(field)) { + const cursors = await this.getSegmentCursors(column.key) + + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) { + valueSet.add(String(entry.value)) + } } - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) { - valueSet.add(String(entry.value)) + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) { + valueSet.add(String(entry.value)) + } } } @@ -450,9 +718,7 @@ export class ColumnStore implements ColumnStoreProvider { * Check if a field has any indexed data. */ hasField(field: string): boolean { - const manifest = this.manifests.get(field) - const buffer = this.tailBuffers.get(field) - return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0) + return this.columnsForField(field).some((c) => this.columnHasData(c.key)) } /** @@ -462,12 +728,11 @@ export class ColumnStore implements ColumnStoreProvider { * store will actually serve queries from. */ getIndexedFields(): string[] { + // Names FIELDS, not columns: a field carrying two kinds is one name here, + // the same name a caller queries with. const fields = new Set() - for (const [field, manifest] of this.manifests) { - if (!manifest.isEmpty()) fields.add(field) - } - for (const [field, buffer] of this.tailBuffers) { - if (buffer.size > 0) fields.add(field) + for (const [field] of this.fieldColumns) { + if (this.hasField(field)) fields.add(field) } return Array.from(fields).sort() } @@ -482,12 +747,16 @@ export class ColumnStore implements ColumnStoreProvider { getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> { const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = [] for (const field of this.getIndexedFields()) { - const manifest = this.manifests.get(field) - const buffer = this.tailBuffers.get(field) - const segmentCount = manifest && !manifest.isEmpty() - ? manifest.getAllSegments().length - : 0 - const tailSize = buffer ? buffer.size : 0 + // Summed across the field's kind columns — the caller asked about a + // field, and a field's size is all of the postings under its name. + let segmentCount = 0 + let tailSize = 0 + for (const column of this.columnsForField(field)) { + const manifest = this.manifests.get(column.key) + const buffer = this.tailBuffers.get(column.key) + if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length + if (buffer) tailSize += buffer.size + } summary.push({ field, segmentCount, tailSize }) } return summary @@ -515,6 +784,8 @@ export class ColumnStore implements ColumnStoreProvider { this.segmentCache.clear() this.manifests.clear() this.deletedEntities.clear() + this.fieldColumns.clear() + this.fieldTypes.clear() this.initialized = false } @@ -523,32 +794,64 @@ export class ColumnStore implements ColumnStoreProvider { // ========================================================================= /** - * Push a single value to a field's tail buffer. - * Creates the buffer and manifest if first write to this field. - * Infers ValueType from the first value seen. + * Push a single value to the posting column for its (field, KIND). + * + * The value's own kind picks the column — a string goes to the field's + * string postings, a number to its number postings — so a field carrying + * `'electronics'` and `5` keeps both, each answerable by an equality filter + * of its own kind. Under the first-writer type freeze this method replaced, + * the first value's type became the field's type and every later value of + * another kind was coerced to it or, when coercion failed, dropped with no + * error at all. + * + * Creates the column's buffer and manifest on its first value. */ private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void { - let buffer = this.tailBuffers.get(field) + const kind = kindOfValue(value) + const columnKey = this.ensureColumnKey(field, kind) + + let buffer = this.tailBuffers.get(columnKey) if (!buffer) { - const valueType = this.inferValueType(value) - buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold) - this.tailBuffers.set(field, buffer) - this.fieldTypes.set(field, valueType) + // A reopened column takes its encoding from its manifest — an integer + // column that widened to f64 in an earlier session stays widened. + const valueType = + this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value) + buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold) + this.tailBuffers.set(columnKey, buffer) + this.fieldTypes.set(columnKey, valueType) // Ensure manifest exists - if (!this.manifests.has(field)) { - const manifest = new ColumnManifest(field, this.basePath) + if (!this.manifests.has(columnKey)) { + const manifest = new ColumnManifest(columnKey, this.basePath) manifest.valueType = valueType manifest.multiValue = isMultiValue - this.manifests.set(field, manifest) + this.manifests.set(columnKey, manifest) } } - // Normalize value to the column type - const normalizedValue = this.normalizeValue(value, buffer.valueType) - if (normalizedValue !== undefined) { - buffer.add(normalizedValue, entityIntId) + // An integer column widens the first time a non-integer number arrives, so + // the value is stored as itself instead of rounded to the nearest integer. + if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) { + buffer.promoteToFloat() + this.fieldTypes.set(columnKey, ValueType.Float) + const manifest = this.manifests.get(columnKey) + if (manifest) manifest.valueType = ValueType.Float } + + const normalizedValue = this.normalizeValue(value, buffer.valueType) + if (normalizedValue === undefined) { + // Unreachable by construction: the column was chosen BY this value's + // kind, so the encoding always accepts it. Reaching here would mean a + // value had been silently dropped from the index — the exact failure + // typed postings exist to end — so it is an error, never a skip. + throw new Error( + `ColumnStore: field '${field}' rejected a ${kind} value for its own ` + + `${ValueType[buffer.valueType]} posting column. The value would have ` + + `been dropped from the index while the row stayed readable by id — ` + + `this is a kind-routing bug, not a value the caller may ignore.` + ) + } + buffer.add(normalizedValue, entityIntId) } /** @@ -677,8 +980,15 @@ export class ColumnStore implements ColumnStoreProvider { /** Torn-segment quarantine entries for a field (observability + heal input). */ quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> { const out: Array<{ segment: string; error: string; hits: number }> = [] - for (const [key, q] of this.segmentQuarantine) { - if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits }) + // Across every kind column of the field — a torn segment in the string + // postings is this field's torn segment as much as one in the numbers. + for (const column of this.columnsForField(field)) { + const prefix = `${column.key}:` + for (const [key, q] of this.segmentQuarantine) { + if (key.startsWith(prefix)) { + out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits }) + } + } } return out } @@ -850,17 +1160,22 @@ export class ColumnStore implements ColumnStoreProvider { k: number, filterBitmap: RoaringBitmap32 | null ): Promise { - // Collect all cursors (segments + tail buffer) - const segCursors = await this.getSegmentCursors(field) - const tailCursor = this.getTailBufferCursor(field) - - // Create iterators for each cursor in the specified direction + // Collect cursors across EVERY kind the field holds. A single-kind field — + // nearly all of them — merges exactly the cursors it always did. const iterators: Generator[] = [] - for (const cursor of segCursors) { - iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward()) - } - if (tailCursor) { - iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward()) + const iteratorKindRank: number[] = [] + for (const column of this.columnsForField(field)) { + const kindRank = POSTING_KINDS.indexOf(column.kind) + const segCursors = await this.getSegmentCursors(column.key) + for (const cursor of segCursors) { + iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward()) + iteratorKindRank.push(kindRank) + } + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward()) + iteratorKindRank.push(kindRank) + } } if (iterators.length === 0) return [] @@ -874,16 +1189,21 @@ export class ColumnStore implements ColumnStoreProvider { value: next.value.value, entityIntId: next.value.entityIntId, cursorIndex: i, + kindRank: iteratorKindRank[i], iterator: iterators[i] }) } } - // Heapify - const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String + // Heapify. A number and a string have no ordering between them, so a + // mixed-kind field orders by KIND first (POSTING_KINDS order) and by value + // within a kind — one defined total order instead of a comparison whose + // answer depends on which value happened to be on the left. const compare = (a: HeapEntry, b: HeapEntry): number => { let cmp: number - if (isString) { + if (a.kindRank !== b.kindRank) { + cmp = a.kindRank - b.kindRank + } else if (POSTING_KINDS[a.kindRank] === 'string') { cmp = compareCodePoints(String(a.value), String(b.value)) } else { cmp = (a.value as number) - (b.value as number) @@ -915,6 +1235,7 @@ export class ColumnStore implements ColumnStoreProvider { value: next.value.value, entityIntId: next.value.entityIntId, cursorIndex: top.cursorIndex, + kindRank: top.kindRank, iterator: top.iterator } } @@ -922,8 +1243,11 @@ export class ColumnStore implements ColumnStoreProvider { this.heapDown(heap, 0, compare) } - // Apply global deleted check, filter, and dedup - const deleted = this.deletedEntities.get(field) + // Apply global deleted check, filter, and dedup. The deleted bitmap is + // per COLUMN, and the entry came from the column its kind names. + const deleted = this.deletedEntities.get( + this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field + ) if (deleted && deleted.has(top.entityIntId)) continue if (seen.has(top.entityIntId)) continue if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue @@ -965,35 +1289,31 @@ export class ColumnStore implements ColumnStoreProvider { } /** - * Infer ValueType from a JavaScript value. - */ - private inferValueType(value: unknown): ValueType { - if (typeof value === 'boolean') return ValueType.Boolean - if (typeof value === 'number') { - return Number.isInteger(value) ? ValueType.Number : ValueType.Float - } - return ValueType.String - } - - /** - * Normalize a JavaScript value to the column's ValueType. + * Encode a value for the column its own kind selected. + * + * This does NOT convert between kinds. It used to: a string reaching a + * numeric column was run through `Number(value)`, and a number reaching a + * numeric column was run through `Math.round`, so `'electronics'` became + * `NaN` and vanished while `4.5` became `5` and answered the wrong query. + * Kind routing removes the need for either — the only work left is picking + * the encoding the column already committed to. + * + * @returns The encoded value, or `undefined` if the value does not belong in + * this column at all — which the caller treats as a routing bug and + * raises, never as a value to skip. */ private normalizeValue(value: unknown, type: ValueType): number | string | undefined { switch (type) { case ValueType.Number: - if (typeof value === 'number') return Math.round(value) - if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) } - if (typeof value === 'boolean') return value ? 1 : 0 - return undefined + // Integer column. Non-integers widen it to Float before reaching here. + return typeof value === 'number' && Number.isInteger(value) ? value : undefined case ValueType.Float: - if (typeof value === 'number') return value - if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n } - return undefined + return typeof value === 'number' ? value : undefined case ValueType.Boolean: - if (typeof value === 'boolean') return value ? 1 : 0 - if (typeof value === 'number') return value ? 1 : 0 - return undefined + return typeof value === 'boolean' ? (value ? 1 : 0) : undefined case ValueType.String: + // The string kind is also where objects and bigints land, exactly as + // they always did. return String(value) default: return undefined diff --git a/src/indexes/columnStore/ColumnTailBuffer.ts b/src/indexes/columnStore/ColumnTailBuffer.ts index c5874ac2..e730f884 100644 --- a/src/indexes/columnStore/ColumnTailBuffer.ts +++ b/src/indexes/columnStore/ColumnTailBuffer.ts @@ -55,8 +55,12 @@ export class ColumnTailBuffer { /** Field name this buffer is for. */ readonly fieldName: string - /** Value type determines sort comparator. */ - readonly valueType: ValueType + /** + * Value type determines sort comparator and segment encoding. + * + * Widened in place by {@link promoteToFloat} — never otherwise reassigned. + */ + valueType: ValueType /** Flush threshold. */ readonly threshold: number @@ -81,6 +85,38 @@ export class ColumnTailBuffer { this.threshold = threshold } + /** + * Widen an integer column to floating point, losslessly and in place. + * + * The number posting kind holds every JavaScript number, but a segment picks + * ONE encoding: i64 for integers, f64 for the rest. A column that has only + * ever seen integers is written as i64; the first non-integer to arrive + * widens it here, so that value is stored as itself instead of being rounded + * to the nearest integer with no error — the rounding that made `4.5` and + * `5.5` both answer `where {score: 5}` and neither answer its own value. + * + * Widening is lossless in both directions it has to be: every value already + * buffered is an integer, and every integer is exactly representable as f64. + * Segments already on disk keep their own i64 encoding in their own headers + * and keep decoding by it — only segments written from here on are f64. + * + * @throws Error if called on a column that is not an integer column — the + * only legal widening is Number → Float, and any other request is a bug in + * the caller's kind routing rather than something to absorb quietly. + */ + promoteToFloat(): void { + if (this.valueType === ValueType.Float) return + if (this.valueType !== ValueType.Number) { + throw new Error( + `ColumnTailBuffer '${this.fieldName}': cannot widen a ` + + `${ValueType[this.valueType]} column to Float — only an integer ` + + `(Number) column widens, and this call means a value reached the ` + + `wrong kind's column` + ) + } + this.valueType = ValueType.Float + } + /** * Add a (value, entityIntId) entry to the buffer. * diff --git a/src/indexes/columnStore/types.ts b/src/indexes/columnStore/types.ts index 71dd99a0..ee949bd0 100644 --- a/src/indexes/columnStore/types.ts +++ b/src/indexes/columnStore/types.ts @@ -58,6 +58,53 @@ export enum ValueType { Boolean = 3 } +/** + * The KIND of a value, as the query language sees it. + * + * A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5` + * are one kind (`'number'`) held in one posting column, even though they need + * different segment encodings (i64 vs f64 — see {@link ValueType}). + * + * A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string + * values and number values at the same time and answer equality on each. This + * replaces the first-writer type freeze, under which the first value's type + * became the field's type and every later value of another kind was coerced — + * or, when coercion failed (`Number('electronics')`), dropped from the index + * with no error: the row stayed readable by id and by vector but vanished from + * every equality filter on that field. + * + * Kinds do not coerce into one another at query time either: `where {c: 5}` + * matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows + * written with the STRING `'5'`. Neither ever matches the other. + * + * Values that are none of these three (objects, bigints) index as strings — + * the same `String(value)` treatment they received before. + */ +export type PostingKind = 'number' | 'string' | 'boolean' + +/** + * Every posting kind, in the order that defines cross-kind sort position. + * + * A mixed-kind field has no natural total order — a number does not compare + * with a string — so `sortTopK` orders by KIND first (numbers, then strings, + * then booleans) and by value within a kind. A single-kind field, which is + * nearly every field, sorts exactly as it always did. + */ +export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean'] + +/** + * Path segment marking a field's NON-PRIMARY kind columns on disk. + * + * The first kind a field ever sees keeps the historical layout — + * `//MANIFEST.json` and `//L0-NNNNNN` — so every + * index written before typed postings opens unchanged, and the byte-for-byte + * interchange with the native column store is untouched for the single-kind + * fields that are nearly all of them. A second kind arriving on the same field + * gets its own column at `//k//…` rather than overwriting or + * being coerced into the first. + */ +export const KIND_PATH_SEGMENT = 'k' + // --------------------------------------------------------------------------- // Segment header and footer // --------------------------------------------------------------------------- @@ -267,6 +314,19 @@ export interface ColumnStoreProvider { */ hasField(field: string): boolean + /** + * Which value KINDS this field actually holds, in {@link POSTING_KINDS} + * order — the honest answer to "what type is this field?" for a field that + * carries more than one. + * + * OPTIONAL so an implementation written against the pre-typed-postings + * contract still satisfies this interface; feature-detect before calling. + * + * @param field - Field name + * @returns Every kind with at least one posting, or `[]` for an unknown field + */ + getFieldKinds?(field: string): PostingKind[] + /** * Flush all in-memory tail buffers to L0 segments on disk. * Saves all manifests. diff --git a/src/utils/fieldTypeInference.ts b/src/utils/fieldTypeInference.ts index 36a415b2..0f085f8c 100644 --- a/src/utils/fieldTypeInference.ts +++ b/src/utils/fieldTypeInference.ts @@ -55,8 +55,30 @@ export enum FieldType { */ export interface FieldTypeInfo { field: string + /** + * The DOMINANT reading of the field — one type, the most specific one every + * sampled value satisfies. + * + * A field is not obliged to hold one kind, so this is not the whole answer + * for a field that holds several. Read {@link kinds} beside it: a field + * carrying `'electronics'` and `5` infers as STRING here and reports + * `['number', 'string']` there, and the metadata index keeps a separate + * posting column for each of them. + */ inferredType: FieldType confidence: number // 0-1 confidence score + /** + * Every value KIND observed in the sample, in the order + * number → string → boolean. More than one entry means a genuinely + * mixed field, and every one of those kinds is independently filterable. + * + * Kinds are JavaScript `typeof` classes, one level coarser than + * {@link FieldType}: a UUID and a category name are both `'string'`, and an + * integer and a timestamp are both `'number'`. + * + * Optional only for cached analyses written before this was reported. + */ + kinds?: Array<'number' | 'string' | 'boolean'> sampleSize: number // Number of values analyzed lastUpdated: number // Timestamp of last analysis detectionMethod: 'value' // Always 'value' (no fallbacks!) @@ -133,14 +155,71 @@ export class FieldTypeInference { } /** - * Analyze values to determine field type + * Analyze values to determine field type, and report every KIND the field + * actually holds alongside it. + * + * The classification below picks ONE type, because every one of its + * heuristics asks `samples.every(...)`: a field carrying `'electronics'` and + * `5` satisfies none of them and lands on STRING. That single answer is true + * as far as it goes — string is the dominant reading — but on its own it + * says nothing about the numbers also in the field, and a caller that treats + * it as the field's only type reproduces the first-writer freeze the index + * itself no longer has. {@link FieldTypeInfo.kinds} carries the rest. + */ + private async analyzeValues(field: string, values: any[]): Promise { + const info = await this.classifyValues(field, values) + info.kinds = FieldTypeInference.observedKinds(values) + if (info.kinds.length > 1 && info.metadata) { + info.metadata.format = `${info.metadata.format} (field also holds: ${info.kinds + .filter((k) => k !== FieldTypeInference.kindOfType(info.inferredType)) + .join(', ')})` + } + return info + } + + /** + * The distinct value kinds present in a sample, in a stable order. + * + * Kinds are JavaScript `typeof` classes — the same classes the metadata + * index keeps separate posting columns for — not the finer + * {@link FieldType} readings, which are interpretations layered on top of + * them (a UUID and a category name are both the `string` kind). + */ + private static observedKinds(values: any[]): Array<'number' | 'string' | 'boolean'> { + const order: Array<'number' | 'string' | 'boolean'> = ['number', 'string', 'boolean'] + const seen = new Set<'number' | 'string' | 'boolean'>() + for (const v of values) { + if (v === null || v === undefined) continue + const t = typeof v + seen.add(t === 'number' ? 'number' : t === 'boolean' ? 'boolean' : 'string') + } + return order.filter((k) => seen.has(k)) + } + + /** The value kind a {@link FieldType} reading is an interpretation of. */ + private static kindOfType(type: FieldType): 'number' | 'string' | 'boolean' { + switch (type) { + case FieldType.BOOLEAN: + return 'boolean' + case FieldType.INTEGER: + case FieldType.FLOAT: + case FieldType.TIMESTAMP_MS: + case FieldType.TIMESTAMP_S: + return 'number' + default: + return 'string' + } + } + + /** + * Classify values into a single field type. * * Uses DuckDB-inspired type detection order: * BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING * * No fallbacks - pure value-based detection */ - private async analyzeValues(field: string, values: any[]): Promise { + private async classifyValues(field: string, values: any[]): Promise { // Filter null/undefined values const validValues = values.filter(v => v !== null && v !== undefined) diff --git a/tests/regression/metadata-field-typing.unit.test.ts b/tests/regression/metadata-field-typing.unit.test.ts new file mode 100644 index 00000000..910d4f2a --- /dev/null +++ b/tests/regression/metadata-field-typing.unit.test.ts @@ -0,0 +1,122 @@ +/** + * @module metadata-field-typing.unit.test + * @description Regression: a metadata field that holds more than one value + * KIND stays fully filterable on every kind it holds. + * + * The defect this pins, reproduced on the released engine: the metadata index + * fixed a field's value type from the FIRST value it saw, and every later value + * of a different type was coerced to that type or, when coercion failed, + * dropped from the index in silence. Writing `category: 'electronics'` rows and + * then `category: 5` rows left `find({ where: { category: 5 } })` returning + * nothing — while the same rows in a numbers-only corpus answered correctly. + * The rows themselves were never lost: they stayed readable by id and by vector + * search, and only ever went missing from equality filters on that one field, + * which is what made it so quiet. + * + * Order is the whole point of these cases. Neither writer owns the field, so + * strings-then-numbers and numbers-then-strings must give the same answers. + */ + +import { describe, it, expect } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** A brain over memory storage, with a corpus written in the given order. */ +async function brainWith( + rows: Array<{ label: string; category: unknown }> +): Promise { + const brainy = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brainy.init() + for (const row of rows) { + await brainy.add({ + data: `item ${row.label}`, + type: NounType.Thing, + metadata: { label: row.label, category: row.category } + }) + } + return brainy +} + +const labelsOf = (results: Array<{ metadata?: Record }>): string[] => + results.map((r) => String(r.metadata?.label)).sort() + +describe('regression: a mixed-kind metadata field filters on every kind', { timeout: 180_000 }, () => { + it('finds number rows written after string rows', async () => { + const brainy = await brainWith([ + { label: 'e1', category: 'electronics' }, + { label: 'f1', category: 'furniture' }, + { label: 'n1', category: 5 }, + { label: 'n2', category: 5 }, + { label: 'n3', category: 7 } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2']) + expect(labelsOf(await brainy.find({ where: { category: 7 }, limit: 100 }))).toEqual(['n3']) + expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1']) + expect(labelsOf(await brainy.find({ where: { category: 'furniture' }, limit: 100 }))).toEqual(['f1']) + } finally { + await brainy.close() + } + }) + + it('finds string rows written after number rows', async () => { + const brainy = await brainWith([ + { label: 'n1', category: 5 }, + { label: 'n2', category: 5 }, + { label: 'e1', category: 'electronics' }, + { label: 'e2', category: 'electronics' } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1', 'e2']) + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2']) + } finally { + await brainy.close() + } + }) + + it('keeps `5` and `\'5\'` apart — a kind is part of the value, not a formatting detail', async () => { + const brainy = await brainWith([ + { label: 'num', category: 5 }, + { label: 'str', category: '5' } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['num']) + expect(labelsOf(await brainy.find({ where: { category: '5' }, limit: 100 }))).toEqual(['str']) + } finally { + await brainy.close() + } + }) + + it('serves booleans mixed into a field that already holds strings', async () => { + const brainy = await brainWith([ + { label: 's1', category: 'yes' }, + { label: 'b1', category: true }, + { label: 'b2', category: false } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: true }, limit: 100 }))).toEqual(['b1']) + expect(labelsOf(await brainy.find({ where: { category: false }, limit: 100 }))).toEqual(['b2']) + expect(labelsOf(await brainy.find({ where: { category: 'yes' }, limit: 100 }))).toEqual(['s1']) + } finally { + await brainy.close() + } + }) + + it('ranges over the numeric part of a mixed field', async () => { + const brainy = await brainWith([ + { label: 'unpriced', category: 'on request' }, + { label: 'cheap', category: 100 }, + { label: 'mid', category: 500 }, + { label: 'dear', category: 900 } + ]) + try { + const found = await brainy.find({ + where: { category: { greaterThan: 200 } }, + limit: 100 + }) + expect(labelsOf(found)).toEqual(['dear', 'mid']) + } finally { + await brainy.close() + } + }) +}) diff --git a/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts b/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts new file mode 100644 index 00000000..1ce21d1f --- /dev/null +++ b/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts @@ -0,0 +1,241 @@ +/** + * @module column-store-mixed-kind.test + * @description Typed posting lists: one field, several value KINDS, each + * answerable on its own. + * + * The behaviour these pin replaced a first-writer type freeze. The first value + * a field ever saw fixed that field's type; every later value of another kind + * was coerced to it, and when coercion failed — `Number('electronics')` — the + * value was dropped from the index with no error at all. The row stayed + * readable by id and by vector and vanished from every equality filter on the + * field. These tests therefore care about ORDER: strings-then-numbers and + * numbers-then-strings have to behave identically, because neither writer owns + * the field. + * + * Kinds never coerce into one another at query time either. `5` and `'5'` are + * different values and match different rows. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' +import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' +import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' + +describe('ColumnStore — typed posting lists per (field, kind)', () => { + let storage: MemoryStorage + let idMapper: EntityIdMapper + let store: ColumnStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) + await idMapper.init() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + }) + + afterEach(async () => { + await store.close() + }) + + /** Resolve a filter to the sorted UUIDs it matched. */ + const uuidsOf = async (field: string, value: unknown): Promise => { + const bitmap = await store.filter(field, value) + return Array.from(bitmap) + .map((id) => idMapper.getUuid(Number(id))) + .filter((u): u is string => u !== undefined) + .sort() + } + + describe('equality answers on the query value’s own kind', () => { + it('serves numbers written AFTER strings on the same field', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 }) + + // The numbers are in the index, though a string got there first. + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + expect(await uuidsOf('category', 7)).toEqual(['n3']) + // And the strings did not move. + expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) + expect(await uuidsOf('category', 'furniture')).toEqual(['s2']) + }) + + it('serves strings written AFTER numbers on the same field', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + + // 'electronics' would have become NaN and been dropped under the freeze. + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + }) + + it('does not coerce a number query into the string postings, or back', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' }) + + expect(await uuidsOf('code', 5)).toEqual(['num']) + expect(await uuidsOf('code', '5')).toEqual(['str']) + }) + + it('serves booleans mixed into a field that already holds strings and numbers', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true }) + store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false }) + + expect(await uuidsOf('flag', true)).toEqual(['b1']) + expect(await uuidsOf('flag', false)).toEqual(['b2']) + // `true` stores as 1 internally; that is an encoding, not a value. + expect(await uuidsOf('flag', 1)).toEqual(['n1']) + expect(await uuidsOf('flag', 'yes')).toEqual(['s1']) + }) + + it('answers nothing — not something coerced — for a kind the field never held', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + + expect(await uuidsOf('category', 5)).toEqual([]) + expect(await uuidsOf('category', true)).toEqual([]) + }) + + it('holds every kind across a flush, not just the one in the tail buffer', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + await store.flush() + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + }) + }) + + describe('range filters read the numeric postings', () => { + it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 }) + store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 }) + store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' }) + await store.flush() + + const inRange = await store.rangeQuery('price', 200, 1000) + const uuids = Array.from(inRange) + .map((id) => idMapper.getUuid(Number(id))) + .sort() + expect(uuids).toEqual(['dear', 'mid']) + }) + + it('an unbounded range still reports every kind — it is the “has a value” probe', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 }) + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true }) + await store.flush() + + const anyValue = await store.rangeQuery('mixed') + const uuids = Array.from(anyValue) + .map((id) => idMapper.getUuid(Number(id))) + .sort() + expect(uuids).toEqual(['b1', 'n1', 's1']) + }) + }) + + describe('the index reports what a field actually holds', () => { + it('names every kind present, not the one that got there first', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + expect(store.getFieldKinds('category')).toEqual(['string']) + + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) + expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) + + // And the field is still ONE field by name. + expect(store.getIndexedFields()).toEqual(['category']) + expect(store.hasField('category')).toBe(true) + }) + + it('reports an unknown field as holding nothing', () => { + expect(store.getFieldKinds('never-written')).toEqual([]) + }) + }) + + describe('an integer column widens rather than rounding', () => { + it('keeps a non-integer written after integers as itself', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 }) + await store.flush() + + // 4.5 used to round to 5 and answer `score === 5` alongside c. + expect(await uuidsOf('score', 4.5)).toEqual(['b']) + expect(await uuidsOf('score', 5)).toEqual(['c']) + expect(await uuidsOf('score', 4)).toEqual(['a']) + }) + }) + + describe('close then reopen', () => { + it('keeps every typed posting, on the same storage', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) + store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 }) + await store.flush() + await store.close() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) + expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) + expect(await uuidsOf('category', 5)).toEqual(['n1']) + expect(await uuidsOf('category', true)).toEqual(['b1']) + expect(await uuidsOf('score', 1.5)).toEqual(['f1']) + }) + + it('accepts new values of every kind after the reopen', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + await store.flush() + await store.close() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false }) + await store.flush() + + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + expect(await uuidsOf('category', false)).toEqual(['b1']) + }) + + it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => { + // A single-kind field is byte-identical to what the old writer produced: + // one manifest at `_column_index//MANIFEST.json`, no kind + // subdirectory anywhere. That IS the old on-disk shape, so proving the + // new reader serves it proves an old index still opens. + store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' }) + await store.flush() + + const keys = await (storage as unknown as { + listObjectsUnderPath: (prefix: string) => Promise + }).listObjectsUnderPath('_column_index/') + expect(keys.some((k) => k.includes('/k/'))).toBe(false) + + await store.close() + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + expect(store.getFieldKinds('status')).toEqual(['string']) + expect(await uuidsOf('status', 'active')).toEqual(['a']) + }) + }) +}) From 4c344782a75d686b878f0e3f2522c516efaceb67 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:00 -0700 Subject: [PATCH 07/21] test(hygiene): close every brain the find suite creates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/integration/find-*.test.ts and tests/unit/brainy/find*.test.ts each opened one or more Brainy instances (via beforeAll/beforeEach) and never closed them — the leaked instance's cadence timer stays armed for the rest of the single-forked vitest run and keeps narrating into every later file. find-unified-integration.test.ts was a real bug, not just a missing hook: its afterAll called a no-op TestCleanup().cleanup() (nothing was ever registered with it) and then discarded the brain reference with `brain = null` — the brain was never actually closed. --- tests/integration/find-fields-projection.test.ts | 6 +++++- tests/integration/find-near.test.ts | 6 +++++- tests/integration/find-orderby-every-path.test.ts | 6 +++++- tests/integration/find-planner-door.test.ts | 6 +++++- tests/integration/find-unified-integration.test.ts | 1 + tests/unit/brainy/find-complement-operators.test.ts | 6 +++++- tests/unit/brainy/find-index-integrity-guard.test.ts | 6 +++++- tests/unit/brainy/find.test.ts | 8 ++++++-- 8 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts index 5d339f08..25ee416c 100644 --- a/tests/integration/find-fields-projection.test.ts +++ b/tests/integration/find-fields-projection.test.ts @@ -19,7 +19,7 @@ * index-served (a body field, or a bucketed timestamp), exactly the owing rows * are read and the rest are still served from the index. */ -import { describe, it, expect, beforeAll, vi } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType } from '../../src/types/graphTypes' import { generateTestVector } from '../helpers/test-factory' @@ -59,6 +59,10 @@ describe('find/get({ fields }) — projection', () => { await brain.flush() }) + afterAll(async () => { + await brain.close() + }) + /** Count canonical record reads for one call. */ const countingReads = async (body: () => Promise): Promise<{ out: R; reads: number }> => { const spy = vi.spyOn(brain as any, 'batchGet') diff --git a/tests/integration/find-near.test.ts b/tests/integration/find-near.test.ts index 3fb235c8..b2bf01cd 100644 --- a/tests/integration/find-near.test.ts +++ b/tests/integration/find-near.test.ts @@ -9,7 +9,7 @@ * it). Now the anchor is fetched with its vector, and an anchor without one * refuses by name instead of failing inside the index. */ -import { describe, it, expect, beforeAll } from 'vitest' +import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType } from '../../src/types/graphTypes' import { v5 } from '../../src/universal/uuid' @@ -28,6 +28,10 @@ describe('find({ near }) uses the anchor vector', () => { await brain.add({ id: 'far', data: 'far row', type: NounType.Thing, vector: generateTestVector() }) }) + afterAll(async () => { + await brain.close() + }) + it('returns the anchor\'s neighbours by its own vector', async () => { const results = await brain.find({ near: { id: 'anchor' }, limit: 3 }) expect(results.length).toBeGreaterThan(0) diff --git a/tests/integration/find-orderby-every-path.test.ts b/tests/integration/find-orderby-every-path.test.ts index 7637a79b..e62ec670 100644 --- a/tests/integration/find-orderby-every-path.test.ts +++ b/tests/integration/find-orderby-every-path.test.ts @@ -40,7 +40,7 @@ * the covering is ASSERTED from the leg's own output rather than assumed. This * pin is about ordering, and it says nothing about recall. */ -import { describe, it, expect, beforeAll } from 'vitest' +import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' import { resolveEntityId } from '../../src/utils/idNormalization' @@ -107,6 +107,10 @@ describe('find(): orderBy is the order on every path', () => { } }) + afterAll(async () => { + await brain.close() + }) + it('the fixture: the hybrid candidate set covers the whole filter universe', async () => { const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) expect(universe).toHaveLength(ROWS) diff --git a/tests/integration/find-planner-door.test.ts b/tests/integration/find-planner-door.test.ts index 964b13f9..e5224f6d 100644 --- a/tests/integration/find-planner-door.test.ts +++ b/tests/integration/find-planner-door.test.ts @@ -23,7 +23,7 @@ * against the adjacency before it is believed, so a not-serving graph refuses * loudly instead of answering `[]` as truth. */ -import { describe, it, expect, beforeAll, vi } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' import { generateTestVector } from '../helpers/test-factory' @@ -56,6 +56,10 @@ describe('find(): the optional planner door', () => { } }) + afterAll(async () => { + await brain.close() + }) + /** Install a planner door for one call, then remove it. */ const withDoor = async ( door: (...a: any[]) => Promise, diff --git a/tests/integration/find-unified-integration.test.ts b/tests/integration/find-unified-integration.test.ts index 94053d55..3c4741c2 100644 --- a/tests/integration/find-unified-integration.test.ts +++ b/tests/integration/find-unified-integration.test.ts @@ -48,6 +48,7 @@ describe('Unified Find() Integration Tests', () => { afterAll(async () => { await cleanup.cleanup() + await brain.close() brain = null as any }) diff --git a/tests/unit/brainy/find-complement-operators.test.ts b/tests/unit/brainy/find-complement-operators.test.ts index 76fbb017..710fbbbf 100644 --- a/tests/unit/brainy/find-complement-operators.test.ts +++ b/tests/unit/brainy/find-complement-operators.test.ts @@ -7,7 +7,7 @@ * soft-delete semantic: `field !== value` MUST include entities that have no * such field at all. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' @@ -26,6 +26,10 @@ describe('find() complement operators (ne / exists:false / missing:true)', () => ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } }) }) + afterEach(async () => { + await brain.close() + }) + it('ne returns everything except the matching value — INCLUDING entities without the field', async () => { const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 }) const got = new Set(rows.map((r) => r.id)) diff --git a/tests/unit/brainy/find-index-integrity-guard.test.ts b/tests/unit/brainy/find-index-integrity-guard.test.ts index 30cfdf1b..3e63d790 100644 --- a/tests/unit/brainy/find-index-integrity-guard.test.ts +++ b/tests/unit/brainy/find-index-integrity-guard.test.ts @@ -12,7 +12,7 @@ * returns an id whose record matches NEITHER the type nor the where filter) and * assert the phantom is dropped while the genuine matches survive. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' @@ -48,6 +48,10 @@ describe('find() index-integrity guard (phantom row class)', () => { }) }) + afterEach(async () => { + await brain.close() + }) + it('healthy index: the discriminant query returns only the staff Person', async () => { const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) expect(rows.map((r) => r.id)).toEqual([staffId]) diff --git a/tests/unit/brainy/find.test.ts b/tests/unit/brainy/find.test.ts index 5bead272..59601456 100644 --- a/tests/unit/brainy/find.test.ts +++ b/tests/unit/brainy/find.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { createAddParams } from '../../helpers/test-factory' import { NounType } from '../../../src/types/graphTypes' @@ -12,7 +12,11 @@ describe('Brainy.find()', () => { }) await brain.init() }) - + + afterEach(async () => { + await brain.close() + }) + describe('success paths', () => { it('should find entities by text query', async () => { // Arrange From d6e7453f1f67ec264a1c5bf565246bf11dbf9235 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:04 -0700 Subject: [PATCH 08/21] test(hygiene): close every brain the integration suite creates Each file opened a Brainy in beforeAll/beforeEach (or a single it()) and never closed it. related-verb-array.test.ts and vfs-containment-batched.test.ts were real bugs: their afterAll discarded the brain with `brain = null as any` without ever calling close() first. --- tests/integration/api-parameter-validation.test.ts | 4 ++++ tests/integration/entity-confidence-weight.test.ts | 6 +++++- tests/integration/related-verb-array.test.ts | 1 + tests/integration/relationship-intelligence.test.ts | 3 ++- tests/integration/rev-and-ifabsent.test.ts | 6 +++++- tests/integration/vfs-containment-batched.test.ts | 1 + tests/integration/vfs-debug.test.ts | 8 ++++++-- 7 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/integration/api-parameter-validation.test.ts b/tests/integration/api-parameter-validation.test.ts index 4e25e781..da4aed14 100644 --- a/tests/integration/api-parameter-validation.test.ts +++ b/tests/integration/api-parameter-validation.test.ts @@ -34,6 +34,10 @@ describe('API Parameter Validation', () => { }) }) + afterAll(async () => { + await brain.close() + }) + it('should use "where" parameter for metadata filtering', async () => { const results = await brain.find({ where: { category: 'test-category' }, diff --git a/tests/integration/entity-confidence-weight.test.ts b/tests/integration/entity-confidence-weight.test.ts index b5bb34c5..031d29f1 100644 --- a/tests/integration/entity-confidence-weight.test.ts +++ b/tests/integration/entity-confidence-weight.test.ts @@ -7,7 +7,7 @@ * - Backward compatibility preserved */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' @@ -19,6 +19,10 @@ describe('Entity Confidence & Weight Exposure', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('Entity interface', () => { it('should expose confidence when adding entity with confidence', async () => { const id = await brain.add({ diff --git a/tests/integration/related-verb-array.test.ts b/tests/integration/related-verb-array.test.ts index 36a49850..7ed1bd3f 100644 --- a/tests/integration/related-verb-array.test.ts +++ b/tests/integration/related-verb-array.test.ts @@ -30,6 +30,7 @@ describe('related() with a verb-type array returns every requested type', () => }) afterAll(async () => { + await brain.close() brain = null as any }) diff --git a/tests/integration/relationship-intelligence.test.ts b/tests/integration/relationship-intelligence.test.ts index b6e11cb5..c18057fb 100644 --- a/tests/integration/relationship-intelligence.test.ts +++ b/tests/integration/relationship-intelligence.test.ts @@ -59,7 +59,8 @@ describe('Relationship Intelligence', () => { await brain.init() }) - afterEach(() => { + afterEach(async () => { + await brain.close() if (fs.existsSync(testDir)) { fs.rmSync(testDir, { recursive: true }) } diff --git a/tests/integration/rev-and-ifabsent.test.ts b/tests/integration/rev-and-ifabsent.test.ts index 64b184a3..3bff59f1 100644 --- a/tests/integration/rev-and-ifabsent.test.ts +++ b/tests/integration/rev-and-ifabsent.test.ts @@ -9,7 +9,7 @@ * - addMany({ ifAbsent: true }) applies the flag to every item */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { RevisionConflictError } from '../../src/transaction/RevisionConflictError.js' import { NounType } from '../../src/types/graphTypes.js' @@ -22,6 +22,10 @@ describe('7.31.0 — _rev CAS + ifAbsent', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('_rev initialization + surface', () => { it('initializes _rev to 1 on add()', async () => { const id = await brain.add({ data: 'hello', type: NounType.Document }) diff --git a/tests/integration/vfs-containment-batched.test.ts b/tests/integration/vfs-containment-batched.test.ts index 0a7919bf..7bbad478 100644 --- a/tests/integration/vfs-containment-batched.test.ts +++ b/tests/integration/vfs-containment-batched.test.ts @@ -81,6 +81,7 @@ describe('repairContainment: batched pass 2', () => { }) afterAll(async () => { + await brain.close() brain = null as any }) diff --git a/tests/integration/vfs-debug.test.ts b/tests/integration/vfs-debug.test.ts index 7e781139..5eeb0ef5 100644 --- a/tests/integration/vfs-debug.test.ts +++ b/tests/integration/vfs-debug.test.ts @@ -9,9 +9,10 @@ import * as XLSX from 'xlsx' describe('VFS Debug', () => { it('minimal VFS writeFile test', async () => { const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() + try { + await brain.init() - console.log('✅ Brain initialized') + console.log('✅ Brain initialized') // Get VFS and initialize const vfs = brain.vfs @@ -77,5 +78,8 @@ describe('VFS Debug', () => { // THE REAL TEST: Can we query VFS? expect(children.length).toBeGreaterThan(0) expect(rootContents.length).toBeGreaterThan(0) + } finally { + await brain.close() + } }) }) From de79d6b5a4ca41701c23a8b0b235a45029737780 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:10 -0700 Subject: [PATCH 09/21] test(hygiene): close every brain the unit suite creates Each file opened one or more Brainy instances (beforeEach, or a small per-test helper like migration-gate-family-scoped's module-level seed()) and never closed them. migration-gate-family-scoped.test.ts now tracks every brain seed() hands back in a describe-scoped array drained by afterEach, since the helper itself lives outside the describe block. --- tests/unit/brainy-core.unit.test.ts | 6 +++++- tests/unit/brainy/metadata-provider-contract.test.ts | 6 +++++- .../unit/brainy/migration-gate-family-scoped.test.ts | 12 +++++++++++- .../brainy/relate-duplicate-optimization.test.ts | 2 +- tests/unit/get-index-status-readiness.test.ts | 6 +++++- .../graph/graph-fastpath-honest-readiness.test.ts | 6 +++++- tests/unit/metadata-cold-read-guard.test.ts | 6 +++++- tests/unit/migration-lock.test.ts | 11 ++++++++++- tests/unit/neural/signals/EmbeddingSignal.test.ts | 3 ++- .../storage/pagination-parallel-hydration.test.ts | 6 +++++- tests/unit/type-filtering.unit.test.ts | 6 +++++- tests/unit/utils/metadataIndex-array-bound.test.ts | 4 ++++ .../metadataIndex-sparse-range-collation.test.ts | 6 +++++- tests/unit/validate-invariants-delegation.test.ts | 6 +++++- tests/unit/vector-cold-read-guard.test.ts | 6 +++++- tests/unit/vfs-multi-instance-diagnostic.test.ts | 6 +++++- 16 files changed, 83 insertions(+), 15 deletions(-) diff --git a/tests/unit/brainy-core.unit.test.ts b/tests/unit/brainy-core.unit.test.ts index eb6614e4..0488057d 100644 --- a/tests/unit/brainy-core.unit.test.ts +++ b/tests/unit/brainy-core.unit.test.ts @@ -5,7 +5,7 @@ * No mocks, no fakes, real implementation */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' @@ -21,6 +21,10 @@ describe('Brainy 3.0 Core (Unit Tests)', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('CRUD Operations', () => { it('should create items with add', async () => { const id = await brain.add({ diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts index 945c0670..466fc654 100644 --- a/tests/unit/brainy/metadata-provider-contract.test.ts +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -18,7 +18,7 @@ * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS * metadata index, which has neither method by default. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' @@ -34,6 +34,10 @@ describe('metadata-provider contract wiring (getIdsForFilter opts)', () => { mi = (brain as any).metadataIndex }) + afterEach(async () => { + await brain.close() + }) + it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => { let probes = 0 let repairs = 0 diff --git a/tests/unit/brainy/migration-gate-family-scoped.test.ts b/tests/unit/brainy/migration-gate-family-scoped.test.ts index b71c3899..ce510a4e 100644 --- a/tests/unit/brainy/migration-gate-family-scoped.test.ts +++ b/tests/unit/brainy/migration-gate-family-scoped.test.ts @@ -8,7 +8,7 @@ * gate that hung getStats / readdir / readFile behind an unrelated family's * migration until the wait timed out. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy.js' import { MigrationInProgressError } from '../../../src/errors/brainyError.js' @@ -38,12 +38,19 @@ const jam = (provider: unknown) => { } describe('migration LOCK is family-scoped', () => { + const opened: Brainy[] = [] + beforeEach(() => { process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' }) + afterEach(async () => { + for (const b of opened.splice(0)) await b.close().catch(() => {}) + }) + it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => { const brain = await seed() + opened.push(brain) const childId = ( (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> )[0].entityId @@ -60,6 +67,7 @@ describe('migration LOCK is family-scoped', () => { it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => { const brain = await seed() + opened.push(brain) jam((brain as any).index) // A semantic query consults the vector index — it must wait, and (bounded by @@ -70,6 +78,7 @@ describe('migration LOCK is family-scoped', () => { it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => { const brain = await seed() + opened.push(brain) const childId = ( (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> )[0].entityId @@ -87,6 +96,7 @@ describe('migration LOCK is family-scoped', () => { it('with no migration in flight, every read serves (the fast path is a no-op)', async () => { const brain = await seed() + opened.push(brain) await expect(brain.getStats()).resolves.toBeDefined() await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) diff --git a/tests/unit/brainy/relate-duplicate-optimization.test.ts b/tests/unit/brainy/relate-duplicate-optimization.test.ts index 8bcb7c7a..910d057d 100644 --- a/tests/unit/brainy/relate-duplicate-optimization.test.ts +++ b/tests/unit/brainy/relate-duplicate-optimization.test.ts @@ -18,7 +18,7 @@ describe('Duplicate Check Optimization', () => { }) afterEach(async () => { - // Cleanup is automatic with memory storage + await brain.close() }) it('should detect duplicate relationships using GraphAdjacencyIndex', async () => { diff --git a/tests/unit/get-index-status-readiness.test.ts b/tests/unit/get-index-status-readiness.test.ts index 7f82ec5d..5e283bc8 100644 --- a/tests/unit/get-index-status-readiness.test.ts +++ b/tests/unit/get-index-status-readiness.test.ts @@ -7,7 +7,7 @@ * _indexRebuildFailed / _indexDegradedIds degraded states (mirroring * validateIndexConsistency / checkHealth). */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' describe('getIndexStatus honest readiness (Finding 9)', () => { @@ -20,6 +20,10 @@ describe('getIndexStatus honest readiness (Finding 9)', () => { await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => { brain.index.isReady = () => false // count present, serving structure NOT loaded const status = await brain.getIndexStatus() diff --git a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts index 46d318b4..95a6c0c4 100644 --- a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts +++ b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts @@ -8,7 +8,7 @@ * scan; and a one-shot probe self-heals a no-isReady provider whose adjacency * did not cold-load. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, VerbType } from '../../../src/index.js' describe('graph fast-path honest readiness (Finding 2)', () => { @@ -33,6 +33,10 @@ describe('graph fast-path honest readiness (Finding 2)', () => { await storage.getVerbsBySource(a) }) + afterEach(async () => { + await brain.close() + }) + it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => { const gi = storage.graphIndex // Simulate a cold native provider: count/manifest loaded (isInitialized) but diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts index b4f82f15..d079982e 100644 --- a/tests/unit/metadata-cold-read-guard.test.ts +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -15,7 +15,7 @@ * The 8.0 JS index cold-loads correctly, so we simulate the cold native failure * mode by intercepting the provider's getIdsForFilter/rebuild. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, MetadataIndexNotReadyError } from '../../src/index.js' const V = () => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) @@ -31,6 +31,10 @@ describe('Metadata cold-read guard (#venue silent-[])', () => { await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('warm brain: filtered find is correct and the guard does not rebuild', async () => { const mi = brain.metadataIndex let rebuilds = 0 diff --git a/tests/unit/migration-lock.test.ts b/tests/unit/migration-lock.test.ts index f0fbbe4c..63f6953e 100644 --- a/tests/unit/migration-lock.test.ts +++ b/tests/unit/migration-lock.test.ts @@ -18,7 +18,7 @@ * the production feature-detection reads it. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, MigrationInProgressError } from '../../src/index.js' import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' @@ -39,6 +39,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { await brain.init() }) + afterEach(async () => { + // The "close() is not gated" test already closes `brain` itself as its + // own assertion — closing an already-closed brain is a safe no-op here. + await brain.close().catch(() => {}) + }) + it('does not gate operations when no provider is migrating (fast path)', async () => { const id = await brain.add({ data: 'hello', type: NounType.Concept }) expect(id).toBeTruthy() @@ -130,6 +136,9 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { expect(e).toBeInstanceOf(MigrationInProgressError) expect(e.retryable).toBe(true) expect(typeof e.elapsedMs).toBe('number') + } finally { + // close() is proven not-gated by the test below — safe even mid-migration. + await shortBrain.close() } }) diff --git a/tests/unit/neural/signals/EmbeddingSignal.test.ts b/tests/unit/neural/signals/EmbeddingSignal.test.ts index 54d34b64..ad08e045 100644 --- a/tests/unit/neural/signals/EmbeddingSignal.test.ts +++ b/tests/unit/neural/signals/EmbeddingSignal.test.ts @@ -13,10 +13,11 @@ describe('EmbeddingSignal', () => { signal = new EmbeddingSignal(brain) }) - afterEach(() => { + afterEach(async () => { signal.clearCache() signal.clearHistory() signal.resetStats() + await brain.close() }) describe('initialization', () => { diff --git a/tests/unit/storage/pagination-parallel-hydration.test.ts b/tests/unit/storage/pagination-parallel-hydration.test.ts index ada324bb..a98fe8c9 100644 --- a/tests/unit/storage/pagination-parallel-hydration.test.ts +++ b/tests/unit/storage/pagination-parallel-hydration.test.ts @@ -7,7 +7,7 @@ * hydration (zero per-entity reads when unfiltered). Both must preserve the exact * pagination contract: same order, cursor continuation, filters, totalCount. */ -import { describe, it, expect, beforeEach, vi } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { Brainy, NounType } from '../../../src/index.js' describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => { @@ -30,6 +30,10 @@ describe('paginated enumeration — parallel hydration + id-only (cortex heal-co storage = brain.storage }) + afterEach(async () => { + await brain.close() + }) + /** Page the whole dataset through a small limit via cursor and collect ordered ids. */ const pageAll = async (fn: (opts: any) => Promise, key: 'items' | 'ids') => { const out: string[] = [] diff --git a/tests/unit/type-filtering.unit.test.ts b/tests/unit/type-filtering.unit.test.ts index 9e4700b2..a1943da9 100644 --- a/tests/unit/type-filtering.unit.test.ts +++ b/tests/unit/type-filtering.unit.test.ts @@ -4,7 +4,7 @@ * Tests to verify that brain.find({ type: NounType.X }) correctly filters entities */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' describe('Type Filtering (A Consumer Team Issue)', () => { @@ -17,6 +17,10 @@ describe('Type Filtering (A Consumer Team Issue)', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + it('should filter entities by NounType.Person', async () => { // Add 3 people await brain.add({ data: 'John Smith', type: NounType.Person, metadata: { name: 'John' } }) diff --git a/tests/unit/utils/metadataIndex-array-bound.test.ts b/tests/unit/utils/metadataIndex-array-bound.test.ts index a96ae1d6..32bf5d8c 100644 --- a/tests/unit/utils/metadataIndex-array-bound.test.ts +++ b/tests/unit/utils/metadataIndex-array-bound.test.ts @@ -48,6 +48,10 @@ describe('the indexable-array bound', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('BELOW the bound: the array indexes, every element of it', () => { it('the eleven-element array that used to vanish is searchable', async () => { // ELEVEN — one over the old silent limit, the whole shape of the defect. diff --git a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts index d6d00568..7a2bf0a7 100644 --- a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts +++ b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts @@ -42,7 +42,7 @@ * column store adopts the field. It is named in `getIdsFromChunksForRange`'s * doc comment rather than papered over. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking' @@ -122,6 +122,10 @@ describe('legacy sparse index: range queries order values, or refuse', () => { expect(index.columnStore.hasField(FIELD)).toBe(false) }) + afterEach(async () => { + await brain.close() + }) + describe('(a) a long BOUND against ordinary short values', () => { // 'apple' < 'mango' < 'zebra', and every bound below is compared against // these three raw keys. diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts index a5def81f..69133733 100644 --- a/tests/unit/validate-invariants-delegation.test.ts +++ b/tests/unit/validate-invariants-delegation.test.ts @@ -6,7 +6,7 @@ * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild' * to that provider's rebuild(). "healthy-while-broken must be impossible." */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' import type { ProviderInvariantReport } from '../../src/index.js' @@ -48,6 +48,10 @@ describe('validateIndexConsistency delegates to provider validateInvariants() (P await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('a broken provider report makes the store unhealthy and names the failing invariant', async () => { brain.index.validateInvariants = async () => brokenReport('vector') const v = await brain.validateIndexConsistency() diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts index 0905f298..963009b7 100644 --- a/tests/unit/vector-cold-read-guard.test.ts +++ b/tests/unit/vector-cold-read-guard.test.ts @@ -12,7 +12,7 @@ * signal (from either strategy) THROWS VectorIndexNotReadyError immediately, * with no rebuild attempt in between — never a silent empty result. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' const V = (): number[] => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) @@ -28,6 +28,10 @@ describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semant await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('warm brain: semantic find is correct and the guard does not rebuild', async () => { const vi = brain.index let rebuilds = 0 diff --git a/tests/unit/vfs-multi-instance-diagnostic.test.ts b/tests/unit/vfs-multi-instance-diagnostic.test.ts index deaa4615..85ff1002 100644 --- a/tests/unit/vfs-multi-instance-diagnostic.test.ts +++ b/tests/unit/vfs-multi-instance-diagnostic.test.ts @@ -4,7 +4,7 @@ * Tests to verify VFS import behavior and identify if VFS creates only wrappers or also graph entities */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' describe('VFS Multi-instance Diagnostic', () => { @@ -17,6 +17,10 @@ describe('VFS Multi-instance Diagnostic', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + it('should verify VFS creates document wrappers AND allows entity filtering', async () => { console.log('\n🔬 VFS Multi-instance Diagnostic Test\n') console.log('='.repeat(70)) From be307a15794246e95799123d9515e1ade0cedf56 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:13 -0700 Subject: [PATCH 10/21] test(hygiene): close every brain the vfs unit suite creates Each file opened a Brainy per test (beforeEach) and never closed it. --- tests/vfs/tree-operations.unit.test.ts | 6 +++++- tests/vfs/vfs-bug-fixes.unit.test.ts | 6 +++++- tests/vfs/vfs-bulkwrite-race.unit.test.ts | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/vfs/tree-operations.unit.test.ts b/tests/vfs/tree-operations.unit.test.ts index 8c717115..91743227 100644 --- a/tests/vfs/tree-operations.unit.test.ts +++ b/tests/vfs/tree-operations.unit.test.ts @@ -3,7 +3,7 @@ * Ensures tree methods prevent recursion and work correctly */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' import { VFSTreeUtils } from '../../src/vfs/TreeUtils.js' @@ -24,6 +24,10 @@ describe('VFS Tree Operations', () => { await vfs.init() }) + afterEach(async () => { + await brain.close() + }) + describe('Critical: No Self-Inclusion Bug', () => { it('should NEVER return a directory as its own child', async () => { // Create test structure diff --git a/tests/vfs/vfs-bug-fixes.unit.test.ts b/tests/vfs/vfs-bug-fixes.unit.test.ts index f98d6a76..12199c8b 100644 --- a/tests/vfs/vfs-bug-fixes.unit.test.ts +++ b/tests/vfs/vfs-bug-fixes.unit.test.ts @@ -6,7 +6,7 @@ * - Issue #2: File read decompression error */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' @@ -25,6 +25,10 @@ describe('VFS Bug Fixes', () => { await vfs.init() }) + afterEach(async () => { + await brain.close() + }) + describe('Issue #1: Duplicate Directory Nodes', () => { it('should not create duplicate directory entries when writing multiple files to same directory', async () => { // Write multiple files to the same directory (reproduce the bug scenario) diff --git a/tests/vfs/vfs-bulkwrite-race.unit.test.ts b/tests/vfs/vfs-bulkwrite-race.unit.test.ts index 238ac6b9..09d68568 100644 --- a/tests/vfs/vfs-bulkwrite-race.unit.test.ts +++ b/tests/vfs/vfs-bulkwrite-race.unit.test.ts @@ -12,7 +12,7 @@ * other operations in parallel batches. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' @@ -30,6 +30,10 @@ describe('VFS bulkWrite Race Condition Fix', () => { await vfs.init() }) + afterEach(async () => { + await brain.close() + }) + describe('operation ordering', () => { it('should create directories before files when mixed in same batch', async () => { // This is the exact scenario that triggered the race condition: From 4e058720b43dcfb469b2823b218673b28be711ea Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 08:52:58 -0700 Subject: [PATCH 11/21] fix(index): a field holds every value kind it was written with, not the first one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metadata index fixed a field's value type from the first value it saw. Every later value of another kind was coerced to that type, and when coercion failed — `Number('electronics')` is NaN — the value was dropped from the index with no error at all. The row stayed readable by id and by vector search and vanished only from equality filters on that one field, which is what made it so quiet: writing `category: 'electronics'` rows and then `category: 5` rows left `where { category: 5 }` returning nothing, while the same rows in a numbers-only corpus answered correctly. The column store now keeps one posting column per (field, kind), where a kind is a JavaScript typeof class. The first kind a field sees keeps the historical `_column_index//` layout, so a single-kind field is byte-identical to what earlier versions wrote and an index written before this opens unchanged; each later kind takes its own column at `_column_index//k//`. Equality reads the column matching the query value's own kind, so `{c: 5}` and `{c: '5'}` match different rows and neither is coerced into the other. Ranges route by the kind of their bounds, and an unbounded range — the "has any value" probe behind `exists` — reads every kind. A mixed field orders by kind first, then by value, because a number and a string have no order between them. A value that cannot be encoded for the column its own kind selected now raises instead of being skipped: that path is unreachable by construction, and if it is ever reached it is the silent drop this change exists to end. Two neighbours fell out of the same routing. A boolean query value is now encoded to the 1/0 the column stores, so boolean equality matches at all. And an integer column widens to f64 the first time a non-integer arrives, so 4.5 is stored as itself rather than rounded to 5 and answering the wrong query. Field type inference reports every kind a field holds beside its dominant reading, rather than leaving callers to treat one type as the whole answer. Pins: mixed-kind equality in both write orders, `5` vs `'5'`, booleans mixed in, a numeric range over a mixed field's numbers, close/reopen keeping every typed posting, and an index in the pre-existing on-disk shape still reading. `tests/critical-neural-validation.test.ts` — which writes `category` as strings in one test and as numbers in another against one shared brain — passes whole for the first time. (cherry picked from commit a128f0eda5b450ebf9caeae8e78ecaceff04feeb) --- .../architecture/data-storage-architecture.md | 34 ++ src/indexes/columnStore/ColumnStore.ts | 578 ++++++++++++++---- src/indexes/columnStore/ColumnTailBuffer.ts | 40 +- src/indexes/columnStore/types.ts | 60 ++ src/utils/fieldTypeInference.ts | 83 ++- .../metadata-field-typing.unit.test.ts | 122 ++++ .../column-store-mixed-kind.test.ts | 241 ++++++++ 7 files changed, 1025 insertions(+), 133 deletions(-) create mode 100644 tests/regression/metadata-field-typing.unit.test.ts create mode 100644 tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 83b9e23a..12398747 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -217,6 +217,40 @@ membership queries at scale: `__words__` for tokenized text…). - `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run segments, stored through the shared `_blobs/.bin` binary convention. +- `_column_index/{field}/k/{kind}/…` — the same two files again, for a + **second value kind** on the same field (see below). Absent for a field that + holds one kind, which is nearly all of them. + +### One posting column per (field, kind) + +A field is not obliged to hold one type of value. `category` may carry +`'electronics'` on some rows and `5` on others, and both are real values of +that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean +— so a field that holds several kinds gets **one column per kind**: + +- The first kind a field ever sees owns the plain `_column_index/{field}/` + layout above. A single-kind field is therefore byte-identical to what earlier + versions wrote, and an index written before typed postings opens unchanged. +- Every later kind gets its own column beside it at + `_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or + `boolean`. + +What that buys at query time: + +| | | +|---|---| +| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. | +| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. | +| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. | +| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. | +| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. | + +`null` and `undefined` are not kinds and are never posted; their absence is +what the `exists` / `missing` operators read. + +Older readers are unaffected by the additional columns: they see the field's +primary column exactly where it has always been, and a `k/{kind}` directory is +simply a name they never query. Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments additionally live as bucketed keys under `_system/idx/` (see §3). Which path diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 48f4a963..6bff86d4 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -23,7 +23,10 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js' import { ValueType, DEFAULT_FLUSH_THRESHOLD, - FLAG_MULTI_VALUE + FLAG_MULTI_VALUE, + POSTING_KINDS, + KIND_PATH_SEGMENT, + type PostingKind } from './types.js' import { ColumnTailBuffer } from './ColumnTailBuffer.js' import { ColumnManifest } from './ColumnManifest.js' @@ -52,10 +55,89 @@ interface HeapEntry { value: number | string entityIntId: number cursorIndex: number + /** + * Rank of the posting kind this entry came from, from {@link POSTING_KINDS}. + * A mixed-kind field has no natural total order, so the merge orders by kind + * first and by value within a kind. + */ + kindRank: number /** Iterator for the cursor — call next() to advance */ iterator: Generator } +/** + * One physical posting column: a (field, kind) pair and the key every internal + * map and every storage path uses for it. + */ +interface KindColumn { + /** The field as the query language names it. */ + field: string + /** The kind of value this column holds. */ + kind: PostingKind + /** + * Internal map / storage key. The field's PRIMARY kind uses the bare field + * name — the historical layout — and every other kind uses + * `//`. + */ + key: string +} + +/** + * The KIND a value indexes under — its JavaScript `typeof` class, not its + * storage encoding. + * + * Anything that is not a number, string or boolean indexes as a string, which + * is the `String(value)` treatment those values already received. `null` and + * `undefined` never reach here: `addEntity` skips them, and their absence is + * what the `exists` / `missing` operators read. + * + * @param value - The value about to be indexed or queried + * @returns The posting kind that owns this value + */ +function kindOfValue(value: unknown): PostingKind { + const t = typeof value + if (t === 'number') return 'number' + if (t === 'boolean') return 'boolean' + return 'string' +} + +/** + * The segment encoding a fresh column of this kind starts with. + * + * Only the number kind has a choice: an integer column starts as i64 and + * widens to f64 the first time a non-integer arrives + * ({@link ColumnTailBuffer.promoteToFloat}). + */ +function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType { + switch (kind) { + case 'boolean': + return ValueType.Boolean + case 'string': + return ValueType.String + case 'number': + return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float + } +} + +/** + * The kind a column of this encoding holds — the inverse of + * {@link initialValueTypeFor}, used to read a kind back off a manifest written + * before typed postings existed. + */ +function kindOfValueType(valueType: ValueType): PostingKind { + switch (valueType) { + case ValueType.Boolean: + return 'boolean' + case ValueType.String: + return 'string' + case ValueType.Number: + case ValueType.Float: + return 'number' + default: + throw new Error(`Unknown ValueType: ${valueType}`) + } +} + /** * Unified column store coordinator. * @@ -121,9 +203,19 @@ export class ColumnStore implements ColumnStoreProvider { */ private deletedEntities: Map = new Map() - /** Known field value types (inferred from first write). */ + /** Segment encoding per COLUMN key (not per field — a field has one per kind). */ private fieldTypes: Map = new Map() + /** + * Every posting column a field owns: field → kind → column key. + * + * This is the map that ends the first-writer type freeze. A field's first + * kind takes the bare field name as its column key, keeping the historical + * on-disk layout; each later kind takes its own column beside it. Nothing is + * coerced across kinds and nothing is dropped for being the wrong type. + */ + private fieldColumns: Map> = new Map() + /** Whether init() has completed. */ private initialized = false @@ -140,6 +232,128 @@ export class ColumnStore implements ColumnStoreProvider { this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4 } + // ========================================================================= + // Posting columns: (field, kind) → one physical column + // ========================================================================= + + /** + * Storage / map key for a (field, kind) column. + * + * `primary` is the kind that owns the bare field name. It is whichever kind + * the field saw first, which for an index written before typed postings is + * simply the kind of its single manifest — so the historical layout is + * preserved rather than migrated. + */ + private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string { + return primary === null || kind === primary + ? field + : `${field}/${KIND_PATH_SEGMENT}/${kind}` + } + + /** + * Split a discovered manifest path back into its (field, kind) column, or + * `null` when the path names a field's primary column rather than a kind + * column. `/k/` is the only shape that reads as a kind column, + * and only for a `` this version knows. + */ + private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null { + const marker = `/${KIND_PATH_SEGMENT}/` + const at = key.lastIndexOf(marker) + if (at <= 0) return null + const kind = key.slice(at + marker.length) + if (!POSTING_KINDS.includes(kind as PostingKind)) return null + return { field: key.slice(0, at), kind: kind as PostingKind } + } + + /** Record a discovered or freshly created column against its field. */ + private registerColumn(field: string, kind: PostingKind, key: string): void { + let byKind = this.fieldColumns.get(field) + if (!byKind) { + byKind = new Map() + this.fieldColumns.set(field, byKind) + } + const existing = byKind.get(kind) + if (existing !== undefined && existing !== key) { + // Two columns claiming one (field, kind) means the layout on disk is not + // one this writer could have produced. Serving it would silently answer + // from half the postings, so say which two and stop. + throw new Error( + `ColumnStore: field '${field}' has two '${kind}' posting columns on ` + + `disk ('${existing}' and '${key}'). The column index layout is ` + + `inconsistent — rebuild/repair the metadata index rather than ` + + `serving from one half of it.` + ) + } + byKind.set(kind, key) + } + + /** The column key for this (field, kind), or `null` if the field has no such kind. */ + private columnKey(field: string, kind: PostingKind): string | null { + return this.fieldColumns.get(field)?.get(kind) ?? null + } + + /** + * The column key for this (field, kind), creating the registration if the + * field has not seen this kind before. Write path only. + */ + private ensureColumnKey(field: string, kind: PostingKind): string { + const byKind = this.fieldColumns.get(field) + const existing = byKind?.get(kind) + if (existing !== undefined) return existing + + // The primary kind is the one already holding the bare field name, if any. + let primary: PostingKind | null = null + if (byKind) { + for (const [k, key] of byKind) { + if (key === field) { primary = k; break } + } + } + const key = ColumnStore.columnKeyFor(field, kind, primary) + this.registerColumn(field, kind, key) + return key + } + + /** + * Every posting column this field owns, in {@link POSTING_KINDS} order. + * + * Read doors that are not about one particular value — an unbounded range + * used as an "any value present" probe, distinct values, sorting — fan out + * over all of them. + */ + private columnsForField(field: string): KindColumn[] { + const byKind = this.fieldColumns.get(field) + if (!byKind) return [] + const out: KindColumn[] = [] + for (const kind of POSTING_KINDS) { + const key = byKind.get(kind) + if (key !== undefined) out.push({ field, kind, key }) + } + return out + } + + /** + * Which value kinds this field actually holds, in {@link POSTING_KINDS} + * order — the honest answer to "what type is this field?". + * + * A field that carries both `'electronics'` and `5` reports + * `['number', 'string']`, not whichever of them was written first. + * + * @param field - Field name + * @returns Every kind with at least one posting, or `[]` for an unknown field + */ + getFieldKinds(field: string): PostingKind[] { + return this.columnsForField(field) + .filter((c) => this.columnHasData(c.key)) + .map((c) => c.kind) + } + + /** Does this physical column hold any postings (persisted or buffered)? */ + private columnHasData(key: string): boolean { + const manifest = this.manifests.get(key) + const buffer = this.tailBuffers.get(key) + return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0) + } + /** * Initialize the column store: discover existing field manifests. */ @@ -157,11 +371,23 @@ export class ColumnStore implements ColumnStoreProvider { }).listObjectsUnderPath(this.basePath + '/') for (const path of paths) { if (path.endsWith('/MANIFEST.json')) { - const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '') - const manifest = new ColumnManifest(fieldName, this.basePath) + // The discovered name is a COLUMN key: either a bare field (that + // field's primary kind, which is every column an index written + // before typed postings has) or `/k/` for a second + // kind that arrived on a field later. + const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '') + const manifest = new ColumnManifest(columnKey, this.basePath) await manifest.load(storage) - this.manifests.set(fieldName, manifest) - this.fieldTypes.set(fieldName, manifest.valueType) + this.manifests.set(columnKey, manifest) + this.fieldTypes.set(columnKey, manifest.valueType) + + const parsed = ColumnStore.parseKindColumnKey(columnKey) + if (parsed) { + this.registerColumn(parsed.field, parsed.kind, columnKey) + } else { + this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey) + } + const fieldName = columnKey // Load global deleted bitmap if it exists. Raw blob preferred // (2.4.0 #4 cortex-shared format); legacy envelope fallback for @@ -264,26 +490,43 @@ export class ColumnStore implements ColumnStoreProvider { /** * Point filter: find entities where field equals value. * - * Searches all segments + tail buffer, returns union as roaring bitmap. - * Excludes globally deleted entities. + * The QUERY VALUE'S OWN KIND picks the posting column, and only that column + * is read. `where {category: 5}` answers from the number postings and + * `where {category: '5'}` from the string postings — neither borrows the + * other's rows, because a row written with the number `5` is not a row whose + * category is the text `'5'`. + * + * A field that has never seen this kind matches nothing, which is the true + * answer rather than a coerced one. + * + * Searches all segments + tail buffer of that column, returns the union as a + * roaring bitmap. Excludes globally deleted entities. */ async filter(field: string, value: unknown): Promise { const result = new RoaringBitmap32() - const deleted = this.deletedEntities.get(field) + const columnKey = this.columnKey(field, kindOfValue(value)) + if (columnKey === null) return result + + // The query value takes the column's encoding — a boolean queried against + // a boolean column has to become the 1/0 the column stores. + const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String) + if (encoded === undefined) return result + + const deleted = this.deletedEntities.get(columnKey) // Search segments - const cursors = await this.getSegmentCursors(field) + const cursors = await this.getSegmentCursors(columnKey) for (const cursor of cursors) { - const ids = cursor.getEntityIdsForValue(value as number | string) + const ids = cursor.getEntityIdsForValue(encoded) for (const id of ids) { if (!deleted || !deleted.has(id)) result.add(id) } } // Search tail buffer - const tailCursor = this.getTailBufferCursor(field) + const tailCursor = this.getTailBufferCursor(columnKey) if (tailCursor) { - const ids = tailCursor.getEntityIdsForValue(value as number | string) + const ids = tailCursor.getEntityIdsForValue(encoded) for (const id of ids) { if (!deleted || !deleted.has(id)) result.add(id) } @@ -324,22 +567,26 @@ export class ColumnStore implements ColumnStoreProvider { const out = new Map() if (wanted.size === 0 || !this.hasField(field)) return out - const deleted = this.deletedEntities.get(field) - const take = (entry: { value: number | string; entityIntId: number }): void => { - if (!wanted.has(entry.entityIntId)) return - if (deleted && deleted.has(entry.entityIntId)) return - out.set(entry.entityIntId, entry.value) - } + // Every kind the field holds is read, in POSTING_KINDS order — a value an + // entity wrote as a string is still that entity's value for this field. + for (const column of this.columnsForField(field)) { + const deleted = this.deletedEntities.get(column.key) + const take = (entry: { value: number | string; entityIntId: number }): void => { + if (!wanted.has(entry.entityIntId)) return + if (deleted && deleted.has(entry.entityIntId)) return + out.set(entry.entityIntId, entry.value) + } - // Segments oldest -> newest, then the tail: a later write overwrites an - // earlier one for the same id. - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) take(entry) - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) take(entry) + // Segments oldest -> newest, then the tail: a later write overwrites an + // earlier one for the same id. + const cursors = await this.getSegmentCursors(column.key) + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) take(entry) + } + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) take(entry) + } } return out } @@ -363,41 +610,59 @@ export class ColumnStore implements ColumnStoreProvider { includeMax: boolean = true ): Promise { const result = new RoaringBitmap32() - const cursors = await this.getSegmentCursors(field) const hasMin = min !== undefined && min !== null const hasMax = max !== undefined && max !== null - for (const cursor of cursors) { - const lo = hasMin ? min as number | string : cursor.minValue - const hi = hasMax ? max as number | string : cursor.maxValue - if (lo === undefined || hi === undefined) continue - // Exclusivity applies only to an explicitly provided bound. A bound taken - // from the segment's own min/max is a real stored value and must stay - // inclusive, or the segment's boundary entities would be wrongly dropped. - const ids = cursor.getEntityIdsInRange( - lo, - hi, - hasMin ? includeMin : true, - hasMax ? includeMax : true - ) - for (const id of ids) result.add(id) - } + // The BOUNDS pick the column: numeric bounds read the numeric postings, + // string bounds the string postings. An unbounded call is not a range at + // all — it is the "has any value here" probe behind `exists` — so it fans + // out over every kind the field holds. + const columns: KindColumn[] = hasMin + ? this.columnsForKind(field, kindOfValue(min)) + : hasMax + ? this.columnsForKind(field, kindOfValue(max)) + : this.columnsForField(field) - // Tail buffer range: linear scan (tail is small) - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) { - const v = entry.value as any - const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any)) - const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any)) - if (loOk && hiOk) result.add(entry.entityIntId) + for (const column of columns) { + const cursors = await this.getSegmentCursors(column.key) + for (const cursor of cursors) { + const lo = hasMin ? min as number | string : cursor.minValue + const hi = hasMax ? max as number | string : cursor.maxValue + if (lo === undefined || hi === undefined) continue + // Exclusivity applies only to an explicitly provided bound. A bound taken + // from the segment's own min/max is a real stored value and must stay + // inclusive, or the segment's boundary entities would be wrongly dropped. + const ids = cursor.getEntityIdsInRange( + lo, + hi, + hasMin ? includeMin : true, + hasMax ? includeMax : true + ) + for (const id of ids) result.add(id) + } + + // Tail buffer range: linear scan (tail is small) + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) { + const v = entry.value as any + const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any)) + const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any)) + if (loOk && hiOk) result.add(entry.entityIntId) + } } } return result } + /** The single column for this (field, kind), as a list, or empty if absent. */ + private columnsForKind(field: string, kind: PostingKind): KindColumn[] { + const key = this.columnKey(field, kind) + return key === null ? [] : [{ field, kind, key }] + } + /** * Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt). * @@ -428,18 +693,21 @@ export class ColumnStore implements ColumnStoreProvider { */ async getFilterValues(field: string): Promise { const valueSet = new Set() - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) { - valueSet.add(String(entry.value)) + for (const column of this.columnsForField(field)) { + const cursors = await this.getSegmentCursors(column.key) + + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) { + valueSet.add(String(entry.value)) + } } - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) { - valueSet.add(String(entry.value)) + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) { + valueSet.add(String(entry.value)) + } } } @@ -450,9 +718,7 @@ export class ColumnStore implements ColumnStoreProvider { * Check if a field has any indexed data. */ hasField(field: string): boolean { - const manifest = this.manifests.get(field) - const buffer = this.tailBuffers.get(field) - return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0) + return this.columnsForField(field).some((c) => this.columnHasData(c.key)) } /** @@ -462,12 +728,11 @@ export class ColumnStore implements ColumnStoreProvider { * store will actually serve queries from. */ getIndexedFields(): string[] { + // Names FIELDS, not columns: a field carrying two kinds is one name here, + // the same name a caller queries with. const fields = new Set() - for (const [field, manifest] of this.manifests) { - if (!manifest.isEmpty()) fields.add(field) - } - for (const [field, buffer] of this.tailBuffers) { - if (buffer.size > 0) fields.add(field) + for (const [field] of this.fieldColumns) { + if (this.hasField(field)) fields.add(field) } return Array.from(fields).sort() } @@ -482,12 +747,16 @@ export class ColumnStore implements ColumnStoreProvider { getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> { const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = [] for (const field of this.getIndexedFields()) { - const manifest = this.manifests.get(field) - const buffer = this.tailBuffers.get(field) - const segmentCount = manifest && !manifest.isEmpty() - ? manifest.getAllSegments().length - : 0 - const tailSize = buffer ? buffer.size : 0 + // Summed across the field's kind columns — the caller asked about a + // field, and a field's size is all of the postings under its name. + let segmentCount = 0 + let tailSize = 0 + for (const column of this.columnsForField(field)) { + const manifest = this.manifests.get(column.key) + const buffer = this.tailBuffers.get(column.key) + if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length + if (buffer) tailSize += buffer.size + } summary.push({ field, segmentCount, tailSize }) } return summary @@ -515,6 +784,8 @@ export class ColumnStore implements ColumnStoreProvider { this.segmentCache.clear() this.manifests.clear() this.deletedEntities.clear() + this.fieldColumns.clear() + this.fieldTypes.clear() this.initialized = false } @@ -523,32 +794,64 @@ export class ColumnStore implements ColumnStoreProvider { // ========================================================================= /** - * Push a single value to a field's tail buffer. - * Creates the buffer and manifest if first write to this field. - * Infers ValueType from the first value seen. + * Push a single value to the posting column for its (field, KIND). + * + * The value's own kind picks the column — a string goes to the field's + * string postings, a number to its number postings — so a field carrying + * `'electronics'` and `5` keeps both, each answerable by an equality filter + * of its own kind. Under the first-writer type freeze this method replaced, + * the first value's type became the field's type and every later value of + * another kind was coerced to it or, when coercion failed, dropped with no + * error at all. + * + * Creates the column's buffer and manifest on its first value. */ private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void { - let buffer = this.tailBuffers.get(field) + const kind = kindOfValue(value) + const columnKey = this.ensureColumnKey(field, kind) + + let buffer = this.tailBuffers.get(columnKey) if (!buffer) { - const valueType = this.inferValueType(value) - buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold) - this.tailBuffers.set(field, buffer) - this.fieldTypes.set(field, valueType) + // A reopened column takes its encoding from its manifest — an integer + // column that widened to f64 in an earlier session stays widened. + const valueType = + this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value) + buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold) + this.tailBuffers.set(columnKey, buffer) + this.fieldTypes.set(columnKey, valueType) // Ensure manifest exists - if (!this.manifests.has(field)) { - const manifest = new ColumnManifest(field, this.basePath) + if (!this.manifests.has(columnKey)) { + const manifest = new ColumnManifest(columnKey, this.basePath) manifest.valueType = valueType manifest.multiValue = isMultiValue - this.manifests.set(field, manifest) + this.manifests.set(columnKey, manifest) } } - // Normalize value to the column type - const normalizedValue = this.normalizeValue(value, buffer.valueType) - if (normalizedValue !== undefined) { - buffer.add(normalizedValue, entityIntId) + // An integer column widens the first time a non-integer number arrives, so + // the value is stored as itself instead of rounded to the nearest integer. + if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) { + buffer.promoteToFloat() + this.fieldTypes.set(columnKey, ValueType.Float) + const manifest = this.manifests.get(columnKey) + if (manifest) manifest.valueType = ValueType.Float } + + const normalizedValue = this.normalizeValue(value, buffer.valueType) + if (normalizedValue === undefined) { + // Unreachable by construction: the column was chosen BY this value's + // kind, so the encoding always accepts it. Reaching here would mean a + // value had been silently dropped from the index — the exact failure + // typed postings exist to end — so it is an error, never a skip. + throw new Error( + `ColumnStore: field '${field}' rejected a ${kind} value for its own ` + + `${ValueType[buffer.valueType]} posting column. The value would have ` + + `been dropped from the index while the row stayed readable by id — ` + + `this is a kind-routing bug, not a value the caller may ignore.` + ) + } + buffer.add(normalizedValue, entityIntId) } /** @@ -677,8 +980,15 @@ export class ColumnStore implements ColumnStoreProvider { /** Torn-segment quarantine entries for a field (observability + heal input). */ quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> { const out: Array<{ segment: string; error: string; hits: number }> = [] - for (const [key, q] of this.segmentQuarantine) { - if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits }) + // Across every kind column of the field — a torn segment in the string + // postings is this field's torn segment as much as one in the numbers. + for (const column of this.columnsForField(field)) { + const prefix = `${column.key}:` + for (const [key, q] of this.segmentQuarantine) { + if (key.startsWith(prefix)) { + out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits }) + } + } } return out } @@ -850,17 +1160,22 @@ export class ColumnStore implements ColumnStoreProvider { k: number, filterBitmap: RoaringBitmap32 | null ): Promise { - // Collect all cursors (segments + tail buffer) - const segCursors = await this.getSegmentCursors(field) - const tailCursor = this.getTailBufferCursor(field) - - // Create iterators for each cursor in the specified direction + // Collect cursors across EVERY kind the field holds. A single-kind field — + // nearly all of them — merges exactly the cursors it always did. const iterators: Generator[] = [] - for (const cursor of segCursors) { - iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward()) - } - if (tailCursor) { - iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward()) + const iteratorKindRank: number[] = [] + for (const column of this.columnsForField(field)) { + const kindRank = POSTING_KINDS.indexOf(column.kind) + const segCursors = await this.getSegmentCursors(column.key) + for (const cursor of segCursors) { + iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward()) + iteratorKindRank.push(kindRank) + } + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward()) + iteratorKindRank.push(kindRank) + } } if (iterators.length === 0) return [] @@ -874,16 +1189,21 @@ export class ColumnStore implements ColumnStoreProvider { value: next.value.value, entityIntId: next.value.entityIntId, cursorIndex: i, + kindRank: iteratorKindRank[i], iterator: iterators[i] }) } } - // Heapify - const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String + // Heapify. A number and a string have no ordering between them, so a + // mixed-kind field orders by KIND first (POSTING_KINDS order) and by value + // within a kind — one defined total order instead of a comparison whose + // answer depends on which value happened to be on the left. const compare = (a: HeapEntry, b: HeapEntry): number => { let cmp: number - if (isString) { + if (a.kindRank !== b.kindRank) { + cmp = a.kindRank - b.kindRank + } else if (POSTING_KINDS[a.kindRank] === 'string') { cmp = compareCodePoints(String(a.value), String(b.value)) } else { cmp = (a.value as number) - (b.value as number) @@ -915,6 +1235,7 @@ export class ColumnStore implements ColumnStoreProvider { value: next.value.value, entityIntId: next.value.entityIntId, cursorIndex: top.cursorIndex, + kindRank: top.kindRank, iterator: top.iterator } } @@ -922,8 +1243,11 @@ export class ColumnStore implements ColumnStoreProvider { this.heapDown(heap, 0, compare) } - // Apply global deleted check, filter, and dedup - const deleted = this.deletedEntities.get(field) + // Apply global deleted check, filter, and dedup. The deleted bitmap is + // per COLUMN, and the entry came from the column its kind names. + const deleted = this.deletedEntities.get( + this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field + ) if (deleted && deleted.has(top.entityIntId)) continue if (seen.has(top.entityIntId)) continue if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue @@ -965,35 +1289,31 @@ export class ColumnStore implements ColumnStoreProvider { } /** - * Infer ValueType from a JavaScript value. - */ - private inferValueType(value: unknown): ValueType { - if (typeof value === 'boolean') return ValueType.Boolean - if (typeof value === 'number') { - return Number.isInteger(value) ? ValueType.Number : ValueType.Float - } - return ValueType.String - } - - /** - * Normalize a JavaScript value to the column's ValueType. + * Encode a value for the column its own kind selected. + * + * This does NOT convert between kinds. It used to: a string reaching a + * numeric column was run through `Number(value)`, and a number reaching a + * numeric column was run through `Math.round`, so `'electronics'` became + * `NaN` and vanished while `4.5` became `5` and answered the wrong query. + * Kind routing removes the need for either — the only work left is picking + * the encoding the column already committed to. + * + * @returns The encoded value, or `undefined` if the value does not belong in + * this column at all — which the caller treats as a routing bug and + * raises, never as a value to skip. */ private normalizeValue(value: unknown, type: ValueType): number | string | undefined { switch (type) { case ValueType.Number: - if (typeof value === 'number') return Math.round(value) - if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) } - if (typeof value === 'boolean') return value ? 1 : 0 - return undefined + // Integer column. Non-integers widen it to Float before reaching here. + return typeof value === 'number' && Number.isInteger(value) ? value : undefined case ValueType.Float: - if (typeof value === 'number') return value - if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n } - return undefined + return typeof value === 'number' ? value : undefined case ValueType.Boolean: - if (typeof value === 'boolean') return value ? 1 : 0 - if (typeof value === 'number') return value ? 1 : 0 - return undefined + return typeof value === 'boolean' ? (value ? 1 : 0) : undefined case ValueType.String: + // The string kind is also where objects and bigints land, exactly as + // they always did. return String(value) default: return undefined diff --git a/src/indexes/columnStore/ColumnTailBuffer.ts b/src/indexes/columnStore/ColumnTailBuffer.ts index c5874ac2..e730f884 100644 --- a/src/indexes/columnStore/ColumnTailBuffer.ts +++ b/src/indexes/columnStore/ColumnTailBuffer.ts @@ -55,8 +55,12 @@ export class ColumnTailBuffer { /** Field name this buffer is for. */ readonly fieldName: string - /** Value type determines sort comparator. */ - readonly valueType: ValueType + /** + * Value type determines sort comparator and segment encoding. + * + * Widened in place by {@link promoteToFloat} — never otherwise reassigned. + */ + valueType: ValueType /** Flush threshold. */ readonly threshold: number @@ -81,6 +85,38 @@ export class ColumnTailBuffer { this.threshold = threshold } + /** + * Widen an integer column to floating point, losslessly and in place. + * + * The number posting kind holds every JavaScript number, but a segment picks + * ONE encoding: i64 for integers, f64 for the rest. A column that has only + * ever seen integers is written as i64; the first non-integer to arrive + * widens it here, so that value is stored as itself instead of being rounded + * to the nearest integer with no error — the rounding that made `4.5` and + * `5.5` both answer `where {score: 5}` and neither answer its own value. + * + * Widening is lossless in both directions it has to be: every value already + * buffered is an integer, and every integer is exactly representable as f64. + * Segments already on disk keep their own i64 encoding in their own headers + * and keep decoding by it — only segments written from here on are f64. + * + * @throws Error if called on a column that is not an integer column — the + * only legal widening is Number → Float, and any other request is a bug in + * the caller's kind routing rather than something to absorb quietly. + */ + promoteToFloat(): void { + if (this.valueType === ValueType.Float) return + if (this.valueType !== ValueType.Number) { + throw new Error( + `ColumnTailBuffer '${this.fieldName}': cannot widen a ` + + `${ValueType[this.valueType]} column to Float — only an integer ` + + `(Number) column widens, and this call means a value reached the ` + + `wrong kind's column` + ) + } + this.valueType = ValueType.Float + } + /** * Add a (value, entityIntId) entry to the buffer. * diff --git a/src/indexes/columnStore/types.ts b/src/indexes/columnStore/types.ts index 71dd99a0..ee949bd0 100644 --- a/src/indexes/columnStore/types.ts +++ b/src/indexes/columnStore/types.ts @@ -58,6 +58,53 @@ export enum ValueType { Boolean = 3 } +/** + * The KIND of a value, as the query language sees it. + * + * A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5` + * are one kind (`'number'`) held in one posting column, even though they need + * different segment encodings (i64 vs f64 — see {@link ValueType}). + * + * A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string + * values and number values at the same time and answer equality on each. This + * replaces the first-writer type freeze, under which the first value's type + * became the field's type and every later value of another kind was coerced — + * or, when coercion failed (`Number('electronics')`), dropped from the index + * with no error: the row stayed readable by id and by vector but vanished from + * every equality filter on that field. + * + * Kinds do not coerce into one another at query time either: `where {c: 5}` + * matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows + * written with the STRING `'5'`. Neither ever matches the other. + * + * Values that are none of these three (objects, bigints) index as strings — + * the same `String(value)` treatment they received before. + */ +export type PostingKind = 'number' | 'string' | 'boolean' + +/** + * Every posting kind, in the order that defines cross-kind sort position. + * + * A mixed-kind field has no natural total order — a number does not compare + * with a string — so `sortTopK` orders by KIND first (numbers, then strings, + * then booleans) and by value within a kind. A single-kind field, which is + * nearly every field, sorts exactly as it always did. + */ +export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean'] + +/** + * Path segment marking a field's NON-PRIMARY kind columns on disk. + * + * The first kind a field ever sees keeps the historical layout — + * `//MANIFEST.json` and `//L0-NNNNNN` — so every + * index written before typed postings opens unchanged, and the byte-for-byte + * interchange with the native column store is untouched for the single-kind + * fields that are nearly all of them. A second kind arriving on the same field + * gets its own column at `//k//…` rather than overwriting or + * being coerced into the first. + */ +export const KIND_PATH_SEGMENT = 'k' + // --------------------------------------------------------------------------- // Segment header and footer // --------------------------------------------------------------------------- @@ -267,6 +314,19 @@ export interface ColumnStoreProvider { */ hasField(field: string): boolean + /** + * Which value KINDS this field actually holds, in {@link POSTING_KINDS} + * order — the honest answer to "what type is this field?" for a field that + * carries more than one. + * + * OPTIONAL so an implementation written against the pre-typed-postings + * contract still satisfies this interface; feature-detect before calling. + * + * @param field - Field name + * @returns Every kind with at least one posting, or `[]` for an unknown field + */ + getFieldKinds?(field: string): PostingKind[] + /** * Flush all in-memory tail buffers to L0 segments on disk. * Saves all manifests. diff --git a/src/utils/fieldTypeInference.ts b/src/utils/fieldTypeInference.ts index 36a415b2..0f085f8c 100644 --- a/src/utils/fieldTypeInference.ts +++ b/src/utils/fieldTypeInference.ts @@ -55,8 +55,30 @@ export enum FieldType { */ export interface FieldTypeInfo { field: string + /** + * The DOMINANT reading of the field — one type, the most specific one every + * sampled value satisfies. + * + * A field is not obliged to hold one kind, so this is not the whole answer + * for a field that holds several. Read {@link kinds} beside it: a field + * carrying `'electronics'` and `5` infers as STRING here and reports + * `['number', 'string']` there, and the metadata index keeps a separate + * posting column for each of them. + */ inferredType: FieldType confidence: number // 0-1 confidence score + /** + * Every value KIND observed in the sample, in the order + * number → string → boolean. More than one entry means a genuinely + * mixed field, and every one of those kinds is independently filterable. + * + * Kinds are JavaScript `typeof` classes, one level coarser than + * {@link FieldType}: a UUID and a category name are both `'string'`, and an + * integer and a timestamp are both `'number'`. + * + * Optional only for cached analyses written before this was reported. + */ + kinds?: Array<'number' | 'string' | 'boolean'> sampleSize: number // Number of values analyzed lastUpdated: number // Timestamp of last analysis detectionMethod: 'value' // Always 'value' (no fallbacks!) @@ -133,14 +155,71 @@ export class FieldTypeInference { } /** - * Analyze values to determine field type + * Analyze values to determine field type, and report every KIND the field + * actually holds alongside it. + * + * The classification below picks ONE type, because every one of its + * heuristics asks `samples.every(...)`: a field carrying `'electronics'` and + * `5` satisfies none of them and lands on STRING. That single answer is true + * as far as it goes — string is the dominant reading — but on its own it + * says nothing about the numbers also in the field, and a caller that treats + * it as the field's only type reproduces the first-writer freeze the index + * itself no longer has. {@link FieldTypeInfo.kinds} carries the rest. + */ + private async analyzeValues(field: string, values: any[]): Promise { + const info = await this.classifyValues(field, values) + info.kinds = FieldTypeInference.observedKinds(values) + if (info.kinds.length > 1 && info.metadata) { + info.metadata.format = `${info.metadata.format} (field also holds: ${info.kinds + .filter((k) => k !== FieldTypeInference.kindOfType(info.inferredType)) + .join(', ')})` + } + return info + } + + /** + * The distinct value kinds present in a sample, in a stable order. + * + * Kinds are JavaScript `typeof` classes — the same classes the metadata + * index keeps separate posting columns for — not the finer + * {@link FieldType} readings, which are interpretations layered on top of + * them (a UUID and a category name are both the `string` kind). + */ + private static observedKinds(values: any[]): Array<'number' | 'string' | 'boolean'> { + const order: Array<'number' | 'string' | 'boolean'> = ['number', 'string', 'boolean'] + const seen = new Set<'number' | 'string' | 'boolean'>() + for (const v of values) { + if (v === null || v === undefined) continue + const t = typeof v + seen.add(t === 'number' ? 'number' : t === 'boolean' ? 'boolean' : 'string') + } + return order.filter((k) => seen.has(k)) + } + + /** The value kind a {@link FieldType} reading is an interpretation of. */ + private static kindOfType(type: FieldType): 'number' | 'string' | 'boolean' { + switch (type) { + case FieldType.BOOLEAN: + return 'boolean' + case FieldType.INTEGER: + case FieldType.FLOAT: + case FieldType.TIMESTAMP_MS: + case FieldType.TIMESTAMP_S: + return 'number' + default: + return 'string' + } + } + + /** + * Classify values into a single field type. * * Uses DuckDB-inspired type detection order: * BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING * * No fallbacks - pure value-based detection */ - private async analyzeValues(field: string, values: any[]): Promise { + private async classifyValues(field: string, values: any[]): Promise { // Filter null/undefined values const validValues = values.filter(v => v !== null && v !== undefined) diff --git a/tests/regression/metadata-field-typing.unit.test.ts b/tests/regression/metadata-field-typing.unit.test.ts new file mode 100644 index 00000000..910d4f2a --- /dev/null +++ b/tests/regression/metadata-field-typing.unit.test.ts @@ -0,0 +1,122 @@ +/** + * @module metadata-field-typing.unit.test + * @description Regression: a metadata field that holds more than one value + * KIND stays fully filterable on every kind it holds. + * + * The defect this pins, reproduced on the released engine: the metadata index + * fixed a field's value type from the FIRST value it saw, and every later value + * of a different type was coerced to that type or, when coercion failed, + * dropped from the index in silence. Writing `category: 'electronics'` rows and + * then `category: 5` rows left `find({ where: { category: 5 } })` returning + * nothing — while the same rows in a numbers-only corpus answered correctly. + * The rows themselves were never lost: they stayed readable by id and by vector + * search, and only ever went missing from equality filters on that one field, + * which is what made it so quiet. + * + * Order is the whole point of these cases. Neither writer owns the field, so + * strings-then-numbers and numbers-then-strings must give the same answers. + */ + +import { describe, it, expect } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** A brain over memory storage, with a corpus written in the given order. */ +async function brainWith( + rows: Array<{ label: string; category: unknown }> +): Promise { + const brainy = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brainy.init() + for (const row of rows) { + await brainy.add({ + data: `item ${row.label}`, + type: NounType.Thing, + metadata: { label: row.label, category: row.category } + }) + } + return brainy +} + +const labelsOf = (results: Array<{ metadata?: Record }>): string[] => + results.map((r) => String(r.metadata?.label)).sort() + +describe('regression: a mixed-kind metadata field filters on every kind', { timeout: 180_000 }, () => { + it('finds number rows written after string rows', async () => { + const brainy = await brainWith([ + { label: 'e1', category: 'electronics' }, + { label: 'f1', category: 'furniture' }, + { label: 'n1', category: 5 }, + { label: 'n2', category: 5 }, + { label: 'n3', category: 7 } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2']) + expect(labelsOf(await brainy.find({ where: { category: 7 }, limit: 100 }))).toEqual(['n3']) + expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1']) + expect(labelsOf(await brainy.find({ where: { category: 'furniture' }, limit: 100 }))).toEqual(['f1']) + } finally { + await brainy.close() + } + }) + + it('finds string rows written after number rows', async () => { + const brainy = await brainWith([ + { label: 'n1', category: 5 }, + { label: 'n2', category: 5 }, + { label: 'e1', category: 'electronics' }, + { label: 'e2', category: 'electronics' } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1', 'e2']) + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2']) + } finally { + await brainy.close() + } + }) + + it('keeps `5` and `\'5\'` apart — a kind is part of the value, not a formatting detail', async () => { + const brainy = await brainWith([ + { label: 'num', category: 5 }, + { label: 'str', category: '5' } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['num']) + expect(labelsOf(await brainy.find({ where: { category: '5' }, limit: 100 }))).toEqual(['str']) + } finally { + await brainy.close() + } + }) + + it('serves booleans mixed into a field that already holds strings', async () => { + const brainy = await brainWith([ + { label: 's1', category: 'yes' }, + { label: 'b1', category: true }, + { label: 'b2', category: false } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: true }, limit: 100 }))).toEqual(['b1']) + expect(labelsOf(await brainy.find({ where: { category: false }, limit: 100 }))).toEqual(['b2']) + expect(labelsOf(await brainy.find({ where: { category: 'yes' }, limit: 100 }))).toEqual(['s1']) + } finally { + await brainy.close() + } + }) + + it('ranges over the numeric part of a mixed field', async () => { + const brainy = await brainWith([ + { label: 'unpriced', category: 'on request' }, + { label: 'cheap', category: 100 }, + { label: 'mid', category: 500 }, + { label: 'dear', category: 900 } + ]) + try { + const found = await brainy.find({ + where: { category: { greaterThan: 200 } }, + limit: 100 + }) + expect(labelsOf(found)).toEqual(['dear', 'mid']) + } finally { + await brainy.close() + } + }) +}) diff --git a/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts b/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts new file mode 100644 index 00000000..1ce21d1f --- /dev/null +++ b/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts @@ -0,0 +1,241 @@ +/** + * @module column-store-mixed-kind.test + * @description Typed posting lists: one field, several value KINDS, each + * answerable on its own. + * + * The behaviour these pin replaced a first-writer type freeze. The first value + * a field ever saw fixed that field's type; every later value of another kind + * was coerced to it, and when coercion failed — `Number('electronics')` — the + * value was dropped from the index with no error at all. The row stayed + * readable by id and by vector and vanished from every equality filter on the + * field. These tests therefore care about ORDER: strings-then-numbers and + * numbers-then-strings have to behave identically, because neither writer owns + * the field. + * + * Kinds never coerce into one another at query time either. `5` and `'5'` are + * different values and match different rows. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' +import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' +import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' + +describe('ColumnStore — typed posting lists per (field, kind)', () => { + let storage: MemoryStorage + let idMapper: EntityIdMapper + let store: ColumnStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) + await idMapper.init() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + }) + + afterEach(async () => { + await store.close() + }) + + /** Resolve a filter to the sorted UUIDs it matched. */ + const uuidsOf = async (field: string, value: unknown): Promise => { + const bitmap = await store.filter(field, value) + return Array.from(bitmap) + .map((id) => idMapper.getUuid(Number(id))) + .filter((u): u is string => u !== undefined) + .sort() + } + + describe('equality answers on the query value’s own kind', () => { + it('serves numbers written AFTER strings on the same field', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 }) + + // The numbers are in the index, though a string got there first. + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + expect(await uuidsOf('category', 7)).toEqual(['n3']) + // And the strings did not move. + expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) + expect(await uuidsOf('category', 'furniture')).toEqual(['s2']) + }) + + it('serves strings written AFTER numbers on the same field', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + + // 'electronics' would have become NaN and been dropped under the freeze. + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + }) + + it('does not coerce a number query into the string postings, or back', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' }) + + expect(await uuidsOf('code', 5)).toEqual(['num']) + expect(await uuidsOf('code', '5')).toEqual(['str']) + }) + + it('serves booleans mixed into a field that already holds strings and numbers', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true }) + store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false }) + + expect(await uuidsOf('flag', true)).toEqual(['b1']) + expect(await uuidsOf('flag', false)).toEqual(['b2']) + // `true` stores as 1 internally; that is an encoding, not a value. + expect(await uuidsOf('flag', 1)).toEqual(['n1']) + expect(await uuidsOf('flag', 'yes')).toEqual(['s1']) + }) + + it('answers nothing — not something coerced — for a kind the field never held', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + + expect(await uuidsOf('category', 5)).toEqual([]) + expect(await uuidsOf('category', true)).toEqual([]) + }) + + it('holds every kind across a flush, not just the one in the tail buffer', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + await store.flush() + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + }) + }) + + describe('range filters read the numeric postings', () => { + it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 }) + store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 }) + store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' }) + await store.flush() + + const inRange = await store.rangeQuery('price', 200, 1000) + const uuids = Array.from(inRange) + .map((id) => idMapper.getUuid(Number(id))) + .sort() + expect(uuids).toEqual(['dear', 'mid']) + }) + + it('an unbounded range still reports every kind — it is the “has a value” probe', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 }) + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true }) + await store.flush() + + const anyValue = await store.rangeQuery('mixed') + const uuids = Array.from(anyValue) + .map((id) => idMapper.getUuid(Number(id))) + .sort() + expect(uuids).toEqual(['b1', 'n1', 's1']) + }) + }) + + describe('the index reports what a field actually holds', () => { + it('names every kind present, not the one that got there first', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + expect(store.getFieldKinds('category')).toEqual(['string']) + + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) + expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) + + // And the field is still ONE field by name. + expect(store.getIndexedFields()).toEqual(['category']) + expect(store.hasField('category')).toBe(true) + }) + + it('reports an unknown field as holding nothing', () => { + expect(store.getFieldKinds('never-written')).toEqual([]) + }) + }) + + describe('an integer column widens rather than rounding', () => { + it('keeps a non-integer written after integers as itself', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 }) + await store.flush() + + // 4.5 used to round to 5 and answer `score === 5` alongside c. + expect(await uuidsOf('score', 4.5)).toEqual(['b']) + expect(await uuidsOf('score', 5)).toEqual(['c']) + expect(await uuidsOf('score', 4)).toEqual(['a']) + }) + }) + + describe('close then reopen', () => { + it('keeps every typed posting, on the same storage', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) + store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 }) + await store.flush() + await store.close() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) + expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) + expect(await uuidsOf('category', 5)).toEqual(['n1']) + expect(await uuidsOf('category', true)).toEqual(['b1']) + expect(await uuidsOf('score', 1.5)).toEqual(['f1']) + }) + + it('accepts new values of every kind after the reopen', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + await store.flush() + await store.close() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false }) + await store.flush() + + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + expect(await uuidsOf('category', false)).toEqual(['b1']) + }) + + it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => { + // A single-kind field is byte-identical to what the old writer produced: + // one manifest at `_column_index//MANIFEST.json`, no kind + // subdirectory anywhere. That IS the old on-disk shape, so proving the + // new reader serves it proves an old index still opens. + store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' }) + await store.flush() + + const keys = await (storage as unknown as { + listObjectsUnderPath: (prefix: string) => Promise + }).listObjectsUnderPath('_column_index/') + expect(keys.some((k) => k.includes('/k/'))).toBe(false) + + await store.close() + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + expect(store.getFieldKinds('status')).toEqual(['string']) + expect(await uuidsOf('status', 'active')).toEqual(['a']) + }) + }) +}) From da7d2498bc8c2225d355437eeecfe4c0e5709899 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:12:20 -0700 Subject: [PATCH 12/21] =?UTF-8?q?docs(changelog):=20the=2010.4.12=20note,?= =?UTF-8?q?=20curated=20=E2=80=94=20and=20the=20rail=20keeps=20a=20curated?= =?UTF-8?q?=20entry=20instead=20of=20generating=20one=20across=20a=20diver?= =?UTF-8?q?ged=20lineage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 15 +++++++++++++++ scripts/release.sh | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d81cfb..fc577c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + +### [10.4.12](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.11...v10.4.12) (2026-09-03) + +- Mixed-kind fields index exactly, arrays to 256, a drained loop is not a shutdown, and finds project from the column store +- fix(index): a metadata field holds every value kind it was written with — one posting column per (field, kind); an equality filter reads the query value's own kind, a range routes by its bounds; nothing is refused and nothing is silently dropped; an index written by the old shape opens unchanged (a128f0ed) +- fix(metadata): metadata arrays index up to 256 elements; a longer array refuses at write time by name (MetadataArrayTooLargeError) — a vector parked in metadata now throws; move it to `vector` (e435da78) +- fix(shutdown): beforeExit runs a non-closing flush only — a script that never calls close() exits with the writer lock on disk and no clean-shutdown marker, and the next open evicts the stale lock and folds the log, bounded; SIGTERM and SIGINT are unchanged (6baa4d7f) +- feat(find): field projection — find({fields}) and get({fields}) resolve scalars from the column store on every leg, including vector-leg finds; absent fields stay absent (ad0f493f) +- fix(find): orderBy is the order on every find path, not only the metadata-only one (5e720d17) +- fix(metadata): the legacy sparse range path orders values, or refuses by name — never ranks by hash (a7eb7f52) +- fix(close): a read-only brain writes nothing under `_system/` (f27a7776) +- fix(contract): the flush gate's internals are private, not doors (72c8ee6a) +- test(hygiene): the triple-intelligence correctness cases sit in the gate; the idle and connected-find pins name the brain they measure (28083981) +- ci(release): the rail writes its own wall entry into the shared releases repo — never hand-written again (adcb883e) + ### [10.4.11](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.9...v10.4.11) (2026-09-02) - ci: superseded pushes cancel their own runs (concurrency per ref) (6053f6d4) diff --git a/scripts/release.sh b/scripts/release.sh index 142fa06f..a9a1f6e9 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -159,9 +159,21 @@ CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs ${COMMITS} " +# A CURATED entry wins over the generated one. When a release is cut from a +# lineage that diverged from the previous tag (a candidate branch carrying +# main's history), `git log ..HEAD` lists every commit the tag never +# saw — old notes, already-shipped fixes under new hashes, merge commits — and a +# wall entry derived from it would misreport the release. If CHANGELOG.md +# already carries a `### [NEW_VERSION]` heading, it was written on purpose: +# keep it, and skip the generated prepend entirely. +CURATED_ENTRY=false +if grep -qE "^### \[${NEW_VERSION}\]" CHANGELOG.md 2>/dev/null; then + CURATED_ENTRY=true + echo -e "${YELLOW}CHANGELOG already carries a curated ### [${NEW_VERSION}] entry — keeping it, not generating one from commits${NC}" +fi # Prepend to CHANGELOG.md after header -if [ -f "CHANGELOG.md" ]; then +if [ "$CURATED_ENTRY" = false ] && [ -f "CHANGELOG.md" ]; then # Read header (first 4 lines) HEADER=$(head -n 4 CHANGELOG.md) # Read rest of file From 7e1ddee4f767950907942f162dbfacb55d3177e6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:15:21 -0700 Subject: [PATCH 13/21] chore(release): 10.4.12 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3e3bf96d..c4757030 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.11", + "version": "10.4.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.11", + "version": "10.4.12", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 8676f8b7..649f2aaf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.11", + "version": "10.4.12", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 656d9f6f92e1ccdc29c9d86ffe1a172d5fc1219a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:18:57 -0700 Subject: [PATCH 14/21] test(hygiene): close every brain the remaining suites create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit id-normalization.test.ts's makeBrain() and degraded-reads-surfaced.test.ts's per-test brains had nothing tracking them — both now use a describe-scoped opened[] array drained by afterEach. find-hybrid-filter-before-hydrate.test.ts had two beforeAll-built brains (one per describe block) with no matching afterAll. multi-process-safety.test.ts and plugin-autodetect.test.ts/plugin.test.ts left a brain whose init() was expected to reject (a rejected init() still registers the instance in Brainy's global instance registry — the constructor does that unconditionally — so it still needs close() to deregister, or the process-level shutdown hooks never see the registry go idle for the rest of the run). --- .../find-hybrid-filter-before-hydrate.test.ts | 10 +++++++++- tests/integration/id-normalization.test.ts | 18 +++++++++++++++++- tests/integration/multi-process-safety.test.ts | 7 ++++++- .../brainy/degraded-reads-surfaced.test.ts | 10 +++++++++- tests/unit/plugin-autodetect.test.ts | 4 ++++ tests/unit/plugin.test.ts | 6 +++++- 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/integration/find-hybrid-filter-before-hydrate.test.ts b/tests/integration/find-hybrid-filter-before-hydrate.test.ts index 3e74f5d8..7f326729 100644 --- a/tests/integration/find-hybrid-filter-before-hydrate.test.ts +++ b/tests/integration/find-hybrid-filter-before-hydrate.test.ts @@ -31,7 +31,7 @@ * never the legs. And the text leg is asked about the universe's ids only — * what it marshals is bounded by the universe, not by the store. */ -import { describe, it, expect, beforeAll, vi } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking' @@ -287,6 +287,10 @@ describe('hybrid find: filter before hydrate — the answer is unchanged', () => expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function') }) + afterAll(async () => { + await brain.close() + }) + it('the fixture does not truncate the text leg — the universe covers every text match', async () => { const index = (brain as any).metadataIndex const textMatches = await index.getIdsForTextQuery(QUERY) @@ -553,6 +557,10 @@ describe('hybrid find: the text leg ranks inside the filter, not around it', () } }) + afterAll(async () => { + await brain.close() + }) + it('the old order let the filter consume the whole text leg', async () => { const index = (brain as any).metadataIndex const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) diff --git a/tests/integration/id-normalization.test.ts b/tests/integration/id-normalization.test.ts index 1ea1a221..1eb14ab1 100644 --- a/tests/integration/id-normalization.test.ts +++ b/tests/integration/id-normalization.test.ts @@ -18,7 +18,7 @@ * All entities carry explicit 384-dim vectors so no test invokes the embedder. */ -import { describe, it, expect } from 'vitest' +import { describe, it, expect, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' import { v5, v7, isUUID } from '../../src/universal/uuid.js' @@ -37,8 +37,15 @@ async function makeBrain(): Promise { } describe('id normalization — transparent string-key round-trips', () => { + const opened: Brainy[] = [] + + afterEach(async () => { + for (const b of opened.splice(0)) await b.close().catch(() => {}) + }) + it('1. add() returns v5(key); get(key) and get(returnedId) both resolve; _originalId preserved', async () => { const brain = await makeBrain() + opened.push(brain) const returnedId = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) @@ -60,6 +67,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('2. relate() by string keys; related(key) and related({from:key}) return the edge to v5(toKey)', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document }) @@ -85,6 +93,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('3. update() by string key reflects on get(key)', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { role: 'admin' } }) await brain.update({ id: 'user-1', metadata: { role: 'owner' } }) @@ -98,6 +107,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('4. remove() by string key deletes; get(key) is null', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) expect(await brain.get('user-1')).not.toBeNull() @@ -110,6 +120,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('5. find({ connected: { from: key } }) resolves the anchor key', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document }) @@ -122,6 +133,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('6. transact() add+relate by string keys round-trips with consistent canonical ids', async () => { const brain = await makeBrain() + opened.push(brain) // Seed user-1 so the relate op has a target to point at. await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) @@ -149,6 +161,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('7. addMany() + relateMany() with string ids round-trip', async () => { const brain = await makeBrain() + opened.push(brain) const added = await brain.addMany({ items: [ @@ -175,6 +188,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('8. determinism: same key maps to same UUID — two adds upsert ONE entity, not two', async () => { const brain = await makeBrain() + opened.push(brain) const id1 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 1 } }) const id2 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 2 } }) @@ -193,6 +207,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('9. valid-UUID passthrough: a real UUID is kept verbatim with NO _originalId', async () => { const brain = await makeBrain() + opened.push(brain) const realUuid = v7() const returnedId = await brain.add({ id: realUuid, vector: vec(5), type: NounType.Thing }) @@ -207,6 +222,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('10. no-id add() mints a v7; newId() mints a v7', async () => { const brain = await makeBrain() + opened.push(brain) const autoId = await brain.add({ vector: vec(6), type: NounType.Thing }) expect(isUUID(autoId)).toBe(true) diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts index 592d7969..dd1b8901 100644 --- a/tests/integration/multi-process-safety.test.ts +++ b/tests/integration/multi-process-safety.test.ts @@ -107,7 +107,11 @@ describe('Multi-process safety + read-only mode', () => { const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await expect(blocked.init()).rejects.toThrow(/another writer holds/i) - // Don't track `blocked` for afterEach cleanup since init failed. + // A rejected init() still registered `blocked` in Brainy's global + // instance registry (the constructor does that unconditionally) — close() + // is safe to call even though init() never completed, and is what + // deregisters it (and, once idle, the process-level shutdown hooks). + await blocked.close().catch(() => {}) }) it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => { @@ -151,6 +155,7 @@ describe('Multi-process safety + read-only mode', () => { const err: any = await blocked.init().catch((e) => e) expect(err.code).toBe('BRAINY_WRITER_LOCKED') expect(err.lockInfo?.pid).toBe(otherPid) + await blocked.close().catch(() => {}) }) it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => { diff --git a/tests/unit/brainy/degraded-reads-surfaced.test.ts b/tests/unit/brainy/degraded-reads-surfaced.test.ts index 29a8a77c..004adeaa 100644 --- a/tests/unit/brainy/degraded-reads-surfaced.test.ts +++ b/tests/unit/brainy/degraded-reads-surfaced.test.ts @@ -19,13 +19,19 @@ import { prodLog } from '../../../src/utils/logger.js' const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` describe('Finding 10 — degraded derived-index state is surfaced on reads', () => { + const opened: Brainy[] = [] + beforeEach(() => { process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' }) - afterEach(() => vi.restoreAllMocks()) + afterEach(async () => { + vi.restoreAllMocks() + for (const b of opened.splice(0)) await b.close().catch(() => {}) + }) it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => { const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + opened.push(brain) await brain.init() ;(brain as any)._indexDegradedIds.add(UUID('de')) @@ -37,6 +43,7 @@ describe('Finding 10 — degraded derived-index state is surfaced on reads', () it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => { const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + opened.push(brain) await brain.init() await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document }) ;(brain as any)._indexRebuildFailed = new Error('rebuild boom') @@ -59,6 +66,7 @@ describe('Finding 10 — degraded derived-index state is surfaced on reads', () it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => { const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + opened.push(brain) await brain.init() // Simulate a degraded receipt by wrapping the generation store's commitSingleOp. const gs: any = (brain as any).generationStore diff --git a/tests/unit/plugin-autodetect.test.ts b/tests/unit/plugin-autodetect.test.ts index 37c181ba..ee830c17 100644 --- a/tests/unit/plugin-autodetect.test.ts +++ b/tests/unit/plugin-autodetect.test.ts @@ -89,12 +89,14 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => { }) const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) await expect(brain.init()).rejects.toThrow(/installed but failed to load/) + await brain.close().catch(() => {}) }) it('installed but not a valid plugin (missing activate) → init() throws', async () => { stubImport(async () => ({ default: { name: '@soulcraft/cor' } })) // no activate() const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) await expect(brain.init()).rejects.toThrow(/not a valid Brainy plugin/) + await brain.close().catch(() => {}) }) it('installed but activation fails → init() throws (activateAll posture applies)', async () => { @@ -108,6 +110,7 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => { })) const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) await expect(brain.init()).rejects.toThrow(/failed to activate/) + await brain.close().catch(() => {}) }) it('plugins: [] and plugins: false → no probe at all (explicit opt-out)', async () => { @@ -132,5 +135,6 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => { silent: true }) await expect(brain.init()).rejects.toThrow(/listed in config\.plugins but could not be loaded/) + await brain.close().catch(() => {}) }) }) diff --git a/tests/unit/plugin.test.ts b/tests/unit/plugin.test.ts index f4064188..82543120 100644 --- a/tests/unit/plugin.test.ts +++ b/tests/unit/plugin.test.ts @@ -298,9 +298,10 @@ describe('Brainy plugin integration', () => { // must surface as a failed init(), NOT a silent degrade to the default // engine (the version-coupling guard; see plugin-version-coupling.test.ts). await expect(brain.init()).rejects.toThrow(/failed to activate|native module not found/) + await brain.close().catch(() => {}) }) - it('should use() return this for chaining', () => { + it('should use() return this for chaining', async () => { const plugin: BrainyPlugin = { name: 'chain-test', activate: async () => true @@ -309,5 +310,8 @@ describe('Brainy plugin integration', () => { const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) const result = brain.use(plugin) expect(result).toBe(brain) + // Never init()'d — the constructor still registered it in Brainy's global + // instance registry, so it still needs a close() to deregister. + await brain.close().catch(() => {}) }) }) From ba10aaf52ed14073509573950927c8f8714236e3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:32:17 -0700 Subject: [PATCH 15/21] test(hygiene): close two more brains found by a broadened rescan A second, structural pass of the honest scan (any local helper that constructs a Brainy directly, not just ones named like openBrain/makeBrain, plus support for new Brainy(...) generics) surfaced two more real leaks outside the first 93-file list: writer-lock-fencing.test.ts's `second` (a rejected-init() brain never pushed into the file's own tracked array) and plugin-version-coupling.test.ts's last case (a rejected-init() brain with no close at all). --- tests/integration/writer-lock-fencing.test.ts | 1 + tests/unit/plugin-version-coupling.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/integration/writer-lock-fencing.test.ts b/tests/integration/writer-lock-fencing.test.ts index e9f98dac..d5b82c30 100644 --- a/tests/integration/writer-lock-fencing.test.ts +++ b/tests/integration/writer-lock-fencing.test.ts @@ -61,6 +61,7 @@ describe('writer-lock fencing', () => { // Old rule: heartbeat-age eviction → silent takeover → split brain. // New rule: live PID = live writer; the second opener throws typed. const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + brains.push(second) await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' }) }, 120000) diff --git a/tests/unit/plugin-version-coupling.test.ts b/tests/unit/plugin-version-coupling.test.ts index ffcc2a88..d4685ae2 100644 --- a/tests/unit/plugin-version-coupling.test.ts +++ b/tests/unit/plugin-version-coupling.test.ts @@ -143,5 +143,6 @@ describe('version coupling at init() — no silent fallback', () => { plugins: ['@soulcraft/this-package-does-not-exist-xyz'] }) await expect(brain.init()).rejects.toThrow(/could not be loaded|config\.plugins/) + await brain.close().catch(() => {}) }) }) From 6eb5e4483de50fc2099c4a6b61bf74c71a453e37 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:38:26 -0700 Subject: [PATCH 16/21] test(hygiene): close the brain typeAware.bench.test.ts creates Excluded from the correctness gate (tests/performance/**, run only via npm run test:perf) but still leaked: brainMemory was created in a beforeEach with no matching afterEach. --- tests/performance/typeAware.bench.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/performance/typeAware.bench.test.ts b/tests/performance/typeAware.bench.test.ts index 72d96fe5..b1153662 100644 --- a/tests/performance/typeAware.bench.test.ts +++ b/tests/performance/typeAware.bench.test.ts @@ -17,7 +17,7 @@ * - Note limitations and edge cases */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { TypeAwareStorageAdapter } from '../../src/storage/adapters/typeAwareStorageAdapter.js' import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' @@ -67,6 +67,10 @@ describe('TypeAware Performance Benchmarks', () => { } }) + afterEach(async () => { + await brainMemory.close() + }) + it('should measure type-based query performance', async () => { // MEASURED: Query for one type (200 entities) const start = performance.now() From aac853d3e8ef7f4668559c744cbbc71dd2bbcf6a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 10:57:03 -0700 Subject: [PATCH 17/21] fix(release): wall-entry commits under an explicit git identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git commit in the cache clone relied on ambient user.name/user.email, which the box has neither globally nor per-repo — every push-side test failed there with "unable to auto-detect email address" while passing on a laptop with a global identity configured. Resolve the identity from the repository the rail is actually running in (process.cwd(), the developer's own checkout release.sh invokes this from) and pass it explicitly via -c user.name/-c user.email on the commit; refuse by name if neither is set. Give the test fixtures a repo-local identity the same way seedRemote already does for the seed clone, so the suite is deterministic on any host. --- scripts/wall-entry.mjs | 37 ++++++++++++++++++++++++++- tests/unit/release/wall-entry.test.ts | 8 ++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index d4ec7ba5..5079cf86 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -337,6 +337,36 @@ function git(args, cwd) { } } +/** + * Resolve the git identity for the wall commit from the repository the rail + * is actually running in — the developer's own checkout (`process.cwd()`; + * `release.sh` invokes this script from the repo root with no `cd`), via + * git's normal config precedence (repo-local, then global, then system). + * Never guessed and never left to git's own "who are you?" prompt: a host + * with no configured identity anywhere (a bare CI box, say) must refuse + * loudly rather than have git manufacture a placeholder identity or hang. + * @returns {{name: string, email: string}} + */ +function resolveWallCommitIdentity() { + const repo = process.cwd() + let name = '' + let email = '' + try { + name = git(['config', 'user.name'], repo) + } catch { + name = '' + } + try { + email = git(['config', 'user.email'], repo) + } catch { + email = '' + } + if (!name || !email) { + fail('no git identity for the wall commit — set user.name/user.email') + } + return { name, email } +} + /** * Ensure a clean, up-to-date local clone of the releases repo at * `cacheDir`, checked out on `main` — cloning fresh if `cacheDir` has no @@ -423,9 +453,14 @@ function publishEntry(entry, product, remote, cacheDir) { return } + const identity = resolveWallCommitIdentity() + try { git(['add', `${product}.json`], cacheDir) - git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir) + git( + ['-c', `user.name=${identity.name}`, '-c', `user.email=${identity.email}`, 'commit', '-m', `chore(wall): ${product} ${entry.version}`], + cacheDir, + ) } catch (err) { fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`) } diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index 8bf9d357..b29ae326 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -104,6 +104,14 @@ let cacheDir: string beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) + // wall-entry.mjs is run with this dir as its cwd, standing in for the real + // developer checkout it reads its commit identity from (process.cwd()) — + // give it a repo-local identity the same way seedRemote gives one to the + // seed clone, so the suite is deterministic on a host with no global git + // config (a bare CI box) as much as one with a developer's own. + execFileSync('git', ['init', '-q', dir]) + git(['config', 'user.name', 'Wall Entry Test'], dir) + git(['config', 'user.email', 'wall-entry-test@example.com'], dir) remoteDir = initBareRemote() cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases') }) From a2ea21b330d49933b70cd9fed1c7fb46afb89cb8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 10:57:10 -0700 Subject: [PATCH 18/21] fix(shutdown): hold the signal listener until the exit decision is made MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closing the last live instance inside the SIGTERM/SIGINT handler calls close() -> deregisterShutdownHooksIfIdle(), which removes Brainy's own signal listener from process synchronously, before that same handler invocation has reached the point where it decides whether to exit. That opens a window with no registered listener for the signal at all: a second/concurrent delivery of the same signal during that window falls through to Node's default disposition and kills the process outright, after the clean shutdown already finished, so the process reports a signal kill instead of the 0 clause (a) and (b) of shutdown-single-owner.test.ts pin — intermittent under load, which is why it only ever showed up on the box. Add a static flag that stays true for the whole closeOnShutdown() run and makes deregisterShutdownHooksIfIdle() defer rather than remove the listener while that run is still deciding; closeOnShutdown()'s own finally re-runs the deregistration check once it is actually done, so the listener never leaks past its use. --- src/brainy.ts | 134 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 43 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index b8eb7f56..fc08f291 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -544,6 +544,23 @@ export class Brainy implements BrainyInterface { * and narrate forever. Reset by {@link deregisterShutdownHooksIfIdle}. */ private static beforeExitNarrated = false + /** True for the entire duration of ONE `closeOnShutdown()` run (the + * signal-path handler in {@link registerShutdownHooks}) — from before it + * starts closing instances until after it has decided whether to exit. + * THE RACE THIS CLOSES: closing the LAST live instance calls + * `close()` → `deregisterShutdownHooksIfIdle()` synchronously, which + * removes `Brainy.sigtermListener` from `process` — while `closeOnShutdown` + * (that very listener's OWN still-running invocation) hasn't yet reached + * `exitIfSoleShutdownOwner()`'s `process.exit(0)`. In that window Node has + * NO registered SIGTERM listener, so a second/concurrent delivery of the + * same signal (a raced re-send, common on a loaded host) falls through to + * Node's default disposition and kills the process outright — the + * clean-shutdown work already finished, but the process never reports the + * 0 it earned. `deregisterShutdownHooksIfIdle()` checks this flag and + * defers; `closeOnShutdown()`'s `finally` re-runs the deregistration check + * once it is done, so the listener never actually leaks past its use. */ + private static shutdownSignalHandlerActive = false + /** Poll cadence (ms) for the migration LOCK when a provider exposes no * event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */ private static readonly MIGRATION_POLL_INTERVAL_MS = 250 @@ -2196,51 +2213,74 @@ export class Brainy implements BrainyInterface { */ const closeOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') - // DEFER ONE MACROTASK. A host application registers its own listener on - // the same signal, and Node runs listeners in registration order — ours - // is usually first, because the brain was opened before the host wired - // its shutdown. Yielding once lets every other listener for this signal - // run its synchronous prologue, so a host that calls close() gets to be - // the owner. It is only a courtesy, never the safety: close()'s own - // single-flight gate is what makes a lost race harmless. - await new Promise((resolve) => setImmediate(resolve)) + // HOLD THE LISTENER FOR THE WHOLE RUN. Closing the LAST live instance + // below calls close() → deregisterShutdownHooksIfIdle(), which removes + // Brainy's own SIGTERM/SIGINT listeners from `process` — synchronously, + // before THIS invocation has reached exitIfSoleShutdownOwner()'s + // process.exit(0). Left alone, that opens a window with no registered + // listener for the signal at all, so a second/concurrent delivery of + // the same signal (a raced re-send — not rare on a loaded host) falls + // through to Node's default disposition and kills the process outright + // AFTER the clean-shutdown work already finished, reporting a signal + // kill instead of the 0 the shutdown earned. Setting this flag makes + // deregisterShutdownHooksIfIdle() defer; the `finally` below re-checks + // it once this run is fully done — closeOnShutdown, not a nested + // close(), owns exactly when the listener actually comes off. + Brainy.shutdownSignalHandlerActive = true + try { + // DEFER ONE MACROTASK. A host application registers its own listener + // on the same signal, and Node runs listeners in registration order — + // ours is usually first, because the brain was opened before the + // host wired its shutdown. Yielding once lets every other listener + // for this signal run its synchronous prologue, so a host that calls + // close() gets to be the owner. It is only a courtesy, never the + // safety: close()'s own single-flight gate is what makes a lost race + // harmless. + await new Promise((resolve) => setImmediate(resolve)) - let closedCount = 0 - let deferredCount = 0 - let failedCount = 0 - // Snapshot: close() splices Brainy.instances while we iterate. - for (const instance of [...Brainy.instances]) { - if (!instance.initialized) continue - // SOMEONE ELSE OWNS THIS ONE. Not a flush, not a lock release, not a - // component close — nothing. Touching a brain whose close is running - // is the whole defect this handler was rewritten for. - if (instance.closed || instance._closeInFlight !== null) { - deferredCount++ - continue + let closedCount = 0 + let deferredCount = 0 + let failedCount = 0 + // Snapshot: close() splices Brainy.instances while we iterate. + for (const instance of [...Brainy.instances]) { + if (!instance.initialized) continue + // SOMEONE ELSE OWNS THIS ONE. Not a flush, not a lock release, not a + // component close — nothing. Touching a brain whose close is running + // is the whole defect this handler was rewritten for. + if (instance.closed || instance._closeInFlight !== null) { + deferredCount++ + continue + } + try { + // Law 1: this try/catch is the isolation — the loop continues. + await instance.close() + closedCount++ + } catch (error) { + failedCount++ + console.error('Failed to close one Brainy instance on shutdown:', error) + } } - try { - // Law 1: this try/catch is the isolation — the loop continues. - await instance.close() - closedCount++ - } catch (error) { - failedCount++ - console.error('Failed to close one Brainy instance on shutdown:', error) + if (closedCount > 0) { + console.log(`Flushed successfully (${closedCount} instance${closedCount > 1 ? 's' : ''})`) } - } - if (closedCount > 0) { - console.log(`Flushed successfully (${closedCount} instance${closedCount > 1 ? 's' : ''})`) - } - if (deferredCount > 0) { - console.log( - `${deferredCount} Brainy instance${deferredCount > 1 ? 's are' : ' is'} already ` + - `closing — left to the caller that owns that close.` - ) - } - if (failedCount > 0) { - console.error( - `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` + - `their writer locks were released, but their next open will run crash recovery.` - ) + if (deferredCount > 0) { + console.log( + `${deferredCount} Brainy instance${deferredCount > 1 ? 's are' : ' is'} already ` + + `closing — left to the caller that owns that close.` + ) + } + if (failedCount > 0) { + console.error( + `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` + + `their writer locks were released, but their next open will run crash recovery.` + ) + } + } finally { + // Release the hold and run the deferred check ourselves — the last + // close() above may have found the flag set and skipped its own + // deregistration, so nobody else will do this if we don't. + Brainy.shutdownSignalHandlerActive = false + Brainy.deregisterShutdownHooksIfIdle() } } @@ -2404,9 +2444,17 @@ export class Brainy implements BrainyInterface { * script that closed every brain exits on its own — a library must never * keep its host process alive. Re-initializing later re-registers them * (the `shutdownHooksRegisteredGlobally` flag resets here). + * + * Deferred (not skipped — {@link closeOnShutdown}'s `finally` always + * re-checks) while a signal-path shutdown is actively running: that + * handler's OWN still-in-flight invocation is `Brainy.sigtermListener`, and + * removing it out from under itself — which closing the LAST instance here + * would otherwise do, synchronously, mid-run — would leave `process` with + * no listener for the signal for the remainder of that run. See + * {@link shutdownSignalHandlerActive}'s doc for the exact race this closes. */ private static deregisterShutdownHooksIfIdle(): void { - if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally) { + if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally || Brainy.shutdownSignalHandlerActive) { return } if (Brainy.sigtermListener) process.off('SIGTERM', Brainy.sigtermListener) From 360feaccf8b403a40bbbad29bf5e87069d2faabc Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 11:41:44 -0700 Subject: [PATCH 19/21] docs(changelog): the 10.4.13 note, curated --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc577c1d..c4f89332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.13](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.12...v10.4.13) (2026-09-03) + +- A shutdown that holds its listener until the exit decision, and a test suite that closes every brain it opens +- fix(shutdown): the engine's signal handler keeps its listener registered until the exit decision is made — closing the last live instance no longer deregisters the handler mid-run, so a second signal delivery during a clean shutdown can never kill the process after the work is done (a2ea21b3) +- fix(release): the release wall entry commits under an explicit git identity read from the developer's checkout; a host with no identity refuses by name instead of failing inside git (aac853d3) +- test(hygiene): every brain a test file creates is closed by that file — 40 files fixed, the leaks that let a stray cadence narrate into later files are gone; brains whose init() was expected to fail are closed too (6eb5e448) + ### [10.4.12](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.11...v10.4.12) (2026-09-03) - Mixed-kind fields index exactly, arrays to 256, a drained loop is not a shutdown, and finds project from the column store From 842ad44b885a318200ae29daecc8ff552cf5733e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 11:59:35 -0700 Subject: [PATCH 20/21] chore(release): 10.4.13 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index c4757030..bd12e46c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.12", + "version": "10.4.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.12", + "version": "10.4.13", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 649f2aaf..a3bd0483 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.12", + "version": "10.4.13", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 1b903fe665cc5dd6864a5090745ccb80601da29e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 12:18:00 -0700 Subject: [PATCH 21/21] =?UTF-8?q?docs:=20frozen=20at=2010.4.13=20=E2=80=94?= =?UTF-8?q?=20the=20last=20release=20of=20the=20reference=20implementation?= =?UTF-8?q?;=20the=20repository=20is=20read-only=20from=20here?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +++++ RELEASES.md | 3 +++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 762c9ec3..fbf129ac 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@

Brainy

+> **Frozen at 10.4.13 (2026-09-03).** This repository is the reference implementation of the Brainy store format and API, +> published under the MIT license. Version 10.4.13 is its last release; the repository is read-only from here. The engine +> continues as `@soulcraft/brainy`, which bundles this layer as owned code; every published version of this package stays +> available on The Source. Use this repository to read a Brainy store independently or to verify the conformance contract. +

Three database paradigms. One API. Zero configuration.
The in-process knowledge database for TypeScript — vector search, graph traversal,
diff --git a/RELEASES.md b/RELEASES.md index c875cb26..64e64873 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,8 @@ # @soulcraft/brainy — Release Notes for Consumers +> **Frozen at 10.4.13 (2026-09-03).** 10.4.13 is the last release of `@soulcraftlabs/brainy`; this repository is read-only from here. +> Release notes for the product engine continue on its own wall. + Machine-readable release notes are published at https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json (this engine) and