Compare commits
13 commits
rel/10.4.9
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 08758c254f | |||
| 3dadbec8f2 | |||
| 4f1e27c9a0 | |||
| 297a3d7657 | |||
| 7ab670b525 | |||
| f097cbf6f2 | |||
| e64e2bc175 | |||
| 655aa13ea7 | |||
| 39c71ecdac | |||
| 0759c03a82 | |||
| 9a888c37e9 | |||
|
|
298cb6daca | ||
| b8475cc86a |
20 changed files with 308 additions and 1118 deletions
|
|
@ -5,6 +5,10 @@ name: CI
|
||||||
# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the
|
# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the
|
||||||
# tag's publish-source run and starve every release (observed on 8.10.3 and
|
# tag's publish-source run and starve every release (observed on 8.10.3 and
|
||||||
# 9.0.0: the publish sat behind the tag's own redundant CI).
|
# 9.0.0: the publish sat behind the tag's own redundant CI).
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: ['**']
|
branches: ['**']
|
||||||
|
|
|
||||||
23
CHANGELOG.md
23
CHANGELOG.md
|
|
@ -2,29 +2,6 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02)
|
|
||||||
|
|
||||||
- Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d)
|
|
||||||
- fix(open): pending-embed recovery keeps the crash-recovery contract — foreground, bounded by the mark (8a2ebacf)
|
|
||||||
- Merge branches 'fix/connected-find-order', 'fix/pending-embed-low-water' and 'fix/related-verb-array' into rel/10.4.9-candidate (d5147ed6)
|
|
||||||
- fix(graph): the verb fast paths honour every requested type, source, and target (6a89adc4)
|
|
||||||
- perf(open): pending-embed recovery is bounded by a low-water mark and runs behind the doors (88e79729)
|
|
||||||
- fix(find): connected finds are graph-first — neighbours, then the filter over those ids, then the page (077cbc0b)
|
|
||||||
- fix(storage): counts persistence is single-flight, coalesced, and never races its own temp file (5e3b343a)
|
|
||||||
|
|
||||||
|
|
||||||
### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31)
|
|
||||||
|
|
||||||
- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d)
|
|
||||||
|
|
||||||
|
|
||||||
### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31)
|
|
||||||
|
|
||||||
- build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f)
|
|
||||||
- fix(generations): a sealed segment may only declare the generations it holds (a963a744)
|
|
||||||
- fix(recovery): a torn generation-log tail is a terminal verdict, never a wait (c9930871)
|
|
||||||
|
|
||||||
|
|
||||||
### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28)
|
### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28)
|
||||||
|
|
||||||
- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1)
|
- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1)
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.9",
|
"version": "10.4.4",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.9",
|
"version": "10.4.4",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.9",
|
"version": "10.4.4",
|
||||||
"brainyContract": 1,
|
"brainyContract": 1,
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
|
|
|
||||||
76
releases/brainy.json
Normal file
76
releases/brainy.json
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
{
|
||||||
|
"product": "brainy",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"version": "11.0.5",
|
||||||
|
"date": "2026-09-02",
|
||||||
|
"headline": "Graph-first finds in production, and opens that stop rescanning history",
|
||||||
|
"items": [
|
||||||
|
"find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.",
|
||||||
|
"related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).",
|
||||||
|
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.4",
|
||||||
|
"date": "2026-09-01",
|
||||||
|
"headline": "Closes in milliseconds, index rebuilds without the disk-sync storm",
|
||||||
|
"items": [
|
||||||
|
"close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.",
|
||||||
|
"The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.",
|
||||||
|
"A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.3",
|
||||||
|
"date": "2026-09-01",
|
||||||
|
"headline": "The embedding upgrade ceremony runs on every brain",
|
||||||
|
"items": [
|
||||||
|
"A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.",
|
||||||
|
"A one-fix release; nothing else changed."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.2",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "One embedding quality everywhere, 3–4× faster imports",
|
||||||
|
"items": [
|
||||||
|
"Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.",
|
||||||
|
"Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.",
|
||||||
|
"The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.1",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "Deletes inside transactions are safe",
|
||||||
|
"items": [
|
||||||
|
"Deleting relations inside a transact() no longer corrupts index bookkeeping.",
|
||||||
|
"A store that deletes its last relation keeps serving instead of refusing."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "11.0.0",
|
||||||
|
"date": "2026-08-28",
|
||||||
|
"headline": "One install, one engine — Brainy",
|
||||||
|
"items": [
|
||||||
|
"The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.",
|
||||||
|
"A missing native build refuses loudly with its cures named; nothing falls back silently.",
|
||||||
|
"Stores open in place — no migration."
|
||||||
|
],
|
||||||
|
"url": null,
|
||||||
|
"thumb": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md."
|
||||||
|
}
|
||||||
122
releases/open-brainy.json
Normal file
122
releases/open-brainy.json
Normal file
|
|
@ -0,0 +1,122 @@
|
||||||
|
{
|
||||||
|
"product": "open-brainy",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"version": "10.4.10",
|
||||||
|
"date": "2026-09-02",
|
||||||
|
"headline": "A planner door for indexes, batched containment repair, and a fixed near()",
|
||||||
|
"items": [
|
||||||
|
"An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.",
|
||||||
|
"repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.",
|
||||||
|
"find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.9",
|
||||||
|
"date": "2026-09-02",
|
||||||
|
"headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history",
|
||||||
|
"items": [
|
||||||
|
"find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).",
|
||||||
|
"related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.",
|
||||||
|
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.7",
|
||||||
|
"date": "2026-09-01",
|
||||||
|
"headline": "Count ledgers can no longer race themselves",
|
||||||
|
"items": [
|
||||||
|
"Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.",
|
||||||
|
"Atomic writes carry a per-process sequence, so two processes' temp files can never collide."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.6",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "Transactions cross the index seam safely",
|
||||||
|
"items": [
|
||||||
|
"Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.",
|
||||||
|
"Fixes a class of transaction failures on stores with integer-mapped relation endpoints."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.5",
|
||||||
|
"date": "2026-08-31",
|
||||||
|
"headline": "Recovery tells the truth, docs live at home",
|
||||||
|
"items": [
|
||||||
|
"A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.",
|
||||||
|
"A sealed segment declares only the generations it actually holds.",
|
||||||
|
"The engine's documentation now publishes from its own repository."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.4",
|
||||||
|
"date": "2026-08-28",
|
||||||
|
"headline": "Faster opens, quieter idle",
|
||||||
|
"items": [
|
||||||
|
"Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.",
|
||||||
|
"The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.",
|
||||||
|
"A slow open now names the exact step it is in, so operators see what is being paid and why."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.3",
|
||||||
|
"date": "2026-08-27",
|
||||||
|
"headline": "Open Brainy, under its own name",
|
||||||
|
"items": [
|
||||||
|
"The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.",
|
||||||
|
"No code changes; your imports change once and everything else stays put."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.2",
|
||||||
|
"date": "2026-08-27",
|
||||||
|
"headline": "Vectors that lie are refused, counts that drift are caught",
|
||||||
|
"items": [
|
||||||
|
"A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.",
|
||||||
|
"The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.",
|
||||||
|
"Plugin activation failures keep their original error as cause, so the real frame reaches your logs."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.1",
|
||||||
|
"date": "2026-08-26",
|
||||||
|
"headline": "Writes that change nothing cost nothing",
|
||||||
|
"items": [
|
||||||
|
"The read gate is per index family, and a write carrying unchanged data never re-embeds.",
|
||||||
|
"The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1",
|
||||||
|
"thumb": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "10.4.0",
|
||||||
|
"date": "2026-08-26",
|
||||||
|
"headline": "Repair routing, the vector ledger, and honest empties",
|
||||||
|
"items": [
|
||||||
|
"Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.",
|
||||||
|
"An empty string is real data, not a missing field.",
|
||||||
|
"The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed."
|
||||||
|
],
|
||||||
|
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0",
|
||||||
|
"thumb": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"history": "Earlier releases are recorded in CHANGELOG.md in this repository."
|
||||||
|
}
|
||||||
|
|
@ -237,7 +237,7 @@ fi
|
||||||
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
||||||
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
||||||
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
||||||
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \
|
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \
|
||||||
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
||||||
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
||||||
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
||||||
|
|
|
||||||
228
src/brainy.ts
228
src/brainy.ts
|
|
@ -15,7 +15,6 @@ import { JsHnswVectorIndex } from './hnsw/hnswIndex.js'
|
||||||
import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js'
|
import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js'
|
||||||
import type { StorageOptions } from './storage/storageFactory.js'
|
import type { StorageOptions } from './storage/storageFactory.js'
|
||||||
import { rebuildCounts } from './utils/rebuildCounts.js'
|
import { rebuildCounts } from './utils/rebuildCounts.js'
|
||||||
import { jsonSafeIndexMetadata } from './utils/jsonSafeIndexMetadata.js'
|
|
||||||
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
|
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
|
||||||
import { BaseStorage } from './storage/baseStorage.js'
|
import { BaseStorage } from './storage/baseStorage.js'
|
||||||
import {
|
import {
|
||||||
|
|
@ -1820,11 +1819,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// a deferred write's ack and its background embed DELAYED a vector;
|
// a deferred write's ack and its background embed DELAYED a vector;
|
||||||
// this is where it lands.
|
// this is where it lands.
|
||||||
if (!this.isReadOnly) {
|
if (!this.isReadOnly) {
|
||||||
// Foreground, as the crash-recovery contract pins it: a reopened brain
|
|
||||||
// has its markers re-armed when open() returns. The low-water mark
|
|
||||||
// bounds this to the log's tail on any brain that has ever drained —
|
|
||||||
// milliseconds — so the foreground cost is the unmarked first open
|
|
||||||
// only, once per upgraded brain.
|
|
||||||
try {
|
try {
|
||||||
await step(
|
await step(
|
||||||
'bridge-pending-embed-sidecars',
|
'bridge-pending-embed-sidecars',
|
||||||
|
|
@ -1833,7 +1827,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
)
|
)
|
||||||
await step(
|
await step(
|
||||||
'recover-pending-embeds',
|
'recover-pending-embeds',
|
||||||
'folding the generation log\'s deferred-embed markers (from the low-water mark) into the pending set',
|
'folding the generation log\'s deferred-embed markers back into the pending set',
|
||||||
() => this.recoverPendingEmbedsFromLog()
|
() => this.recoverPendingEmbedsFromLog()
|
||||||
)
|
)
|
||||||
if (this._pendingEmbedIds.size > 0) {
|
if (this._pendingEmbedIds.size > 0) {
|
||||||
|
|
@ -2413,17 +2407,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
*/
|
*/
|
||||||
private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/'
|
private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/'
|
||||||
|
|
||||||
/**
|
|
||||||
* Storage-root-relative path of the ADVISORY pending-embed low-water mark:
|
|
||||||
* `{ generation, writtenAt }`, written whenever the pending set drains to
|
|
||||||
* empty (and at clean close when empty). Every marker in facts at or below
|
|
||||||
* `generation` is consumed, so recovery scans from `generation + 1`. The
|
|
||||||
* mark is advisory and monotone-safe: stale-low costs a longer scan, never
|
|
||||||
* a lost marker; it is never required for correctness.
|
|
||||||
*/
|
|
||||||
private static readonly PENDING_EMBED_LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Mark a deferred embed pending (MT5): the id joins the
|
* @description Mark a deferred embed pending (MT5): the id joins the
|
||||||
* in-memory fast-path set and the returned `embed.pending` record is
|
* in-memory fast-path set and the returned `embed.pending` record is
|
||||||
|
|
@ -2451,40 +2434,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
*/
|
*/
|
||||||
private clearPendingEmbed(id: string): void {
|
private clearPendingEmbed(id: string): void {
|
||||||
this._pendingEmbedIds.delete(id)
|
this._pendingEmbedIds.delete(id)
|
||||||
if (this._pendingEmbedIds.size === 0) this.maybeWriteEmbedLowWater()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @description Advance the advisory low-water mark: called at drain-to-empty
|
|
||||||
* (and at clean close when empty), it records the fact log's CURRENT head —
|
|
||||||
* with the set empty, every marker at or below the head has been consumed,
|
|
||||||
* so the next open's recovery fold scans only what comes after. Fire-and-
|
|
||||||
* forget at the drain (close() awaits the core); loud on failure: a missed
|
|
||||||
* write costs the next open a longer scan, never a marker. No-op without a
|
|
||||||
* fact log (no durable markers exist there) and on read-only opens.
|
|
||||||
*/
|
|
||||||
private maybeWriteEmbedLowWater(): void {
|
|
||||||
void this.writeEmbedLowWater()
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The awaitable core of {@link maybeWriteEmbedLowWater} — close() awaits it. */
|
|
||||||
private async writeEmbedLowWater(): Promise<void> {
|
|
||||||
if (this.isReadOnly) return
|
|
||||||
const log = this.generationStore ? this.generationStore.getFactLog() : null
|
|
||||||
if (!log) return
|
|
||||||
const generation = log.headGeneration()
|
|
||||||
if (!(generation > 0)) return
|
|
||||||
try {
|
|
||||||
await this.storage.writeRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH, {
|
|
||||||
generation,
|
|
||||||
writtenAt: Date.now()
|
|
||||||
})
|
|
||||||
} catch (err) {
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] pending-embed low-water write failed at generation ${generation}: ` +
|
|
||||||
`${(err as Error).message} — the next open scans from the previous mark`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -2495,14 +2444,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* survives the fold is exactly the set of acknowledged deferred writes
|
* survives the fold is exactly the set of acknowledged deferred writes
|
||||||
* whose vectors have not landed.
|
* whose vectors have not landed.
|
||||||
*
|
*
|
||||||
* BOUND: the scan starts at the advisory low-water mark
|
* BOUND (honest): no durable low-water mark exists for the earliest
|
||||||
* ({@link Brainy.PENDING_EMBED_LOWWATER_PATH}) — the log head at which the
|
* unconsumed pending, so the fold scans the log's committed facts from
|
||||||
* pending set last drained to empty — so a settled brain reads only the
|
* generation 1 — a sequential read of the log at open, O(log bytes).
|
||||||
* facts since then, not its whole history. Without a mark (first open
|
|
||||||
* after upgrade) it scans from generation 1, once; a stale-low mark costs
|
|
||||||
* a longer scan, never a marker. The fold stays on the open's foreground —
|
|
||||||
* the crash-recovery contract pins that a reopened brain has its markers
|
|
||||||
* re-armed when open() returns — and the mark is what makes that cheap.
|
|
||||||
* It is SKIPPED WHOLESALE when the log has never had a v2 tail
|
* It is SKIPPED WHOLESALE when the log has never had a v2 tail
|
||||||
* ({@link FactLog.hasV2History} — v1 facts cannot carry marker records),
|
* ({@link FactLog.hasV2History} — v1 facts cannot carry marker records),
|
||||||
* so pre-cutover brains pay nothing; on a mixed log the scan still reads
|
* so pre-cutover brains pay nothing; on a mixed log the scan still reads
|
||||||
|
|
@ -2515,18 +2459,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
private async recoverPendingEmbedsFromLog(): Promise<void> {
|
private async recoverPendingEmbedsFromLog(): Promise<void> {
|
||||||
const log = this.generationStore.getFactLog()
|
const log = this.generationStore.getFactLog()
|
||||||
if (!log || !log.hasV2History()) return
|
if (!log || !log.hasV2History()) return
|
||||||
let fromGeneration = 1
|
const scan = log.scanFacts({ fromGeneration: 1 })
|
||||||
try {
|
|
||||||
const mark = (await this.storage.readRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH)) as {
|
|
||||||
generation?: number
|
|
||||||
} | null
|
|
||||||
if (mark && typeof mark.generation === 'number' && mark.generation > 0) {
|
|
||||||
fromGeneration = mark.generation + 1
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// No mark (or unreadable): scan from 1 — correctness over cost.
|
|
||||||
}
|
|
||||||
const scan = log.scanFacts({ fromGeneration })
|
|
||||||
for await (const batch of scan.batches()) {
|
for await (const batch of scan.batches()) {
|
||||||
for (const fact of batch.facts) {
|
for (const fact of batch.facts) {
|
||||||
for (const record of fact.records ?? []) {
|
for (const record of fact.records ?? []) {
|
||||||
|
|
@ -4270,19 +4203,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
*/
|
*/
|
||||||
/**
|
/**
|
||||||
* @description A JSON-safe view of a record bound for the metadata-index
|
* @description A JSON-safe view of a record bound for the metadata-index
|
||||||
* crossing — delegates to the shared {@link jsonSafeIndexMetadata} leaf,
|
* crossing. The seam's metadata is JSON-safe BY CONTRACT (a native provider
|
||||||
* which the metadata-index transaction operations ALSO apply at execute
|
* serializes it; u64 ints as Number corrupt above 2^53) — but
|
||||||
* and rollback time. This plan-time wrap alone proved insufficient: it
|
* {@link resolveVerbEndpointInts} MIRRORS the resolved endpoint ints onto
|
||||||
* returns the same reference when the record is clean, and `transact()`'s
|
* the verb object itself as BigInt (`verb.sourceInt`/`targetInt`), so a
|
||||||
* delete legs share that reference with a graph-retraction op whose
|
* verb object reused as index metadata carried BigInts into
|
||||||
* execute-time endpoint resolution mirrors BigInt ints onto it (the full
|
* JSON.stringify, which throws, aborting the whole transaction (found by
|
||||||
* aliasing story lives on the leaf module's doc).
|
* the first joint pair gate). Endpoint ints ride their OWN op params on the
|
||||||
|
* graph legs — the metadata crossing drops every BigInt-valued top-level
|
||||||
|
* key instead of guessing at a lossy numeric encoding.
|
||||||
* @param metadata - The candidate index-metadata record.
|
* @param metadata - The candidate index-metadata record.
|
||||||
* @returns The same object when already JSON-safe, else a shallow copy
|
* @returns The same object when already JSON-safe, else a shallow copy
|
||||||
* without the BigInt-valued keys.
|
* without the BigInt-valued keys.
|
||||||
*/
|
*/
|
||||||
private static jsonSafeIndexMetadata(metadata: unknown): unknown {
|
private static jsonSafeIndexMetadata(metadata: unknown): unknown {
|
||||||
return jsonSafeIndexMetadata(metadata)
|
if (metadata === null || typeof metadata !== 'object') return metadata
|
||||||
|
const rec = metadata as Record<string, unknown>
|
||||||
|
let hasBigint = false
|
||||||
|
for (const k in rec) {
|
||||||
|
if (typeof rec[k] === 'bigint') { hasBigint = true; break }
|
||||||
|
}
|
||||||
|
if (!hasBigint) return metadata
|
||||||
|
const out: Record<string, unknown> = {}
|
||||||
|
for (const k in rec) {
|
||||||
|
if (typeof rec[k] !== 'bigint') out[k] = rec[k]
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
private metadataIndexRetractionOp(
|
private metadataIndexRetractionOp(
|
||||||
|
|
@ -7536,37 +7482,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// JS path — there the materialized `candidateIds` restricts the walk instead.
|
// JS path — there the materialized `candidateIds` restricts the walk instead.
|
||||||
let preResolvedAllowedIds: OpaqueIdSet | undefined
|
let preResolvedAllowedIds: OpaqueIdSet | undefined
|
||||||
|
|
||||||
// Graph-first law (10.4.8, BRAINY-PROD-LATENCY-TRIAD rounds 44/45): with
|
if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
|
||||||
// `connected` present the NEIGHBOUR SET is the candidate universe. It is
|
|
||||||
// resolved first from the adjacency (O(neighbours)), the metadata filter
|
|
||||||
// is evaluated over those ids only, and paging happens LAST. The earlier
|
|
||||||
// order materialized the whole-store filtered id list, paged it, hydrated
|
|
||||||
// the page, and only then intersected with the neighbours — O(store) per
|
|
||||||
// call, and a neighbour outside the first page was silently dropped.
|
|
||||||
let graphFirstIds: string[] | null = null
|
|
||||||
if (hasGraphCriteria) {
|
|
||||||
graphFirstIds = await this.resolveConnectedIds(params)
|
|
||||||
if (hiddenIds.size > 0) {
|
|
||||||
graphFirstIds = graphFirstIds.filter((id) => !hiddenIds.has(id))
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
graphFirstIds.length > 0 &&
|
|
||||||
(params.where || params.type || params.subtype || params.service || params.excludeVFS)
|
|
||||||
) {
|
|
||||||
preResolvedFilter = this.buildMetadataFilter(params)
|
|
||||||
graphFirstIds = await this.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
|
|
||||||
}
|
|
||||||
if (graphFirstIds.length === 0) {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
if (!hasVectorSearchCriteria) {
|
|
||||||
return await this.pageConnectedIds(params, graphFirstIds)
|
|
||||||
}
|
|
||||||
// The vector leg walks ONLY the neighbours (its candidate walk). The
|
|
||||||
// filter is already applied above, so no opaque universe is produced —
|
|
||||||
// it would describe the whole store, not the neighbour set.
|
|
||||||
preResolvedMetadataIds = graphFirstIds
|
|
||||||
} else if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
|
|
||||||
preResolvedFilter = this.buildMetadataFilter(params)
|
preResolvedFilter = this.buildMetadataFilter(params)
|
||||||
preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)
|
preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)
|
||||||
|
|
||||||
|
|
@ -7755,11 +7671,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The text leg of a hybrid find has no candidate door, so its hits are
|
// Graph search component with O(1) traversal
|
||||||
// held to the neighbour set here; the vector leg walked only the neighbours.
|
if (params.connected) {
|
||||||
if (graphFirstIds !== null && results.length > 0) {
|
results = await this.executeGraphSearch(params, results)
|
||||||
const neighbourSet = new Set(graphFirstIds)
|
|
||||||
results = results.filter((r) => neighbourSet.has(r.id))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply fusion scoring if requested
|
// Apply fusion scoring if requested
|
||||||
|
|
@ -12874,29 +12788,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* The id-scoped twin of {@link filterIdsBelted}: evaluate `filter` over `ids`
|
|
||||||
* only, through the provider's own evaluation so the answer can never drift
|
|
||||||
* from `getIdsForFilter`'s. A provider without the door is served by its
|
|
||||||
* whole-store answer intersected here (the reference index implements the
|
|
||||||
* door itself). Same belt: field refusals cross as `BrainyFieldRefusal`.
|
|
||||||
*/
|
|
||||||
private async filterIdsWithinBelted(filter: unknown, ids: readonly string[]): Promise<string[]> {
|
|
||||||
this.ensureIndexesLoaded(['metadata'])
|
|
||||||
const mip = this.metadataIndex as unknown as MetadataIndexProvider
|
|
||||||
try {
|
|
||||||
if (typeof mip.filterIdsWithin === 'function') {
|
|
||||||
return await mip.filterIdsWithin(filter, ids)
|
|
||||||
}
|
|
||||||
const matched = new Set(await this.metadataIndex.getIdsForFilter(filter))
|
|
||||||
return ids.filter((id) => matched.has(id))
|
|
||||||
} catch (err) {
|
|
||||||
const normalized = asBrainyFieldRefusal(err)
|
|
||||||
if (normalized) throw normalized
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async getIndexStatus(): Promise<{
|
async getIndexStatus(): Promise<{
|
||||||
initialized: boolean
|
initialized: boolean
|
||||||
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
||||||
|
|
@ -15880,16 +15771,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `params.connected` to the neighbour id set — the graph-first
|
* Execute graph search component.
|
||||||
* find's candidate universe (deterministic traversal order, anchors excluded).
|
|
||||||
*
|
*
|
||||||
* Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via
|
* Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via
|
||||||
* `neighbors()`), `via`/`type` verb-type filtering, and `direction`. An empty set
|
* `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read
|
||||||
* is re-verified against the adjacency before it is believed — a not-serving
|
* only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via`
|
||||||
* adjacency throws rather than answering `[]` as truth.
|
* were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the
|
||||||
|
* immediate neighbour at every depth.
|
||||||
*/
|
*/
|
||||||
private async resolveConnectedIds(params: FindParams<T>): Promise<string[]> {
|
private async executeGraphSearch(params: FindParams<T>, existingResults: Result<T>[]): Promise<Result<T>[]> {
|
||||||
if (!params.connected) return []
|
if (!params.connected) return existingResults
|
||||||
|
|
||||||
const { from, to, depth, direction = 'both' } = params.connected
|
const { from, to, depth, direction = 'both' } = params.connected
|
||||||
const via = params.connected.via ?? params.connected.type
|
const via = params.connected.via ?? params.connected.type
|
||||||
|
|
@ -15943,8 +15834,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
if (anchorInt === undefined) return new Set() // unmapped → no relations
|
if (anchorInt === undefined) return new Set() // unmapped → no relations
|
||||||
|
|
||||||
const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
|
const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
|
||||||
// No limit: match the JS BFS exactly — the page is cut downstream,
|
// No limit: match the JS BFS exactly — overall result limiting happens
|
||||||
// after the metadata filter, by pageConnectedIds / the candidate walk.
|
// downstream against existingResults.
|
||||||
const reachedInts = await provider.findConnectedSubtype(
|
const reachedInts = await provider.findConnectedSubtype(
|
||||||
anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
|
anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
|
||||||
)
|
)
|
||||||
|
|
@ -16029,44 +15920,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
await this.verifyGraphAdjacencyLive()
|
await this.verifyGraphAdjacencyLive()
|
||||||
}
|
}
|
||||||
|
|
||||||
return [...connectedIds]
|
// Filter existing results to only connected entities
|
||||||
}
|
if (existingResults.length > 0) {
|
||||||
|
return existingResults.filter(r => connectedIds.has(r.id))
|
||||||
/**
|
|
||||||
* Page and hydrate an already-filtered neighbour set — the pure graph (and
|
|
||||||
* graph + metadata) find's tail. `orderBy` sorts the WHOLE set by field value
|
|
||||||
* before the page is cut (never the page after), null values last on `asc`
|
|
||||||
* and first on `desc`; without `orderBy` the traversal order stands.
|
|
||||||
*/
|
|
||||||
private async pageConnectedIds(params: FindParams<T>, ids: string[]): Promise<Result<T>[]> {
|
|
||||||
const limit = params.limit || 10
|
|
||||||
const offset = params.offset || 0
|
|
||||||
let ordered = ids
|
|
||||||
if (params.orderBy) {
|
|
||||||
const field = params.orderBy
|
|
||||||
const asc = (params.order || 'asc') === 'asc'
|
|
||||||
const valued = await Promise.all(
|
|
||||||
ids.map(async (id) => ({ id, value: await this.metadataIndex.getFieldValueForEntity(id, field) }))
|
|
||||||
)
|
|
||||||
valued.sort((a, b) => {
|
|
||||||
if (a.value == null && b.value == null) return 0
|
|
||||||
if (a.value == null) return asc ? 1 : -1
|
|
||||||
if (b.value == null) return asc ? -1 : 1
|
|
||||||
if (a.value === b.value) return 0
|
|
||||||
const comparison = a.value < b.value ? -1 : 1
|
|
||||||
return asc ? comparison : -comparison
|
|
||||||
})
|
|
||||||
ordered = valued.map((v) => v.id)
|
|
||||||
}
|
}
|
||||||
const pageIds = ordered.slice(offset, offset + limit)
|
|
||||||
const entitiesMap = await this.batchGet(pageIds)
|
// Batch-load connected entities for fast cloud-storage performance
|
||||||
const results: Result<T>[] = []
|
const results: Result<T>[] = []
|
||||||
for (const id of pageIds) {
|
const ids = [...connectedIds]
|
||||||
|
const entitiesMap = await this.batchGet(ids)
|
||||||
|
for (const id of ids) {
|
||||||
const entity = entitiesMap.get(id)
|
const entity = entitiesMap.get(id)
|
||||||
if (entity) {
|
if (entity) {
|
||||||
results.push(this.createResult(id, 1.0, entity))
|
results.push(this.createResult(id, 1.0, entity))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -19586,7 +19455,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* terminal releases have run.
|
* terminal releases have run.
|
||||||
*/
|
*/
|
||||||
async close(): Promise<void> {
|
async close(): Promise<void> {
|
||||||
if (this._pendingEmbedIds.size === 0) await this.writeEmbedLowWater()
|
|
||||||
let closeFailure: unknown = null
|
let closeFailure: unknown = null
|
||||||
try {
|
try {
|
||||||
await this.closeDurableSteps()
|
await this.closeDurableSteps()
|
||||||
|
|
|
||||||
|
|
@ -411,19 +411,6 @@ export interface MetadataIndexProvider {
|
||||||
* @returns The matching id universe as an opaque set.
|
* @returns The matching id universe as an opaque set.
|
||||||
*/
|
*/
|
||||||
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
|
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
|
||||||
/**
|
|
||||||
* @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the
|
|
||||||
* survivors in the caller's order — the door a graph-first
|
|
||||||
* `find({ connected, where })` walks. The neighbour set is the universe there,
|
|
||||||
* so the filter must cost O(|ids|) membership checks, never a whole-store
|
|
||||||
* materialization. A native index answers from its roaring filter result
|
|
||||||
* (membership by entity int); the reference index answers from its own
|
|
||||||
* `getIdsForFilter`, so the two doors can never disagree. Absent → Brainy
|
|
||||||
* intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)).
|
|
||||||
* @param filter - The same filter shape accepted by `getIdsForFilter`.
|
|
||||||
* @param ids - The candidate ids (canonical). The answer is a subsequence.
|
|
||||||
*/
|
|
||||||
filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
|
|
||||||
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
||||||
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
||||||
getFilterValues(field: string): Promise<string[]>
|
getFilterValues(field: string): Promise<string[]>
|
||||||
|
|
|
||||||
|
|
@ -1089,10 +1089,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
|
|
||||||
// Counts changed since the last persist? Drives the write-through flush.
|
// Counts changed since the last persist? Drives the write-through flush.
|
||||||
protected pendingCountPersist = false
|
protected pendingCountPersist = false
|
||||||
/** The one persist running right now, if any (single-flight law — see flushCounts). */
|
|
||||||
private countPersistInFlight: Promise<void> | null = null
|
|
||||||
/** The one trailing persist a burst has queued behind the in-flight one. */
|
|
||||||
private countPersistTrailing: Promise<void> | null = null
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get total noun count - O(1) operation
|
* Get total noun count - O(1) operation
|
||||||
|
|
@ -1345,46 +1341,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// SINGLE-FLIGHT, COALESCED. Counts are write-through on every change, so
|
try {
|
||||||
// a burst of writes used to launch one persist per change, all in flight
|
// Persist to storage (implemented by subclass)
|
||||||
// together. Two of them inside the same millisecond shared the atomic
|
await this.persistCounts()
|
||||||
// writer's temp path (`.tmp-<pid>-<ms>`): both wrote it, the first rename
|
this.pendingCountPersist = false
|
||||||
// consumed it, the second rename found nothing — ENOENT, ~1,500 times a
|
} catch (error) {
|
||||||
// day on a busy production brain, with a full ledger write per change
|
console.error('CRITICAL: Failed to flush counts to storage:', error)
|
||||||
// behind it. Now exactly one persist runs at a time; requests that arrive
|
// Keep pending flag set so we retry on next operation
|
||||||
// while it runs collapse into ONE trailing persist that carries the final
|
throw error
|
||||||
// state. A burst of N changes costs at most two writes and never races
|
|
||||||
// itself.
|
|
||||||
if (this.countPersistInFlight) {
|
|
||||||
// The in-flight write may have already serialised a stale snapshot —
|
|
||||||
// ask for one more pass after it, and let every caller in this burst
|
|
||||||
// await that same pass.
|
|
||||||
if (!this.countPersistTrailing) {
|
|
||||||
this.countPersistTrailing = this.countPersistInFlight
|
|
||||||
.catch(() => undefined)
|
|
||||||
.then(() => {
|
|
||||||
this.countPersistTrailing = null
|
|
||||||
return this.flushCounts()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return this.countPersistTrailing
|
|
||||||
}
|
}
|
||||||
|
|
||||||
this.countPersistInFlight = (async () => {
|
|
||||||
try {
|
|
||||||
// Persist to storage (implemented by subclass)
|
|
||||||
this.pendingCountPersist = false
|
|
||||||
await this.persistCounts()
|
|
||||||
} catch (error) {
|
|
||||||
// Keep the flag set so the next operation retries.
|
|
||||||
this.pendingCountPersist = true
|
|
||||||
console.error('CRITICAL: Failed to flush counts to storage:', error)
|
|
||||||
throw error
|
|
||||||
} finally {
|
|
||||||
this.countPersistInFlight = null
|
|
||||||
}
|
|
||||||
})()
|
|
||||||
return this.countPersistInFlight
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -2400,15 +2400,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
||||||
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
||||||
*/
|
*/
|
||||||
/** Monotonic per-process sequence so two atomic writes never share a temp path. */
|
|
||||||
private static atomicWriteSeq = 0
|
|
||||||
|
|
||||||
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
||||||
// pid + timestamp alone collided: two writers of the same target inside
|
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`
|
||||||
// one millisecond shared this path, and the loser's rename found the
|
|
||||||
// winner had already moved it (ENOENT). The sequence makes every call's
|
|
||||||
// temp path its own.
|
|
||||||
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}-${++FileSystemStorage.atomicWriteSeq}`
|
|
||||||
await fs.promises.writeFile(tmp, contents)
|
await fs.promises.writeFile(tmp, contents)
|
||||||
await fs.promises.rename(tmp, filePath)
|
await fs.promises.rename(tmp, filePath)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2942,33 +2942,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
const sourceId = Array.isArray(options.filter.sourceId)
|
||||||
? options.filter.sourceId
|
? options.filter.sourceId[0]
|
||||||
: [options.filter.sourceId]
|
: options.filter.sourceId
|
||||||
|
|
||||||
// EVERY requested verb type is honoured — an array used to collapse to
|
const verbType = Array.isArray(options.filter.verbType)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.verbType[0]
|
||||||
const verbTypes = new Set(
|
: options.filter.verbType
|
||||||
Array.isArray(options.filter.verbType)
|
|
||||||
? options.filter.verbType
|
|
||||||
: [options.filter.verbType]
|
|
||||||
)
|
|
||||||
|
|
||||||
// Get verbs by source (union over every requested source), filter by the
|
// Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter),
|
||||||
// requested type SET (O(1) graph lookup + O(n) type filter), then apply
|
// then apply the subtype / visibility metadata filters on the candidate set.
|
||||||
// the subtype / visibility metadata filters on the candidate set.
|
const verbsBySource = await this.getVerbsBySource_internal(sourceId)
|
||||||
const bySource: HNSWVerbWithMetadata[] = []
|
|
||||||
const seenVerbIds = new Set<string>()
|
|
||||||
for (const oneSource of sourceIds) {
|
|
||||||
for (const v of await this.getVerbsBySource_internal(oneSource)) {
|
|
||||||
if (!seenVerbIds.has(v.id)) {
|
|
||||||
seenVerbIds.add(v.id)
|
|
||||||
bySource.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const filteredVerbs = this.applyVerbMetadataFilters(
|
const filteredVerbs = this.applyVerbMetadataFilters(
|
||||||
bySource.filter(v => verbTypes.has(v.verb)),
|
verbsBySource.filter(v => v.verb === verbType),
|
||||||
options.filter
|
options.filter
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -2999,22 +2985,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
// EVERY requested source is honoured — an array used to collapse to
|
const sourceId = Array.isArray(options.filter.sourceId)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.sourceId[0]
|
||||||
const onlySourceIds = Array.isArray(options.filter.sourceId)
|
: options.filter.sourceId
|
||||||
? options.filter.sourceId
|
|
||||||
: [options.filter.sourceId]
|
// Get verbs by source directly (hydrated with metadata), then apply the
|
||||||
const sourceUnion: HNSWVerbWithMetadata[] = []
|
// subtype / visibility metadata filters on the O(degree) candidate set.
|
||||||
const seenSourceVerbIds = new Set<string>()
|
const verbsBySource = this.applyVerbMetadataFilters(
|
||||||
for (const oneSource of onlySourceIds) {
|
await this.getVerbsBySource_internal(sourceId),
|
||||||
for (const v of await this.getVerbsBySource_internal(oneSource)) {
|
options.filter
|
||||||
if (!seenSourceVerbIds.has(v.id)) {
|
)
|
||||||
seenSourceVerbIds.add(v.id)
|
|
||||||
sourceUnion.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const verbsBySource = this.applyVerbMetadataFilters(sourceUnion, options.filter)
|
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
||||||
|
|
@ -3043,22 +3023,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
// EVERY requested target is honoured — an array used to collapse to
|
const targetId = Array.isArray(options.filter.targetId)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.targetId[0]
|
||||||
const onlyTargetIds = Array.isArray(options.filter.targetId)
|
: options.filter.targetId
|
||||||
? options.filter.targetId
|
|
||||||
: [options.filter.targetId]
|
// Get verbs by target directly (hydrated with metadata), then apply the
|
||||||
const targetUnion: HNSWVerbWithMetadata[] = []
|
// subtype / visibility metadata filters on the O(degree) candidate set.
|
||||||
const seenTargetVerbIds = new Set<string>()
|
const verbsByTarget = this.applyVerbMetadataFilters(
|
||||||
for (const oneTarget of onlyTargetIds) {
|
await this.getVerbsByTarget_internal(targetId),
|
||||||
for (const v of await this.getVerbsByTarget_internal(oneTarget)) {
|
options.filter
|
||||||
if (!seenTargetVerbIds.has(v.id)) {
|
)
|
||||||
seenTargetVerbIds.add(v.id)
|
|
||||||
targetUnion.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const verbsByTarget = this.applyVerbMetadataFilters(targetUnion, options.filter)
|
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
||||||
|
|
@ -3087,25 +3061,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
// EVERY requested verb type is honoured — an array used to collapse to
|
const verbType = Array.isArray(options.filter.verbType)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.verbType[0]
|
||||||
const verbTypes = Array.isArray(options.filter.verbType)
|
: options.filter.verbType
|
||||||
? options.filter.verbType
|
|
||||||
: [options.filter.verbType]
|
|
||||||
|
|
||||||
// Get verbs by each requested type (hydrated with metadata), deduped by
|
// Get verbs by type directly (hydrated with metadata), then apply the
|
||||||
// id, then apply the subtype / visibility metadata filters on the set.
|
// subtype / visibility metadata filters on the candidate set.
|
||||||
const byType: HNSWVerbWithMetadata[] = []
|
const verbsByType = this.applyVerbMetadataFilters(
|
||||||
const seenTypeVerbIds = new Set<string>()
|
await this.getVerbsByType_internal(verbType),
|
||||||
for (const oneType of verbTypes) {
|
options.filter
|
||||||
for (const v of await this.getVerbsByType_internal(oneType)) {
|
)
|
||||||
if (!seenTypeVerbIds.has(v.id)) {
|
|
||||||
seenTypeVerbIds.add(v.id)
|
|
||||||
byType.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const verbsByType = this.applyVerbMetadataFilters(byType, options.filter)
|
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
||||||
import type { GraphVerb } from '../../coreTypes.js'
|
import type { GraphVerb } from '../../coreTypes.js'
|
||||||
import type { Operation, RollbackAction } from '../types.js'
|
import type { Operation, RollbackAction } from '../types.js'
|
||||||
import { isZeroNormVector } from '../../utils/distance.js'
|
import { isZeroNormVector } from '../../utils/distance.js'
|
||||||
import { jsonSafeIndexMetadata } from '../../utils/jsonSafeIndexMetadata.js'
|
|
||||||
import { prodLog } from '../../utils/logger.js'
|
import { prodLog } from '../../utils/logger.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -391,21 +390,13 @@ export class AddToMetadataIndexOperation implements Operation {
|
||||||
// rollback so add + undo reference the same watermark.
|
// rollback so add + undo reference the same watermark.
|
||||||
const generation = this.generationFn?.()
|
const generation = this.generationFn?.()
|
||||||
|
|
||||||
// The JSON-safe view is taken HERE, per crossing, never at construction:
|
// Add to metadata index (skipFlush=true for transaction atomicity)
|
||||||
// the entity reference this op holds can be mutated between plan and
|
await this.index.addToIndex(this.id, this.entity, true, false, generation)
|
||||||
// execute (a graph op's execute-time endpoint-int resolution mirrors
|
|
||||||
// BigInts onto a shared verb object) — see jsonSafeIndexMetadata's
|
|
||||||
// module doc.
|
|
||||||
await this.index.addToIndex(
|
|
||||||
this.id, jsonSafeIndexMetadata(this.entity), true, false, generation
|
|
||||||
)
|
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
// Remove from metadata index
|
// Remove from metadata index
|
||||||
await this.index.removeFromIndex(
|
await this.index.removeFromIndex(this.id, this.entity, generation)
|
||||||
this.id, jsonSafeIndexMetadata(this.entity), generation
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -441,21 +432,13 @@ export class RemoveFromMetadataIndexOperation implements Operation {
|
||||||
// Resolve the removal generation once; reuse it for the rollback re-add.
|
// Resolve the removal generation once; reuse it for the rollback re-add.
|
||||||
const generation = this.generationFn?.()
|
const generation = this.generationFn?.()
|
||||||
|
|
||||||
// Sanitized per crossing, never at construction — transact()'s delete
|
// Remove from metadata index
|
||||||
// legs hand this op the SAME verb object the graph-retraction op's
|
await this.index.removeFromIndex(this.id, this.entity, generation)
|
||||||
// execute-time endpoint resolution mutates (BigInt sourceInt/targetInt),
|
|
||||||
// so a plan-time view aliases the pollution. See jsonSafeIndexMetadata's
|
|
||||||
// module doc.
|
|
||||||
await this.index.removeFromIndex(
|
|
||||||
this.id, jsonSafeIndexMetadata(this.entity), generation
|
|
||||||
)
|
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
// Re-add with original metadata (skipFlush=true)
|
// Re-add with original metadata (skipFlush=true)
|
||||||
await this.index.addToIndex(
|
await this.index.addToIndex(this.id, this.entity, true, false, generation)
|
||||||
this.id, jsonSafeIndexMetadata(this.entity), true, false, generation
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
/**
|
|
||||||
* @module utils/jsonSafeIndexMetadata
|
|
||||||
* @description The metadata-index crossing's JSON-safety law, as a leaf
|
|
||||||
* function both the coordinator and the transaction operations share.
|
|
||||||
*
|
|
||||||
* The seam's metadata is JSON-safe BY CONTRACT (a native provider serializes
|
|
||||||
* it; u64 ints as Number corrupt above 2^53) — but `resolveVerbEndpointInts`
|
|
||||||
* MIRRORS the resolved endpoint ints onto the verb object itself as BigInt
|
|
||||||
* (`verb.sourceInt`/`targetInt`), so a verb object reused as index metadata
|
|
||||||
* carries BigInts into JSON.stringify, which throws, aborting the whole
|
|
||||||
* transaction. Endpoint ints ride their OWN op params on the graph legs — the
|
|
||||||
* metadata crossing drops every BigInt-valued top-level key instead of
|
|
||||||
* guessing at a lossy numeric encoding.
|
|
||||||
*
|
|
||||||
* WHY THIS IS A LEAF MODULE, ENFORCED AT THE CROSSING: sanitizing only at
|
|
||||||
* operation-construction time is not enough. `transact()`'s delete legs pass
|
|
||||||
* the SAME verb object to both the graph-retraction op (whose endpoint-int
|
|
||||||
* thunk deliberately resolves at EXECUTE time, for same-batch forward refs)
|
|
||||||
* and the metadata-retraction op. At plan time the verb is still clean, so a
|
|
||||||
* plan-time sanitize returns the same reference — then the graph op executes
|
|
||||||
* first, mirrors the BigInt ints onto the shared object, and the metadata op
|
|
||||||
* crosses the seam with them (found by the first fleet adoption of the native
|
|
||||||
* pair: every transact-wrapped edge delete aborted). The crossing itself is
|
|
||||||
* the only place ordering cannot bypass.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A JSON-safe view of a record bound for the metadata-index crossing.
|
|
||||||
*
|
|
||||||
* @param metadata - The candidate index-metadata record.
|
|
||||||
* @returns The same object when already JSON-safe, else a shallow copy
|
|
||||||
* without the BigInt-valued keys.
|
|
||||||
*/
|
|
||||||
export function jsonSafeIndexMetadata(metadata: unknown): unknown {
|
|
||||||
if (metadata === null || typeof metadata !== 'object') return metadata
|
|
||||||
const rec = metadata as Record<string, unknown>
|
|
||||||
let hasBigint = false
|
|
||||||
for (const k in rec) {
|
|
||||||
if (typeof rec[k] === 'bigint') { hasBigint = true; break }
|
|
||||||
}
|
|
||||||
if (!hasBigint) return metadata
|
|
||||||
const out: Record<string, unknown> = {}
|
|
||||||
for (const k in rec) {
|
|
||||||
if (typeof rec[k] !== 'bigint') out[k] = rec[k]
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
@ -2575,19 +2575,6 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
/** Once-per-field flag for the fallback-degradation announcement. */
|
/** Once-per-field flag for the fallback-degradation announcement. */
|
||||||
private static announcedFallbackSorts = new Set<string>()
|
private static announcedFallbackSorts = new Set<string>()
|
||||||
|
|
||||||
/**
|
|
||||||
* Evaluate `filter` over `ids` only — the graph-first find's door (the
|
|
||||||
* neighbour set filtered by id, never the store filtered and then
|
|
||||||
* intersected). This index answers from its own `getIdsForFilter`, so the
|
|
||||||
* two doors cannot disagree; the cost is that of the filter over this
|
|
||||||
* in-memory index, and the answer keeps the caller's order.
|
|
||||||
*/
|
|
||||||
async filterIdsWithin(filter: any, ids: readonly string[]): Promise<string[]> {
|
|
||||||
if (ids.length === 0) return []
|
|
||||||
const matched = new Set(await this.getIdsForFilter(filter))
|
|
||||||
return ids.filter((id) => matched.has(id))
|
|
||||||
}
|
|
||||||
|
|
||||||
async getSortedIdsForFilter(
|
async getSortedIdsForFilter(
|
||||||
filter: any,
|
filter: any,
|
||||||
orderBy: string,
|
orderBy: string,
|
||||||
|
|
|
||||||
|
|
@ -1,111 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/counts-persist-single-flight
|
|
||||||
* @description Regression for a production race in FileSystemStorage's
|
|
||||||
* counts ledger: `persistCounts()` was write-through on every count change
|
|
||||||
* with no serialization, and the atomic writer named its temp file with
|
|
||||||
* millisecond granularity (`.tmp-<pid>-<ms>`). Two persists inside one
|
|
||||||
* millisecond shared the temp path — both wrote it, the first rename
|
|
||||||
* consumed it, the second rename found nothing: ENOENT, ~1,500 times a day
|
|
||||||
* on a busy production brain, with a full ledger write per change behind it.
|
|
||||||
*
|
|
||||||
* Under pin: persists are single-flight and coalesced — one in flight, at
|
|
||||||
* most one trailing pass carrying the burst's final state — and every atomic
|
|
||||||
* write owns a unique temp path. A burst of N count changes costs at most
|
|
||||||
* two ledger writes, never errors, and leaves a ledger equal to memory.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
||||||
import * as fs from 'node:fs'
|
|
||||||
import * as os from 'node:os'
|
|
||||||
import * as path from 'node:path'
|
|
||||||
import { Brainy } from '../../src/brainy.js'
|
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
|
||||||
|
|
||||||
describe('counts persistence is single-flight, coalesced, and never races its own temp file', () => {
|
|
||||||
let dir: string
|
|
||||||
let brain: any
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
||||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-counts-race-'))
|
|
||||||
brain = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
dimensions: 384,
|
|
||||||
silent: true
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
vi.restoreAllMocks()
|
|
||||||
await brain.close()
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a burst of concurrent count changes → at most two ledger writes, zero errors, ledger == memory', async () => {
|
|
||||||
const storage = brain.storage
|
|
||||||
const countsPath: string = storage.countsFilePath
|
|
||||||
expect(countsPath, 'the filesystem adapter persists a counts ledger').toBeTruthy()
|
|
||||||
|
|
||||||
// Let init's own persists settle so the burst is measured alone.
|
|
||||||
await storage.flushCounts?.()
|
|
||||||
|
|
||||||
const renameSpy = vi.spyOn(fs.promises, 'rename')
|
|
||||||
const errorSpy = vi.spyOn(console, 'error')
|
|
||||||
|
|
||||||
// Twenty-five concurrent count changes — the shape of a write burst; each
|
|
||||||
// used to launch its own persist.
|
|
||||||
const BURST = 25
|
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: BURST }, () => storage.scheduleCountPersist())
|
|
||||||
)
|
|
||||||
|
|
||||||
const ledgerRenames = renameSpy.mock.calls.filter(([, to]) => String(to) === countsPath)
|
|
||||||
expect(ledgerRenames.length, 'single-flight + one trailing pass').toBeLessThanOrEqual(2)
|
|
||||||
expect(ledgerRenames.length, 'the burst was persisted at all').toBeGreaterThanOrEqual(1)
|
|
||||||
|
|
||||||
const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts'))
|
|
||||||
expect(persistErrors).toEqual([])
|
|
||||||
|
|
||||||
const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
|
||||||
expect(ledger.totalNounCount).toBe(storage.totalNounCount)
|
|
||||||
expect(ledger.totalVerbCount).toBe(storage.totalVerbCount)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('real writes in parallel: the ledger lands complete and no persist error is logged', async () => {
|
|
||||||
const storage = brain.storage
|
|
||||||
const countsPath: string = storage.countsFilePath
|
|
||||||
const errorSpy = vi.spyOn(console, 'error')
|
|
||||||
|
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: 12 }, (_, i) =>
|
|
||||||
brain.add({ data: `burst row ${i}`, type: NounType.Thing })
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await storage.flushCounts?.()
|
|
||||||
|
|
||||||
const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts'))
|
|
||||||
expect(persistErrors).toEqual([])
|
|
||||||
const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
|
||||||
expect(ledger.totalNounCount).toBe(storage.totalNounCount)
|
|
||||||
expect(await brain.getNounCount()).toBe(ledger.totalNounCount)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('every atomic write owns its own temp path — two writes in one millisecond never collide', async () => {
|
|
||||||
const storage = brain.storage
|
|
||||||
const tmpNames: string[] = []
|
|
||||||
vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (p: any) => {
|
|
||||||
tmpNames.push(String(p))
|
|
||||||
})
|
|
||||||
vi.spyOn(fs.promises, 'rename').mockImplementation(async () => undefined)
|
|
||||||
const target = path.join(dir, 'probe.json')
|
|
||||||
await Promise.all([
|
|
||||||
storage.writeFileAtomic(target, '{"a":1}'),
|
|
||||||
storage.writeFileAtomic(target, '{"a":2}'),
|
|
||||||
storage.writeFileAtomic(target, '{"a":3}')
|
|
||||||
])
|
|
||||||
const probeTmps = tmpNames.filter((n) => n.startsWith(`${target}.tmp-`))
|
|
||||||
expect(probeTmps.length).toBe(3)
|
|
||||||
expect(new Set(probeTmps).size, 'no two writes shared a temp path').toBe(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,165 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/find-connected-order
|
|
||||||
* @description The graph-first law for `find({ connected })` (10.4.8).
|
|
||||||
*
|
|
||||||
* With `connected` present the neighbour set is the candidate universe: it is
|
|
||||||
* resolved from the adjacency first, the metadata filter is evaluated over
|
|
||||||
* those ids only, and the page is cut last. The earlier order materialized the
|
|
||||||
* whole-store filtered id list, paged it, hydrated the page, and only then
|
|
||||||
* intersected with the neighbours — so a neighbour outside the first page of
|
|
||||||
* the filtered STORE was silently dropped, and every call paid O(store).
|
|
||||||
*
|
|
||||||
* These pins hold both halves. The answer: every matching neighbour is
|
|
||||||
* reachable by paging, a non-neighbour never appears, a negation (`missing`)
|
|
||||||
* is evaluated over the neighbours, `orderBy` sorts the whole neighbour set
|
|
||||||
* before the page is cut, and the vector leg walks the neighbours only. The
|
|
||||||
* cost shape: the metadata index is asked about the neighbour ids only, and
|
|
||||||
* hydration is one page — never the store.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
|
||||||
import { Brainy } from '../../src/brainy'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
|
||||||
import { v5 } from '../../src/universal/uuid'
|
|
||||||
import { generateTestVector } from '../helpers/test-factory'
|
|
||||||
|
|
||||||
/** Matching rows that are NOT neighbours — added FIRST, so the whole-store filtered list leads with them. */
|
|
||||||
const NOISE = 120
|
|
||||||
/** Matching rows that ARE neighbours of the anchor. */
|
|
||||||
const NEIGHBOURS = 30
|
|
||||||
/** Neighbours carrying `retracted: true` — excluded by the `missing` negation. */
|
|
||||||
const RETRACTED = 4
|
|
||||||
|
|
||||||
describe('find({ connected }) is graph-first: neighbours → filter → page', () => {
|
|
||||||
let brain: Brainy<any>
|
|
||||||
const anchor = 'anchor'
|
|
||||||
const sharedVector = generateTestVector()
|
|
||||||
const neighbourIds = new Set(Array.from({ length: NEIGHBOURS }, (_, i) => v5(`nb-${i}`)))
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
await brain.add({
|
|
||||||
id: anchor,
|
|
||||||
data: 'the anchor',
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { kind: 'anchor' },
|
|
||||||
vector: generateTestVector()
|
|
||||||
})
|
|
||||||
for (let i = 0; i < NOISE; i++) {
|
|
||||||
await brain.add({
|
|
||||||
id: `noise-${i}`,
|
|
||||||
data: `noise ${i}`,
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { kind: 'note', rank: 1000 + i },
|
|
||||||
vector: sharedVector
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for (let i = 0; i < NEIGHBOURS; i++) {
|
|
||||||
await brain.add({
|
|
||||||
id: `nb-${i}`,
|
|
||||||
data: `neighbour ${i}`,
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { kind: 'note', rank: i + 1, ...(i < RETRACTED ? { retracted: true } : {}) },
|
|
||||||
vector: sharedVector
|
|
||||||
})
|
|
||||||
await brain.relate({ from: anchor, to: `nb-${i}`, type: VerbType.Knows })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
brain = null as any
|
|
||||||
})
|
|
||||||
|
|
||||||
it('returns the matching neighbours page by page — none dropped, never a non-neighbour', async () => {
|
|
||||||
const seen = new Set<string>()
|
|
||||||
for (let offset = 0; offset <= NEIGHBOURS; offset += 10) {
|
|
||||||
const page = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 10,
|
|
||||||
offset
|
|
||||||
})
|
|
||||||
expect(page).toHaveLength(offset < NEIGHBOURS ? 10 : 0)
|
|
||||||
for (const r of page) {
|
|
||||||
expect(neighbourIds.has(r.entity.id)).toBe(true)
|
|
||||||
expect(seen.has(r.entity.id)).toBe(false)
|
|
||||||
seen.add(r.entity.id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expect(seen.size).toBe(NEIGHBOURS)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('evaluates a negation (`missing`) over the neighbour set, not the store', async () => {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note', retracted: { missing: true } },
|
|
||||||
limit: 100
|
|
||||||
})
|
|
||||||
expect(results).toHaveLength(NEIGHBOURS - RETRACTED)
|
|
||||||
for (const r of results) {
|
|
||||||
expect(neighbourIds.has(r.entity.id)).toBe(true)
|
|
||||||
expect(r.entity.metadata.retracted).toBeUndefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('asks the metadata index about the neighbour ids only, and hydrates one page', async () => {
|
|
||||||
const index = (brain as any).metadataIndex
|
|
||||||
const within = vi.spyOn(index, 'filterIdsWithin')
|
|
||||||
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
|
||||||
try {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
expect(results).toHaveLength(10)
|
|
||||||
expect(within).toHaveBeenCalledTimes(1)
|
|
||||||
const askedIds = within.mock.calls[0][1] as string[]
|
|
||||||
expect(askedIds).toHaveLength(NEIGHBOURS)
|
|
||||||
for (const id of askedIds) expect(neighbourIds.has(id)).toBe(true)
|
|
||||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
|
||||||
expect(hydrate.mock.calls[0][0]).toHaveLength(10)
|
|
||||||
} finally {
|
|
||||||
within.mockRestore()
|
|
||||||
hydrate.mockRestore()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('orders the WHOLE neighbour set before cutting the page', async () => {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
orderBy: 'rank',
|
|
||||||
order: 'desc',
|
|
||||||
limit: 5
|
|
||||||
})
|
|
||||||
expect(results.map((r) => r.entity.metadata.rank)).toEqual([30, 29, 28, 27, 26])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('walks the vector leg over the neighbours only', async () => {
|
|
||||||
const results = await brain.find({
|
|
||||||
vector: sharedVector,
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 5
|
|
||||||
})
|
|
||||||
expect(results).toHaveLength(5)
|
|
||||||
for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('an anchor without neighbours answers [] before the filter is asked', async () => {
|
|
||||||
const index = (brain as any).metadataIndex
|
|
||||||
const within = vi.spyOn(index, 'filterIdsWithin')
|
|
||||||
try {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: 'noise-0', direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
expect(results).toEqual([])
|
|
||||||
expect(within).not.toHaveBeenCalled()
|
|
||||||
} finally {
|
|
||||||
within.mockRestore()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,141 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/pending-embed-low-water
|
|
||||||
* @description The pending-embed recovery fold is bounded and background (10.4.9).
|
|
||||||
*
|
|
||||||
* The fold used to scan the generation log from generation 1 at EVERY open,
|
|
||||||
* on the open's foreground — O(whole history) per open on long-lived brains.
|
|
||||||
* Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`)
|
|
||||||
* records the committed generation whenever the pending set drains to empty,
|
|
||||||
* recovery scans from `mark + 1` on the open's foreground — the crash-recovery
|
|
||||||
* contract keeps markers re-armed when open() returns. The mark is advisory: stale-low costs a longer scan, never a
|
|
||||||
* marker — a pending embed enqueued before a crash is still recovered.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, afterEach, vi } from 'vitest'
|
|
||||||
import { mkdtempSync, rmSync } from 'node:fs'
|
|
||||||
import { tmpdir } from 'node:os'
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { Brainy } from '../../src/brainy'
|
|
||||||
import { NounType } from '../../src/types/graphTypes'
|
|
||||||
|
|
||||||
const LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
|
|
||||||
|
|
||||||
describe('pending-embed recovery: bounded by the low-water mark', () => {
|
|
||||||
const roots: string[] = []
|
|
||||||
const dir = (): string => {
|
|
||||||
const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-'))
|
|
||||||
roots.push(d)
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
const open = async (root: string): Promise<Brainy<any>> => {
|
|
||||||
const brain = new Brainy<any>({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: root }
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
return brain
|
|
||||||
}
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('drain-to-empty writes the mark, and the next open scans from mark + 1', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
// Hold the worker so the pending state is observable, then release it.
|
|
||||||
const realKick = (brain as any).kickEmbedWorker.bind(brain)
|
|
||||||
;(brain as any).kickEmbedWorker = () => {}
|
|
||||||
await brain.add({
|
|
||||||
id: 'row-1',
|
|
||||||
data: 'the first deferred row',
|
|
||||||
type: NounType.Thing,
|
|
||||||
deferEmbedding: true
|
|
||||||
})
|
|
||||||
expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
|
|
||||||
;(brain as any).kickEmbedWorker = realKick
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
// The drain wrote the advisory mark (fire-and-forget: settle the microtask).
|
|
||||||
await new Promise((r) => setTimeout(r, 50))
|
|
||||||
const mark = (await (brain as any).storage.readRawObject(LOWWATER_PATH)) as {
|
|
||||||
generation: number
|
|
||||||
} | null
|
|
||||||
expect(mark).not.toBeNull()
|
|
||||||
expect(mark!.generation).toBeGreaterThan(0)
|
|
||||||
await brain.close()
|
|
||||||
|
|
||||||
const brain2 = await open(root)
|
|
||||||
const log = (brain2 as any).generationStore.getFactLog()
|
|
||||||
const scanSpy = vi.spyOn(log, 'scanFacts')
|
|
||||||
try {
|
|
||||||
await (brain2 as any).recoverPendingEmbedsFromLog()
|
|
||||||
expect(scanSpy).toHaveBeenCalledTimes(1)
|
|
||||||
const opts = scanSpy.mock.calls[0][0] as { fromGeneration?: number }
|
|
||||||
expect(opts.fromGeneration).toBeGreaterThanOrEqual(mark!.generation + 1)
|
|
||||||
} finally {
|
|
||||||
scanSpy.mockRestore()
|
|
||||||
await brain2.close()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a pending embed enqueued after the mark survives an unclean stop', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
await brain.add({ id: 'settled', data: 'lands before the mark', type: NounType.Thing })
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
await new Promise((r) => setTimeout(r, 50))
|
|
||||||
|
|
||||||
// A deferred write whose embed never lands: block the worker, then drop
|
|
||||||
// the instance without close() — the unclean-stop shape.
|
|
||||||
;(brain as any).kickEmbedWorker = () => {}
|
|
||||||
await brain.add({
|
|
||||||
id: 'orphan',
|
|
||||||
data: 'enqueued then abandoned',
|
|
||||||
type: NounType.Thing,
|
|
||||||
deferEmbedding: true
|
|
||||||
})
|
|
||||||
expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
|
|
||||||
// No close(): simulate the crash by releasing only the writer lock so the
|
|
||||||
// next open can proceed.
|
|
||||||
await (brain as any).storage.releaseWriterLock()
|
|
||||||
|
|
||||||
const brain2 = await open(root)
|
|
||||||
expect(brain2.pendingEmbedCount()).toBeGreaterThan(0)
|
|
||||||
await brain2.awaitPendingEmbeds()
|
|
||||||
expect(brain2.pendingEmbedCount()).toBe(0)
|
|
||||||
await brain2.close()
|
|
||||||
// Reap the crashed instance: its fence is gone, so close() fails loudly —
|
|
||||||
// swallow that here; the point is clearing its watchers and registry entry.
|
|
||||||
await brain.close().catch(() => undefined)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a reopened brain has its pending set settled when open() returns', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing })
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
await brain.close()
|
|
||||||
|
|
||||||
const brain2 = await open(root)
|
|
||||||
// The crash-recovery contract: markers are re-armed by open itself —
|
|
||||||
// no latch, no background race. (Here the drain landed, so zero.)
|
|
||||||
expect(brain2.pendingEmbedCount()).toBe(0)
|
|
||||||
await brain2.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a clean close with an empty set writes the mark even if no drain happened', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
await brain.add({ id: 'r1', data: 'row one', type: NounType.Thing })
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
await brain.close()
|
|
||||||
// Read the mark back through the storage door (the adapter owns the
|
|
||||||
// on-disk encoding), on a fresh instance.
|
|
||||||
const brain2 = await open(root)
|
|
||||||
const mark = (await (brain2 as any).storage.readRawObject(LOWWATER_PATH)) as {
|
|
||||||
generation: number
|
|
||||||
} | null
|
|
||||||
expect(mark).not.toBeNull()
|
|
||||||
expect(mark!.generation).toBeGreaterThan(0)
|
|
||||||
await brain2.close()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/related-verb-array
|
|
||||||
* @description related() honours EVERY verb type in an array (10.4.9).
|
|
||||||
*
|
|
||||||
* The storage fast paths for `sourceId + verbType` and `verbType` collapsed a
|
|
||||||
* verb-type ARRAY to its first element — `related({ from, type: [a, b] })`
|
|
||||||
* silently returned only `a` edges, whichever order the array came in. The
|
|
||||||
* same quiet-loss class as the graph-first paging defect, one seam over.
|
|
||||||
* These pins seed a store where the SECOND requested type's edge must come
|
|
||||||
* back, on every path the collapse lived in.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
|
||||||
import { Brainy } from '../../src/brainy'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
|
||||||
import { v5 } from '../../src/universal/uuid'
|
|
||||||
|
|
||||||
describe('related() with a verb-type array returns every requested type', () => {
|
|
||||||
let brain: Brainy<any>
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
for (const id of ['a', 'b', 'c', 'd']) {
|
|
||||||
await brain.add({ id, data: `node ${id}`, type: NounType.Person })
|
|
||||||
}
|
|
||||||
await brain.relate({ from: 'a', to: 'b', type: VerbType.Supports })
|
|
||||||
await brain.relate({ from: 'a', to: 'c', type: VerbType.RelatedTo })
|
|
||||||
await brain.relate({ from: 'a', to: 'd', type: VerbType.Knows })
|
|
||||||
await brain.relate({ from: 'b', to: 'c', type: VerbType.RelatedTo })
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
brain = null as any
|
|
||||||
})
|
|
||||||
|
|
||||||
it('from + type array: the second type\'s edge comes back, both orders', async () => {
|
|
||||||
for (const types of [
|
|
||||||
[VerbType.Supports, VerbType.RelatedTo],
|
|
||||||
[VerbType.RelatedTo, VerbType.Supports]
|
|
||||||
]) {
|
|
||||||
const edges = await brain.related({ from: 'a', type: types })
|
|
||||||
const targets = new Set(edges.map((e) => e.to))
|
|
||||||
expect(targets.has(v5('b')), `types [${types}] missing Supports edge`).toBe(true)
|
|
||||||
expect(targets.has(v5('c')), `types [${types}] missing RelatedTo edge`).toBe(true)
|
|
||||||
expect(targets.has(v5('d'))).toBe(false)
|
|
||||||
expect(edges).toHaveLength(2)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a single-element array behaves exactly like the scalar', async () => {
|
|
||||||
const scalar = await brain.related({ from: 'a', type: VerbType.Supports })
|
|
||||||
const array = await brain.related({ from: 'a', type: [VerbType.Supports] })
|
|
||||||
expect(array.map((e) => e.id).sort()).toEqual(scalar.map((e) => e.id).sort())
|
|
||||||
expect(array).toHaveLength(1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('no duplicate edges when types overlap the same edge set', async () => {
|
|
||||||
const edges = await brain.related({
|
|
||||||
from: 'a',
|
|
||||||
type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows]
|
|
||||||
})
|
|
||||||
const ids = edges.map((e) => e.id)
|
|
||||||
expect(new Set(ids).size).toBe(ids.length)
|
|
||||||
expect(edges).toHaveLength(3)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('type-only asks (no anchor) honour the whole array too', async () => {
|
|
||||||
const edges = await brain.related({ type: [VerbType.Supports, VerbType.Knows] })
|
|
||||||
const verbs = new Set(edges.map((e) => e.type))
|
|
||||||
expect(verbs.has(VerbType.Supports)).toBe(true)
|
|
||||||
expect(verbs.has(VerbType.Knows)).toBe(true)
|
|
||||||
expect(edges).toHaveLength(2)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('to + type array: the target side honours every type too', async () => {
|
|
||||||
const edges = await brain.related({ to: 'c', type: [VerbType.RelatedTo, VerbType.Supports] })
|
|
||||||
const froms = new Set(edges.map((e) => e.from))
|
|
||||||
expect(froms.has(v5('a'))).toBe(true)
|
|
||||||
expect(froms.has(v5('b'))).toBe(true)
|
|
||||||
expect(edges).toHaveLength(2)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('pagination stays consistent across the union', async () => {
|
|
||||||
const page1 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2 })
|
|
||||||
const page2 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2, offset: 2 })
|
|
||||||
const all = [...page1, ...page2].map((e) => e.id)
|
|
||||||
expect(new Set(all).size).toBe(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,184 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/transact-edge-delete-bigint-aliasing
|
|
||||||
* @description Regression for a fleet-adoption blocker: ANY edge delete
|
|
||||||
* inside `transact()` — a direct unrelate or a noun-remove's cascade —
|
|
||||||
* aborted with the metadata seam's BigInt JSON-guard error on a strict
|
|
||||||
* (native) metadata provider.
|
|
||||||
*
|
|
||||||
* The aliasing chain: `planTxUnrelate`/the remove-cascade pass the SAME verb
|
|
||||||
* object to the graph-retraction op and the metadata-retraction op. The
|
|
||||||
* metadata leg's JSON-safe wrap ran at PLAN time, when the verb was still
|
|
||||||
* clean — so it returned the same reference. At EXECUTE time the graph op
|
|
||||||
* runs first and `resolveVerbEndpointInts` mirrors BigInt
|
|
||||||
* `sourceInt`/`targetInt` onto the shared object (deliberately deferred for
|
|
||||||
* same-batch forward refs — see transact-forward-ref-graph.test.ts); the
|
|
||||||
* metadata op then crossed the seam with the polluted object. Direct
|
|
||||||
* `unrelate()` resolves ints at BUILD time, before its sanitize, which is why
|
|
||||||
* only the transact() shapes ever hit it.
|
|
||||||
*
|
|
||||||
* Fix under pin: the JSON-safe view is taken AT THE CROSSING — inside the
|
|
||||||
* metadata-index operations' execute/rollback — so no plan-vs-execute
|
|
||||||
* ordering can bypass it. The JS baseline index tolerates BigInts (it would
|
|
||||||
* mask the bug), so these pins SPY on the seam and assert what actually
|
|
||||||
* crossed, exactly as a strict native provider would judge it.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
||||||
import * as fs from 'node:fs'
|
|
||||||
import * as os from 'node:os'
|
|
||||||
import * as path from 'node:path'
|
|
||||||
import { Brainy } from '../../src/brainy.js'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
|
||||||
import {
|
|
||||||
AddToMetadataIndexOperation,
|
|
||||||
RemoveFromMetadataIndexOperation
|
|
||||||
} from '../../src/transaction/operations/index.js'
|
|
||||||
|
|
||||||
let seq = 0
|
|
||||||
const freshId = (): string =>
|
|
||||||
`00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}`
|
|
||||||
|
|
||||||
/** Top-level BigInt-valued keys of a candidate seam crossing (the guard's law). */
|
|
||||||
const bigintKeys = (metadata: unknown): string[] => {
|
|
||||||
if (metadata === null || typeof metadata !== 'object') return []
|
|
||||||
return Object.entries(metadata as Record<string, unknown>)
|
|
||||||
.filter(([, v]) => typeof v === 'bigint')
|
|
||||||
.map(([k]) => k)
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('transact() edge deletes never carry BigInt across the metadata seam', () => {
|
|
||||||
let dir: string
|
|
||||||
let brain: any
|
|
||||||
let crossings: Array<{ door: string; id: string; keys: string[] }>
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
||||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-tx-bigint-'))
|
|
||||||
brain = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
dimensions: 384,
|
|
||||||
silent: true
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// Spy on the seam the way a strict native provider judges it: record the
|
|
||||||
// BigInt-valued top-level keys of every metadata argument that crosses.
|
|
||||||
// The JS baseline index tolerates BigInts, so without this the baseline
|
|
||||||
// run would green a shape the native pair aborts on.
|
|
||||||
crossings = []
|
|
||||||
const index = brain.metadataIndex
|
|
||||||
for (const door of ['addToIndex', 'removeFromIndex'] as const) {
|
|
||||||
const real = index[door].bind(index)
|
|
||||||
index[door] = (id: string, metadata: unknown, ...rest: unknown[]) => {
|
|
||||||
crossings.push({ door, id, keys: bigintKeys(metadata) })
|
|
||||||
return real(id, metadata, ...rest)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
await brain.close()
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('CASE 1 (the fleet repro): relate, then transact([{op: unrelate}])', async () => {
|
|
||||||
const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing })
|
|
||||||
const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing })
|
|
||||||
const verbId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
|
||||||
|
|
||||||
crossings.length = 0
|
|
||||||
await brain.transact([{ op: 'unrelate', id: verbId }])
|
|
||||||
|
|
||||||
const polluted = crossings.filter((c) => c.keys.length > 0)
|
|
||||||
expect(polluted).toEqual([])
|
|
||||||
expect(await brain.storage.getVerb(verbId)).toBeFalsy()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('CASE 2 (the cascade shape): transact([{op: remove}]) cascading edge deletes', async () => {
|
|
||||||
const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing })
|
|
||||||
const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing })
|
|
||||||
const c = await brain.add({ id: freshId(), data: 'c', type: NounType.Thing })
|
|
||||||
const ab = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
|
||||||
const ca = await brain.relate({ from: c, to: a, type: VerbType.RelatedTo })
|
|
||||||
|
|
||||||
crossings.length = 0
|
|
||||||
await brain.transact([{ op: 'remove', id: a }])
|
|
||||||
|
|
||||||
const polluted = crossings.filter((c2) => c2.keys.length > 0)
|
|
||||||
expect(polluted).toEqual([])
|
|
||||||
expect(await brain.get(a)).toBeFalsy()
|
|
||||||
expect(await brain.storage.getVerb(ab)).toBeFalsy()
|
|
||||||
expect(await brain.storage.getVerb(ca)).toBeFalsy()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('CASE 3 (one batch, both legs): adds + relate + unrelate of a pre-existing edge', async () => {
|
|
||||||
const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing })
|
|
||||||
const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing })
|
|
||||||
const old = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
|
||||||
|
|
||||||
const x = freshId()
|
|
||||||
crossings.length = 0
|
|
||||||
await brain.transact([
|
|
||||||
{ op: 'add', id: x, data: 'x', type: NounType.Thing },
|
|
||||||
{ op: 'relate', from: a, to: x, type: VerbType.RelatedTo },
|
|
||||||
{ op: 'unrelate', id: old }
|
|
||||||
])
|
|
||||||
|
|
||||||
const polluted = crossings.filter((c) => c.keys.length > 0)
|
|
||||||
expect(polluted).toEqual([])
|
|
||||||
expect(await brain.storage.getVerb(old)).toBeFalsy()
|
|
||||||
const edges = await brain.related({ from: a })
|
|
||||||
expect(edges.length).toBe(1)
|
|
||||||
expect(edges[0].id).not.toBe(old)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('the metadata-index operations sanitize at the crossing, not at construction', () => {
|
|
||||||
/** A strict seam: refuses BigInts exactly as the native provider does. */
|
|
||||||
const strictIndex = () => {
|
|
||||||
const seen: Array<{ door: string; keys: string[] }> = []
|
|
||||||
const judge = (door: string, metadata: unknown) => {
|
|
||||||
const keys = bigintKeys(metadata)
|
|
||||||
seen.push({ door, keys })
|
|
||||||
if (keys.length > 0) {
|
|
||||||
throw new Error(
|
|
||||||
`${door}: the metadata object violates the provider seam's JSON ` +
|
|
||||||
`contract — BigInt at ${keys.join(', ')}.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
seen,
|
|
||||||
addToIndex: async (_id: string, metadata: unknown) => judge('addToIndex', metadata),
|
|
||||||
removeFromIndex: async (_id: string, metadata: unknown) => judge('removeFromIndex', metadata)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
it('RemoveFromMetadataIndexOperation: entity mutated AFTER construction still crosses clean', async () => {
|
|
||||||
const index = strictIndex()
|
|
||||||
const verb: Record<string, unknown> = { id: 'v1', sourceId: 'a', targetId: 'b' }
|
|
||||||
const op = new RemoveFromMetadataIndexOperation(index as any, 'v1', verb, () => 7n)
|
|
||||||
|
|
||||||
// The graph leg's execute-time endpoint resolution, simulated: the shared
|
|
||||||
// object is polluted between plan and execute.
|
|
||||||
verb.sourceInt = 800_000n
|
|
||||||
verb.targetInt = 800_001n
|
|
||||||
|
|
||||||
const rollback = await op.execute()
|
|
||||||
await rollback()
|
|
||||||
expect(index.seen.map((s) => s.keys)).toEqual([[], []])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('AddToMetadataIndexOperation: same law on the add leg and its rollback', async () => {
|
|
||||||
const index = strictIndex()
|
|
||||||
const verb: Record<string, unknown> = { id: 'v2', sourceId: 'a', targetId: 'b' }
|
|
||||||
const op = new AddToMetadataIndexOperation(index as any, 'v2', verb, () => 7n)
|
|
||||||
|
|
||||||
verb.sourceInt = 800_000n
|
|
||||||
verb.targetInt = 800_001n
|
|
||||||
|
|
||||||
const rollback = await op.execute()
|
|
||||||
await rollback()
|
|
||||||
expect(index.seen.map((s) => s.keys)).toEqual([[], []])
|
|
||||||
})
|
|
||||||
})
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue