Compare commits
No commits in common. "v10.4.9" and "v10.4.6" have entirely different histories.
13 changed files with 78 additions and 842 deletions
11
CHANGELOG.md
11
CHANGELOG.md
|
|
@ -2,17 +2,6 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02)
|
|
||||||
|
|
||||||
- Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d)
|
|
||||||
- fix(open): pending-embed recovery keeps the crash-recovery contract — foreground, bounded by the mark (8a2ebacf)
|
|
||||||
- Merge branches 'fix/connected-find-order', 'fix/pending-embed-low-water' and 'fix/related-verb-array' into rel/10.4.9-candidate (d5147ed6)
|
|
||||||
- fix(graph): the verb fast paths honour every requested type, source, and target (6a89adc4)
|
|
||||||
- perf(open): pending-embed recovery is bounded by a low-water mark and runs behind the doors (88e79729)
|
|
||||||
- fix(find): connected finds are graph-first — neighbours, then the filter over those ids, then the page (077cbc0b)
|
|
||||||
- fix(storage): counts persistence is single-flight, coalesced, and never races its own temp file (5e3b343a)
|
|
||||||
|
|
||||||
|
|
||||||
### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31)
|
### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31)
|
||||||
|
|
||||||
- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d)
|
- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d)
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.9",
|
"version": "10.4.6",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.9",
|
"version": "10.4.6",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraftlabs/brainy",
|
"name": "@soulcraftlabs/brainy",
|
||||||
"version": "10.4.9",
|
"version": "10.4.6",
|
||||||
"brainyContract": 1,
|
"brainyContract": 1,
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
|
|
|
||||||
198
src/brainy.ts
198
src/brainy.ts
|
|
@ -1820,11 +1820,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// a deferred write's ack and its background embed DELAYED a vector;
|
// a deferred write's ack and its background embed DELAYED a vector;
|
||||||
// this is where it lands.
|
// this is where it lands.
|
||||||
if (!this.isReadOnly) {
|
if (!this.isReadOnly) {
|
||||||
// Foreground, as the crash-recovery contract pins it: a reopened brain
|
|
||||||
// has its markers re-armed when open() returns. The low-water mark
|
|
||||||
// bounds this to the log's tail on any brain that has ever drained —
|
|
||||||
// milliseconds — so the foreground cost is the unmarked first open
|
|
||||||
// only, once per upgraded brain.
|
|
||||||
try {
|
try {
|
||||||
await step(
|
await step(
|
||||||
'bridge-pending-embed-sidecars',
|
'bridge-pending-embed-sidecars',
|
||||||
|
|
@ -1833,7 +1828,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
)
|
)
|
||||||
await step(
|
await step(
|
||||||
'recover-pending-embeds',
|
'recover-pending-embeds',
|
||||||
'folding the generation log\'s deferred-embed markers (from the low-water mark) into the pending set',
|
'folding the generation log\'s deferred-embed markers back into the pending set',
|
||||||
() => this.recoverPendingEmbedsFromLog()
|
() => this.recoverPendingEmbedsFromLog()
|
||||||
)
|
)
|
||||||
if (this._pendingEmbedIds.size > 0) {
|
if (this._pendingEmbedIds.size > 0) {
|
||||||
|
|
@ -2413,17 +2408,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
*/
|
*/
|
||||||
private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/'
|
private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/'
|
||||||
|
|
||||||
/**
|
|
||||||
* Storage-root-relative path of the ADVISORY pending-embed low-water mark:
|
|
||||||
* `{ generation, writtenAt }`, written whenever the pending set drains to
|
|
||||||
* empty (and at clean close when empty). Every marker in facts at or below
|
|
||||||
* `generation` is consumed, so recovery scans from `generation + 1`. The
|
|
||||||
* mark is advisory and monotone-safe: stale-low costs a longer scan, never
|
|
||||||
* a lost marker; it is never required for correctness.
|
|
||||||
*/
|
|
||||||
private static readonly PENDING_EMBED_LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Mark a deferred embed pending (MT5): the id joins the
|
* @description Mark a deferred embed pending (MT5): the id joins the
|
||||||
* in-memory fast-path set and the returned `embed.pending` record is
|
* in-memory fast-path set and the returned `embed.pending` record is
|
||||||
|
|
@ -2451,40 +2435,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
*/
|
*/
|
||||||
private clearPendingEmbed(id: string): void {
|
private clearPendingEmbed(id: string): void {
|
||||||
this._pendingEmbedIds.delete(id)
|
this._pendingEmbedIds.delete(id)
|
||||||
if (this._pendingEmbedIds.size === 0) this.maybeWriteEmbedLowWater()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @description Advance the advisory low-water mark: called at drain-to-empty
|
|
||||||
* (and at clean close when empty), it records the fact log's CURRENT head —
|
|
||||||
* with the set empty, every marker at or below the head has been consumed,
|
|
||||||
* so the next open's recovery fold scans only what comes after. Fire-and-
|
|
||||||
* forget at the drain (close() awaits the core); loud on failure: a missed
|
|
||||||
* write costs the next open a longer scan, never a marker. No-op without a
|
|
||||||
* fact log (no durable markers exist there) and on read-only opens.
|
|
||||||
*/
|
|
||||||
private maybeWriteEmbedLowWater(): void {
|
|
||||||
void this.writeEmbedLowWater()
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The awaitable core of {@link maybeWriteEmbedLowWater} — close() awaits it. */
|
|
||||||
private async writeEmbedLowWater(): Promise<void> {
|
|
||||||
if (this.isReadOnly) return
|
|
||||||
const log = this.generationStore ? this.generationStore.getFactLog() : null
|
|
||||||
if (!log) return
|
|
||||||
const generation = log.headGeneration()
|
|
||||||
if (!(generation > 0)) return
|
|
||||||
try {
|
|
||||||
await this.storage.writeRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH, {
|
|
||||||
generation,
|
|
||||||
writtenAt: Date.now()
|
|
||||||
})
|
|
||||||
} catch (err) {
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] pending-embed low-water write failed at generation ${generation}: ` +
|
|
||||||
`${(err as Error).message} — the next open scans from the previous mark`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -2495,14 +2445,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* survives the fold is exactly the set of acknowledged deferred writes
|
* survives the fold is exactly the set of acknowledged deferred writes
|
||||||
* whose vectors have not landed.
|
* whose vectors have not landed.
|
||||||
*
|
*
|
||||||
* BOUND: the scan starts at the advisory low-water mark
|
* BOUND (honest): no durable low-water mark exists for the earliest
|
||||||
* ({@link Brainy.PENDING_EMBED_LOWWATER_PATH}) — the log head at which the
|
* unconsumed pending, so the fold scans the log's committed facts from
|
||||||
* pending set last drained to empty — so a settled brain reads only the
|
* generation 1 — a sequential read of the log at open, O(log bytes).
|
||||||
* facts since then, not its whole history. Without a mark (first open
|
|
||||||
* after upgrade) it scans from generation 1, once; a stale-low mark costs
|
|
||||||
* a longer scan, never a marker. The fold stays on the open's foreground —
|
|
||||||
* the crash-recovery contract pins that a reopened brain has its markers
|
|
||||||
* re-armed when open() returns — and the mark is what makes that cheap.
|
|
||||||
* It is SKIPPED WHOLESALE when the log has never had a v2 tail
|
* It is SKIPPED WHOLESALE when the log has never had a v2 tail
|
||||||
* ({@link FactLog.hasV2History} — v1 facts cannot carry marker records),
|
* ({@link FactLog.hasV2History} — v1 facts cannot carry marker records),
|
||||||
* so pre-cutover brains pay nothing; on a mixed log the scan still reads
|
* so pre-cutover brains pay nothing; on a mixed log the scan still reads
|
||||||
|
|
@ -2515,18 +2460,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
private async recoverPendingEmbedsFromLog(): Promise<void> {
|
private async recoverPendingEmbedsFromLog(): Promise<void> {
|
||||||
const log = this.generationStore.getFactLog()
|
const log = this.generationStore.getFactLog()
|
||||||
if (!log || !log.hasV2History()) return
|
if (!log || !log.hasV2History()) return
|
||||||
let fromGeneration = 1
|
const scan = log.scanFacts({ fromGeneration: 1 })
|
||||||
try {
|
|
||||||
const mark = (await this.storage.readRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH)) as {
|
|
||||||
generation?: number
|
|
||||||
} | null
|
|
||||||
if (mark && typeof mark.generation === 'number' && mark.generation > 0) {
|
|
||||||
fromGeneration = mark.generation + 1
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// No mark (or unreadable): scan from 1 — correctness over cost.
|
|
||||||
}
|
|
||||||
const scan = log.scanFacts({ fromGeneration })
|
|
||||||
for await (const batch of scan.batches()) {
|
for await (const batch of scan.batches()) {
|
||||||
for (const fact of batch.facts) {
|
for (const fact of batch.facts) {
|
||||||
for (const record of fact.records ?? []) {
|
for (const record of fact.records ?? []) {
|
||||||
|
|
@ -7536,37 +7470,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// JS path — there the materialized `candidateIds` restricts the walk instead.
|
// JS path — there the materialized `candidateIds` restricts the walk instead.
|
||||||
let preResolvedAllowedIds: OpaqueIdSet | undefined
|
let preResolvedAllowedIds: OpaqueIdSet | undefined
|
||||||
|
|
||||||
// Graph-first law (10.4.8, BRAINY-PROD-LATENCY-TRIAD rounds 44/45): with
|
if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
|
||||||
// `connected` present the NEIGHBOUR SET is the candidate universe. It is
|
|
||||||
// resolved first from the adjacency (O(neighbours)), the metadata filter
|
|
||||||
// is evaluated over those ids only, and paging happens LAST. The earlier
|
|
||||||
// order materialized the whole-store filtered id list, paged it, hydrated
|
|
||||||
// the page, and only then intersected with the neighbours — O(store) per
|
|
||||||
// call, and a neighbour outside the first page was silently dropped.
|
|
||||||
let graphFirstIds: string[] | null = null
|
|
||||||
if (hasGraphCriteria) {
|
|
||||||
graphFirstIds = await this.resolveConnectedIds(params)
|
|
||||||
if (hiddenIds.size > 0) {
|
|
||||||
graphFirstIds = graphFirstIds.filter((id) => !hiddenIds.has(id))
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
graphFirstIds.length > 0 &&
|
|
||||||
(params.where || params.type || params.subtype || params.service || params.excludeVFS)
|
|
||||||
) {
|
|
||||||
preResolvedFilter = this.buildMetadataFilter(params)
|
|
||||||
graphFirstIds = await this.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
|
|
||||||
}
|
|
||||||
if (graphFirstIds.length === 0) {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
if (!hasVectorSearchCriteria) {
|
|
||||||
return await this.pageConnectedIds(params, graphFirstIds)
|
|
||||||
}
|
|
||||||
// The vector leg walks ONLY the neighbours (its candidate walk). The
|
|
||||||
// filter is already applied above, so no opaque universe is produced —
|
|
||||||
// it would describe the whole store, not the neighbour set.
|
|
||||||
preResolvedMetadataIds = graphFirstIds
|
|
||||||
} else if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
|
|
||||||
preResolvedFilter = this.buildMetadataFilter(params)
|
preResolvedFilter = this.buildMetadataFilter(params)
|
||||||
preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)
|
preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter)
|
||||||
|
|
||||||
|
|
@ -7755,11 +7659,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The text leg of a hybrid find has no candidate door, so its hits are
|
// Graph search component with O(1) traversal
|
||||||
// held to the neighbour set here; the vector leg walked only the neighbours.
|
if (params.connected) {
|
||||||
if (graphFirstIds !== null && results.length > 0) {
|
results = await this.executeGraphSearch(params, results)
|
||||||
const neighbourSet = new Set(graphFirstIds)
|
|
||||||
results = results.filter((r) => neighbourSet.has(r.id))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply fusion scoring if requested
|
// Apply fusion scoring if requested
|
||||||
|
|
@ -12874,29 +12776,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* The id-scoped twin of {@link filterIdsBelted}: evaluate `filter` over `ids`
|
|
||||||
* only, through the provider's own evaluation so the answer can never drift
|
|
||||||
* from `getIdsForFilter`'s. A provider without the door is served by its
|
|
||||||
* whole-store answer intersected here (the reference index implements the
|
|
||||||
* door itself). Same belt: field refusals cross as `BrainyFieldRefusal`.
|
|
||||||
*/
|
|
||||||
private async filterIdsWithinBelted(filter: unknown, ids: readonly string[]): Promise<string[]> {
|
|
||||||
this.ensureIndexesLoaded(['metadata'])
|
|
||||||
const mip = this.metadataIndex as unknown as MetadataIndexProvider
|
|
||||||
try {
|
|
||||||
if (typeof mip.filterIdsWithin === 'function') {
|
|
||||||
return await mip.filterIdsWithin(filter, ids)
|
|
||||||
}
|
|
||||||
const matched = new Set(await this.metadataIndex.getIdsForFilter(filter))
|
|
||||||
return ids.filter((id) => matched.has(id))
|
|
||||||
} catch (err) {
|
|
||||||
const normalized = asBrainyFieldRefusal(err)
|
|
||||||
if (normalized) throw normalized
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async getIndexStatus(): Promise<{
|
async getIndexStatus(): Promise<{
|
||||||
initialized: boolean
|
initialized: boolean
|
||||||
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
||||||
|
|
@ -15880,16 +15759,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `params.connected` to the neighbour id set — the graph-first
|
* Execute graph search component.
|
||||||
* find's candidate universe (deterministic traversal order, anchors excluded).
|
|
||||||
*
|
*
|
||||||
* Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via
|
* Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via
|
||||||
* `neighbors()`), `via`/`type` verb-type filtering, and `direction`. An empty set
|
* `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read
|
||||||
* is re-verified against the adjacency before it is believed — a not-serving
|
* only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via`
|
||||||
* adjacency throws rather than answering `[]` as truth.
|
* were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the
|
||||||
|
* immediate neighbour at every depth.
|
||||||
*/
|
*/
|
||||||
private async resolveConnectedIds(params: FindParams<T>): Promise<string[]> {
|
private async executeGraphSearch(params: FindParams<T>, existingResults: Result<T>[]): Promise<Result<T>[]> {
|
||||||
if (!params.connected) return []
|
if (!params.connected) return existingResults
|
||||||
|
|
||||||
const { from, to, depth, direction = 'both' } = params.connected
|
const { from, to, depth, direction = 'both' } = params.connected
|
||||||
const via = params.connected.via ?? params.connected.type
|
const via = params.connected.via ?? params.connected.type
|
||||||
|
|
@ -15943,8 +15822,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
if (anchorInt === undefined) return new Set() // unmapped → no relations
|
if (anchorInt === undefined) return new Set() // unmapped → no relations
|
||||||
|
|
||||||
const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
|
const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
|
||||||
// No limit: match the JS BFS exactly — the page is cut downstream,
|
// No limit: match the JS BFS exactly — overall result limiting happens
|
||||||
// after the metadata filter, by pageConnectedIds / the candidate walk.
|
// downstream against existingResults.
|
||||||
const reachedInts = await provider.findConnectedSubtype(
|
const reachedInts = await provider.findConnectedSubtype(
|
||||||
anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
|
anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
|
||||||
)
|
)
|
||||||
|
|
@ -16029,44 +15908,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
await this.verifyGraphAdjacencyLive()
|
await this.verifyGraphAdjacencyLive()
|
||||||
}
|
}
|
||||||
|
|
||||||
return [...connectedIds]
|
// Filter existing results to only connected entities
|
||||||
}
|
if (existingResults.length > 0) {
|
||||||
|
return existingResults.filter(r => connectedIds.has(r.id))
|
||||||
/**
|
|
||||||
* Page and hydrate an already-filtered neighbour set — the pure graph (and
|
|
||||||
* graph + metadata) find's tail. `orderBy` sorts the WHOLE set by field value
|
|
||||||
* before the page is cut (never the page after), null values last on `asc`
|
|
||||||
* and first on `desc`; without `orderBy` the traversal order stands.
|
|
||||||
*/
|
|
||||||
private async pageConnectedIds(params: FindParams<T>, ids: string[]): Promise<Result<T>[]> {
|
|
||||||
const limit = params.limit || 10
|
|
||||||
const offset = params.offset || 0
|
|
||||||
let ordered = ids
|
|
||||||
if (params.orderBy) {
|
|
||||||
const field = params.orderBy
|
|
||||||
const asc = (params.order || 'asc') === 'asc'
|
|
||||||
const valued = await Promise.all(
|
|
||||||
ids.map(async (id) => ({ id, value: await this.metadataIndex.getFieldValueForEntity(id, field) }))
|
|
||||||
)
|
|
||||||
valued.sort((a, b) => {
|
|
||||||
if (a.value == null && b.value == null) return 0
|
|
||||||
if (a.value == null) return asc ? 1 : -1
|
|
||||||
if (b.value == null) return asc ? -1 : 1
|
|
||||||
if (a.value === b.value) return 0
|
|
||||||
const comparison = a.value < b.value ? -1 : 1
|
|
||||||
return asc ? comparison : -comparison
|
|
||||||
})
|
|
||||||
ordered = valued.map((v) => v.id)
|
|
||||||
}
|
}
|
||||||
const pageIds = ordered.slice(offset, offset + limit)
|
|
||||||
const entitiesMap = await this.batchGet(pageIds)
|
// Batch-load connected entities for fast cloud-storage performance
|
||||||
const results: Result<T>[] = []
|
const results: Result<T>[] = []
|
||||||
for (const id of pageIds) {
|
const ids = [...connectedIds]
|
||||||
|
const entitiesMap = await this.batchGet(ids)
|
||||||
|
for (const id of ids) {
|
||||||
const entity = entitiesMap.get(id)
|
const entity = entitiesMap.get(id)
|
||||||
if (entity) {
|
if (entity) {
|
||||||
results.push(this.createResult(id, 1.0, entity))
|
results.push(this.createResult(id, 1.0, entity))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -19586,7 +19443,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* terminal releases have run.
|
* terminal releases have run.
|
||||||
*/
|
*/
|
||||||
async close(): Promise<void> {
|
async close(): Promise<void> {
|
||||||
if (this._pendingEmbedIds.size === 0) await this.writeEmbedLowWater()
|
|
||||||
let closeFailure: unknown = null
|
let closeFailure: unknown = null
|
||||||
try {
|
try {
|
||||||
await this.closeDurableSteps()
|
await this.closeDurableSteps()
|
||||||
|
|
|
||||||
|
|
@ -411,19 +411,6 @@ export interface MetadataIndexProvider {
|
||||||
* @returns The matching id universe as an opaque set.
|
* @returns The matching id universe as an opaque set.
|
||||||
*/
|
*/
|
||||||
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
|
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
|
||||||
/**
|
|
||||||
* @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the
|
|
||||||
* survivors in the caller's order — the door a graph-first
|
|
||||||
* `find({ connected, where })` walks. The neighbour set is the universe there,
|
|
||||||
* so the filter must cost O(|ids|) membership checks, never a whole-store
|
|
||||||
* materialization. A native index answers from its roaring filter result
|
|
||||||
* (membership by entity int); the reference index answers from its own
|
|
||||||
* `getIdsForFilter`, so the two doors can never disagree. Absent → Brainy
|
|
||||||
* intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)).
|
|
||||||
* @param filter - The same filter shape accepted by `getIdsForFilter`.
|
|
||||||
* @param ids - The candidate ids (canonical). The answer is a subsequence.
|
|
||||||
*/
|
|
||||||
filterIdsWithin?(filter: any, ids: readonly string[]): Promise<string[]>
|
|
||||||
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
||||||
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
||||||
getFilterValues(field: string): Promise<string[]>
|
getFilterValues(field: string): Promise<string[]>
|
||||||
|
|
|
||||||
|
|
@ -1089,10 +1089,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
|
|
||||||
// Counts changed since the last persist? Drives the write-through flush.
|
// Counts changed since the last persist? Drives the write-through flush.
|
||||||
protected pendingCountPersist = false
|
protected pendingCountPersist = false
|
||||||
/** The one persist running right now, if any (single-flight law — see flushCounts). */
|
|
||||||
private countPersistInFlight: Promise<void> | null = null
|
|
||||||
/** The one trailing persist a burst has queued behind the in-flight one. */
|
|
||||||
private countPersistTrailing: Promise<void> | null = null
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get total noun count - O(1) operation
|
* Get total noun count - O(1) operation
|
||||||
|
|
@ -1345,46 +1341,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// SINGLE-FLIGHT, COALESCED. Counts are write-through on every change, so
|
try {
|
||||||
// a burst of writes used to launch one persist per change, all in flight
|
// Persist to storage (implemented by subclass)
|
||||||
// together. Two of them inside the same millisecond shared the atomic
|
await this.persistCounts()
|
||||||
// writer's temp path (`.tmp-<pid>-<ms>`): both wrote it, the first rename
|
this.pendingCountPersist = false
|
||||||
// consumed it, the second rename found nothing — ENOENT, ~1,500 times a
|
} catch (error) {
|
||||||
// day on a busy production brain, with a full ledger write per change
|
console.error('CRITICAL: Failed to flush counts to storage:', error)
|
||||||
// behind it. Now exactly one persist runs at a time; requests that arrive
|
// Keep pending flag set so we retry on next operation
|
||||||
// while it runs collapse into ONE trailing persist that carries the final
|
throw error
|
||||||
// state. A burst of N changes costs at most two writes and never races
|
|
||||||
// itself.
|
|
||||||
if (this.countPersistInFlight) {
|
|
||||||
// The in-flight write may have already serialised a stale snapshot —
|
|
||||||
// ask for one more pass after it, and let every caller in this burst
|
|
||||||
// await that same pass.
|
|
||||||
if (!this.countPersistTrailing) {
|
|
||||||
this.countPersistTrailing = this.countPersistInFlight
|
|
||||||
.catch(() => undefined)
|
|
||||||
.then(() => {
|
|
||||||
this.countPersistTrailing = null
|
|
||||||
return this.flushCounts()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return this.countPersistTrailing
|
|
||||||
}
|
}
|
||||||
|
|
||||||
this.countPersistInFlight = (async () => {
|
|
||||||
try {
|
|
||||||
// Persist to storage (implemented by subclass)
|
|
||||||
this.pendingCountPersist = false
|
|
||||||
await this.persistCounts()
|
|
||||||
} catch (error) {
|
|
||||||
// Keep the flag set so the next operation retries.
|
|
||||||
this.pendingCountPersist = true
|
|
||||||
console.error('CRITICAL: Failed to flush counts to storage:', error)
|
|
||||||
throw error
|
|
||||||
} finally {
|
|
||||||
this.countPersistInFlight = null
|
|
||||||
}
|
|
||||||
})()
|
|
||||||
return this.countPersistInFlight
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -2400,15 +2400,8 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
* Atomic write via temp-file-then-rename so concurrent readers never see a
|
||||||
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
* half-written lock JSON. Reused by writer-lock writes + heartbeat.
|
||||||
*/
|
*/
|
||||||
/** Monotonic per-process sequence so two atomic writes never share a temp path. */
|
|
||||||
private static atomicWriteSeq = 0
|
|
||||||
|
|
||||||
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
private async writeFileAtomic(filePath: string, contents: string): Promise<void> {
|
||||||
// pid + timestamp alone collided: two writers of the same target inside
|
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`
|
||||||
// one millisecond shared this path, and the loser's rename found the
|
|
||||||
// winner had already moved it (ENOENT). The sequence makes every call's
|
|
||||||
// temp path its own.
|
|
||||||
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}-${++FileSystemStorage.atomicWriteSeq}`
|
|
||||||
await fs.promises.writeFile(tmp, contents)
|
await fs.promises.writeFile(tmp, contents)
|
||||||
await fs.promises.rename(tmp, filePath)
|
await fs.promises.rename(tmp, filePath)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2942,33 +2942,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
const sourceId = Array.isArray(options.filter.sourceId)
|
||||||
? options.filter.sourceId
|
? options.filter.sourceId[0]
|
||||||
: [options.filter.sourceId]
|
: options.filter.sourceId
|
||||||
|
|
||||||
// EVERY requested verb type is honoured — an array used to collapse to
|
const verbType = Array.isArray(options.filter.verbType)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.verbType[0]
|
||||||
const verbTypes = new Set(
|
: options.filter.verbType
|
||||||
Array.isArray(options.filter.verbType)
|
|
||||||
? options.filter.verbType
|
|
||||||
: [options.filter.verbType]
|
|
||||||
)
|
|
||||||
|
|
||||||
// Get verbs by source (union over every requested source), filter by the
|
// Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter),
|
||||||
// requested type SET (O(1) graph lookup + O(n) type filter), then apply
|
// then apply the subtype / visibility metadata filters on the candidate set.
|
||||||
// the subtype / visibility metadata filters on the candidate set.
|
const verbsBySource = await this.getVerbsBySource_internal(sourceId)
|
||||||
const bySource: HNSWVerbWithMetadata[] = []
|
|
||||||
const seenVerbIds = new Set<string>()
|
|
||||||
for (const oneSource of sourceIds) {
|
|
||||||
for (const v of await this.getVerbsBySource_internal(oneSource)) {
|
|
||||||
if (!seenVerbIds.has(v.id)) {
|
|
||||||
seenVerbIds.add(v.id)
|
|
||||||
bySource.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const filteredVerbs = this.applyVerbMetadataFilters(
|
const filteredVerbs = this.applyVerbMetadataFilters(
|
||||||
bySource.filter(v => verbTypes.has(v.verb)),
|
verbsBySource.filter(v => v.verb === verbType),
|
||||||
options.filter
|
options.filter
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -2999,22 +2985,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
// EVERY requested source is honoured — an array used to collapse to
|
const sourceId = Array.isArray(options.filter.sourceId)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.sourceId[0]
|
||||||
const onlySourceIds = Array.isArray(options.filter.sourceId)
|
: options.filter.sourceId
|
||||||
? options.filter.sourceId
|
|
||||||
: [options.filter.sourceId]
|
// Get verbs by source directly (hydrated with metadata), then apply the
|
||||||
const sourceUnion: HNSWVerbWithMetadata[] = []
|
// subtype / visibility metadata filters on the O(degree) candidate set.
|
||||||
const seenSourceVerbIds = new Set<string>()
|
const verbsBySource = this.applyVerbMetadataFilters(
|
||||||
for (const oneSource of onlySourceIds) {
|
await this.getVerbsBySource_internal(sourceId),
|
||||||
for (const v of await this.getVerbsBySource_internal(oneSource)) {
|
options.filter
|
||||||
if (!seenSourceVerbIds.has(v.id)) {
|
)
|
||||||
seenSourceVerbIds.add(v.id)
|
|
||||||
sourceUnion.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const verbsBySource = this.applyVerbMetadataFilters(sourceUnion, options.filter)
|
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
const paginatedVerbs = verbsBySource.slice(offset, offset + limit)
|
||||||
|
|
@ -3043,22 +3023,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
// EVERY requested target is honoured — an array used to collapse to
|
const targetId = Array.isArray(options.filter.targetId)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.targetId[0]
|
||||||
const onlyTargetIds = Array.isArray(options.filter.targetId)
|
: options.filter.targetId
|
||||||
? options.filter.targetId
|
|
||||||
: [options.filter.targetId]
|
// Get verbs by target directly (hydrated with metadata), then apply the
|
||||||
const targetUnion: HNSWVerbWithMetadata[] = []
|
// subtype / visibility metadata filters on the O(degree) candidate set.
|
||||||
const seenTargetVerbIds = new Set<string>()
|
const verbsByTarget = this.applyVerbMetadataFilters(
|
||||||
for (const oneTarget of onlyTargetIds) {
|
await this.getVerbsByTarget_internal(targetId),
|
||||||
for (const v of await this.getVerbsByTarget_internal(oneTarget)) {
|
options.filter
|
||||||
if (!seenTargetVerbIds.has(v.id)) {
|
)
|
||||||
seenTargetVerbIds.add(v.id)
|
|
||||||
targetUnion.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const verbsByTarget = this.applyVerbMetadataFilters(targetUnion, options.filter)
|
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
const paginatedVerbs = verbsByTarget.slice(offset, offset + limit)
|
||||||
|
|
@ -3087,25 +3061,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
!options.filter.service &&
|
!options.filter.service &&
|
||||||
!options.filter.metadata
|
!options.filter.metadata
|
||||||
) {
|
) {
|
||||||
// EVERY requested verb type is honoured — an array used to collapse to
|
const verbType = Array.isArray(options.filter.verbType)
|
||||||
// its first element here, silently dropping the rest of the ask.
|
? options.filter.verbType[0]
|
||||||
const verbTypes = Array.isArray(options.filter.verbType)
|
: options.filter.verbType
|
||||||
? options.filter.verbType
|
|
||||||
: [options.filter.verbType]
|
|
||||||
|
|
||||||
// Get verbs by each requested type (hydrated with metadata), deduped by
|
// Get verbs by type directly (hydrated with metadata), then apply the
|
||||||
// id, then apply the subtype / visibility metadata filters on the set.
|
// subtype / visibility metadata filters on the candidate set.
|
||||||
const byType: HNSWVerbWithMetadata[] = []
|
const verbsByType = this.applyVerbMetadataFilters(
|
||||||
const seenTypeVerbIds = new Set<string>()
|
await this.getVerbsByType_internal(verbType),
|
||||||
for (const oneType of verbTypes) {
|
options.filter
|
||||||
for (const v of await this.getVerbsByType_internal(oneType)) {
|
)
|
||||||
if (!seenTypeVerbIds.has(v.id)) {
|
|
||||||
seenTypeVerbIds.add(v.id)
|
|
||||||
byType.push(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const verbsByType = this.applyVerbMetadataFilters(byType, options.filter)
|
|
||||||
|
|
||||||
// Apply pagination
|
// Apply pagination
|
||||||
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
const paginatedVerbs = verbsByType.slice(offset, offset + limit)
|
||||||
|
|
|
||||||
|
|
@ -2575,19 +2575,6 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
/** Once-per-field flag for the fallback-degradation announcement. */
|
/** Once-per-field flag for the fallback-degradation announcement. */
|
||||||
private static announcedFallbackSorts = new Set<string>()
|
private static announcedFallbackSorts = new Set<string>()
|
||||||
|
|
||||||
/**
|
|
||||||
* Evaluate `filter` over `ids` only — the graph-first find's door (the
|
|
||||||
* neighbour set filtered by id, never the store filtered and then
|
|
||||||
* intersected). This index answers from its own `getIdsForFilter`, so the
|
|
||||||
* two doors cannot disagree; the cost is that of the filter over this
|
|
||||||
* in-memory index, and the answer keeps the caller's order.
|
|
||||||
*/
|
|
||||||
async filterIdsWithin(filter: any, ids: readonly string[]): Promise<string[]> {
|
|
||||||
if (ids.length === 0) return []
|
|
||||||
const matched = new Set(await this.getIdsForFilter(filter))
|
|
||||||
return ids.filter((id) => matched.has(id))
|
|
||||||
}
|
|
||||||
|
|
||||||
async getSortedIdsForFilter(
|
async getSortedIdsForFilter(
|
||||||
filter: any,
|
filter: any,
|
||||||
orderBy: string,
|
orderBy: string,
|
||||||
|
|
|
||||||
|
|
@ -1,111 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/counts-persist-single-flight
|
|
||||||
* @description Regression for a production race in FileSystemStorage's
|
|
||||||
* counts ledger: `persistCounts()` was write-through on every count change
|
|
||||||
* with no serialization, and the atomic writer named its temp file with
|
|
||||||
* millisecond granularity (`.tmp-<pid>-<ms>`). Two persists inside one
|
|
||||||
* millisecond shared the temp path — both wrote it, the first rename
|
|
||||||
* consumed it, the second rename found nothing: ENOENT, ~1,500 times a day
|
|
||||||
* on a busy production brain, with a full ledger write per change behind it.
|
|
||||||
*
|
|
||||||
* Under pin: persists are single-flight and coalesced — one in flight, at
|
|
||||||
* most one trailing pass carrying the burst's final state — and every atomic
|
|
||||||
* write owns a unique temp path. A burst of N count changes costs at most
|
|
||||||
* two ledger writes, never errors, and leaves a ledger equal to memory.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
||||||
import * as fs from 'node:fs'
|
|
||||||
import * as os from 'node:os'
|
|
||||||
import * as path from 'node:path'
|
|
||||||
import { Brainy } from '../../src/brainy.js'
|
|
||||||
import { NounType } from '../../src/types/graphTypes.js'
|
|
||||||
|
|
||||||
describe('counts persistence is single-flight, coalesced, and never races its own temp file', () => {
|
|
||||||
let dir: string
|
|
||||||
let brain: any
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
||||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-counts-race-'))
|
|
||||||
brain = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
dimensions: 384,
|
|
||||||
silent: true
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
vi.restoreAllMocks()
|
|
||||||
await brain.close()
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a burst of concurrent count changes → at most two ledger writes, zero errors, ledger == memory', async () => {
|
|
||||||
const storage = brain.storage
|
|
||||||
const countsPath: string = storage.countsFilePath
|
|
||||||
expect(countsPath, 'the filesystem adapter persists a counts ledger').toBeTruthy()
|
|
||||||
|
|
||||||
// Let init's own persists settle so the burst is measured alone.
|
|
||||||
await storage.flushCounts?.()
|
|
||||||
|
|
||||||
const renameSpy = vi.spyOn(fs.promises, 'rename')
|
|
||||||
const errorSpy = vi.spyOn(console, 'error')
|
|
||||||
|
|
||||||
// Twenty-five concurrent count changes — the shape of a write burst; each
|
|
||||||
// used to launch its own persist.
|
|
||||||
const BURST = 25
|
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: BURST }, () => storage.scheduleCountPersist())
|
|
||||||
)
|
|
||||||
|
|
||||||
const ledgerRenames = renameSpy.mock.calls.filter(([, to]) => String(to) === countsPath)
|
|
||||||
expect(ledgerRenames.length, 'single-flight + one trailing pass').toBeLessThanOrEqual(2)
|
|
||||||
expect(ledgerRenames.length, 'the burst was persisted at all').toBeGreaterThanOrEqual(1)
|
|
||||||
|
|
||||||
const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts'))
|
|
||||||
expect(persistErrors).toEqual([])
|
|
||||||
|
|
||||||
const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
|
||||||
expect(ledger.totalNounCount).toBe(storage.totalNounCount)
|
|
||||||
expect(ledger.totalVerbCount).toBe(storage.totalVerbCount)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('real writes in parallel: the ledger lands complete and no persist error is logged', async () => {
|
|
||||||
const storage = brain.storage
|
|
||||||
const countsPath: string = storage.countsFilePath
|
|
||||||
const errorSpy = vi.spyOn(console, 'error')
|
|
||||||
|
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: 12 }, (_, i) =>
|
|
||||||
brain.add({ data: `burst row ${i}`, type: NounType.Thing })
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await storage.flushCounts?.()
|
|
||||||
|
|
||||||
const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts'))
|
|
||||||
expect(persistErrors).toEqual([])
|
|
||||||
const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
|
||||||
expect(ledger.totalNounCount).toBe(storage.totalNounCount)
|
|
||||||
expect(await brain.getNounCount()).toBe(ledger.totalNounCount)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('every atomic write owns its own temp path — two writes in one millisecond never collide', async () => {
|
|
||||||
const storage = brain.storage
|
|
||||||
const tmpNames: string[] = []
|
|
||||||
vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (p: any) => {
|
|
||||||
tmpNames.push(String(p))
|
|
||||||
})
|
|
||||||
vi.spyOn(fs.promises, 'rename').mockImplementation(async () => undefined)
|
|
||||||
const target = path.join(dir, 'probe.json')
|
|
||||||
await Promise.all([
|
|
||||||
storage.writeFileAtomic(target, '{"a":1}'),
|
|
||||||
storage.writeFileAtomic(target, '{"a":2}'),
|
|
||||||
storage.writeFileAtomic(target, '{"a":3}')
|
|
||||||
])
|
|
||||||
const probeTmps = tmpNames.filter((n) => n.startsWith(`${target}.tmp-`))
|
|
||||||
expect(probeTmps.length).toBe(3)
|
|
||||||
expect(new Set(probeTmps).size, 'no two writes shared a temp path').toBe(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,165 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/find-connected-order
|
|
||||||
* @description The graph-first law for `find({ connected })` (10.4.8).
|
|
||||||
*
|
|
||||||
* With `connected` present the neighbour set is the candidate universe: it is
|
|
||||||
* resolved from the adjacency first, the metadata filter is evaluated over
|
|
||||||
* those ids only, and the page is cut last. The earlier order materialized the
|
|
||||||
* whole-store filtered id list, paged it, hydrated the page, and only then
|
|
||||||
* intersected with the neighbours — so a neighbour outside the first page of
|
|
||||||
* the filtered STORE was silently dropped, and every call paid O(store).
|
|
||||||
*
|
|
||||||
* These pins hold both halves. The answer: every matching neighbour is
|
|
||||||
* reachable by paging, a non-neighbour never appears, a negation (`missing`)
|
|
||||||
* is evaluated over the neighbours, `orderBy` sorts the whole neighbour set
|
|
||||||
* before the page is cut, and the vector leg walks the neighbours only. The
|
|
||||||
* cost shape: the metadata index is asked about the neighbour ids only, and
|
|
||||||
* hydration is one page — never the store.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
|
||||||
import { Brainy } from '../../src/brainy'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
|
||||||
import { v5 } from '../../src/universal/uuid'
|
|
||||||
import { generateTestVector } from '../helpers/test-factory'
|
|
||||||
|
|
||||||
/** Matching rows that are NOT neighbours — added FIRST, so the whole-store filtered list leads with them. */
|
|
||||||
const NOISE = 120
|
|
||||||
/** Matching rows that ARE neighbours of the anchor. */
|
|
||||||
const NEIGHBOURS = 30
|
|
||||||
/** Neighbours carrying `retracted: true` — excluded by the `missing` negation. */
|
|
||||||
const RETRACTED = 4
|
|
||||||
|
|
||||||
describe('find({ connected }) is graph-first: neighbours → filter → page', () => {
|
|
||||||
let brain: Brainy<any>
|
|
||||||
const anchor = 'anchor'
|
|
||||||
const sharedVector = generateTestVector()
|
|
||||||
const neighbourIds = new Set(Array.from({ length: NEIGHBOURS }, (_, i) => v5(`nb-${i}`)))
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
await brain.add({
|
|
||||||
id: anchor,
|
|
||||||
data: 'the anchor',
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { kind: 'anchor' },
|
|
||||||
vector: generateTestVector()
|
|
||||||
})
|
|
||||||
for (let i = 0; i < NOISE; i++) {
|
|
||||||
await brain.add({
|
|
||||||
id: `noise-${i}`,
|
|
||||||
data: `noise ${i}`,
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { kind: 'note', rank: 1000 + i },
|
|
||||||
vector: sharedVector
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for (let i = 0; i < NEIGHBOURS; i++) {
|
|
||||||
await brain.add({
|
|
||||||
id: `nb-${i}`,
|
|
||||||
data: `neighbour ${i}`,
|
|
||||||
type: NounType.Person,
|
|
||||||
metadata: { kind: 'note', rank: i + 1, ...(i < RETRACTED ? { retracted: true } : {}) },
|
|
||||||
vector: sharedVector
|
|
||||||
})
|
|
||||||
await brain.relate({ from: anchor, to: `nb-${i}`, type: VerbType.Knows })
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
brain = null as any
|
|
||||||
})
|
|
||||||
|
|
||||||
it('returns the matching neighbours page by page — none dropped, never a non-neighbour', async () => {
|
|
||||||
const seen = new Set<string>()
|
|
||||||
for (let offset = 0; offset <= NEIGHBOURS; offset += 10) {
|
|
||||||
const page = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 10,
|
|
||||||
offset
|
|
||||||
})
|
|
||||||
expect(page).toHaveLength(offset < NEIGHBOURS ? 10 : 0)
|
|
||||||
for (const r of page) {
|
|
||||||
expect(neighbourIds.has(r.entity.id)).toBe(true)
|
|
||||||
expect(seen.has(r.entity.id)).toBe(false)
|
|
||||||
seen.add(r.entity.id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expect(seen.size).toBe(NEIGHBOURS)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('evaluates a negation (`missing`) over the neighbour set, not the store', async () => {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note', retracted: { missing: true } },
|
|
||||||
limit: 100
|
|
||||||
})
|
|
||||||
expect(results).toHaveLength(NEIGHBOURS - RETRACTED)
|
|
||||||
for (const r of results) {
|
|
||||||
expect(neighbourIds.has(r.entity.id)).toBe(true)
|
|
||||||
expect(r.entity.metadata.retracted).toBeUndefined()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('asks the metadata index about the neighbour ids only, and hydrates one page', async () => {
|
|
||||||
const index = (brain as any).metadataIndex
|
|
||||||
const within = vi.spyOn(index, 'filterIdsWithin')
|
|
||||||
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
|
||||||
try {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
expect(results).toHaveLength(10)
|
|
||||||
expect(within).toHaveBeenCalledTimes(1)
|
|
||||||
const askedIds = within.mock.calls[0][1] as string[]
|
|
||||||
expect(askedIds).toHaveLength(NEIGHBOURS)
|
|
||||||
for (const id of askedIds) expect(neighbourIds.has(id)).toBe(true)
|
|
||||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
|
||||||
expect(hydrate.mock.calls[0][0]).toHaveLength(10)
|
|
||||||
} finally {
|
|
||||||
within.mockRestore()
|
|
||||||
hydrate.mockRestore()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('orders the WHOLE neighbour set before cutting the page', async () => {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
orderBy: 'rank',
|
|
||||||
order: 'desc',
|
|
||||||
limit: 5
|
|
||||||
})
|
|
||||||
expect(results.map((r) => r.entity.metadata.rank)).toEqual([30, 29, 28, 27, 26])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('walks the vector leg over the neighbours only', async () => {
|
|
||||||
const results = await brain.find({
|
|
||||||
vector: sharedVector,
|
|
||||||
connected: { from: anchor, direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 5
|
|
||||||
})
|
|
||||||
expect(results).toHaveLength(5)
|
|
||||||
for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('an anchor without neighbours answers [] before the filter is asked', async () => {
|
|
||||||
const index = (brain as any).metadataIndex
|
|
||||||
const within = vi.spyOn(index, 'filterIdsWithin')
|
|
||||||
try {
|
|
||||||
const results = await brain.find({
|
|
||||||
connected: { from: 'noise-0', direction: 'out' },
|
|
||||||
where: { kind: 'note' },
|
|
||||||
limit: 10
|
|
||||||
})
|
|
||||||
expect(results).toEqual([])
|
|
||||||
expect(within).not.toHaveBeenCalled()
|
|
||||||
} finally {
|
|
||||||
within.mockRestore()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,141 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/pending-embed-low-water
|
|
||||||
* @description The pending-embed recovery fold is bounded and background (10.4.9).
|
|
||||||
*
|
|
||||||
* The fold used to scan the generation log from generation 1 at EVERY open,
|
|
||||||
* on the open's foreground — O(whole history) per open on long-lived brains.
|
|
||||||
* Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`)
|
|
||||||
* records the committed generation whenever the pending set drains to empty,
|
|
||||||
* recovery scans from `mark + 1` on the open's foreground — the crash-recovery
|
|
||||||
* contract keeps markers re-armed when open() returns. The mark is advisory: stale-low costs a longer scan, never a
|
|
||||||
* marker — a pending embed enqueued before a crash is still recovered.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, afterEach, vi } from 'vitest'
|
|
||||||
import { mkdtempSync, rmSync } from 'node:fs'
|
|
||||||
import { tmpdir } from 'node:os'
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { Brainy } from '../../src/brainy'
|
|
||||||
import { NounType } from '../../src/types/graphTypes'
|
|
||||||
|
|
||||||
const LOWWATER_PATH = '_system/pending_embeds_lowwater.json'
|
|
||||||
|
|
||||||
describe('pending-embed recovery: bounded by the low-water mark', () => {
|
|
||||||
const roots: string[] = []
|
|
||||||
const dir = (): string => {
|
|
||||||
const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-'))
|
|
||||||
roots.push(d)
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
const open = async (root: string): Promise<Brainy<any>> => {
|
|
||||||
const brain = new Brainy<any>({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: root }
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
return brain
|
|
||||||
}
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('drain-to-empty writes the mark, and the next open scans from mark + 1', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
// Hold the worker so the pending state is observable, then release it.
|
|
||||||
const realKick = (brain as any).kickEmbedWorker.bind(brain)
|
|
||||||
;(brain as any).kickEmbedWorker = () => {}
|
|
||||||
await brain.add({
|
|
||||||
id: 'row-1',
|
|
||||||
data: 'the first deferred row',
|
|
||||||
type: NounType.Thing,
|
|
||||||
deferEmbedding: true
|
|
||||||
})
|
|
||||||
expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
|
|
||||||
;(brain as any).kickEmbedWorker = realKick
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
// The drain wrote the advisory mark (fire-and-forget: settle the microtask).
|
|
||||||
await new Promise((r) => setTimeout(r, 50))
|
|
||||||
const mark = (await (brain as any).storage.readRawObject(LOWWATER_PATH)) as {
|
|
||||||
generation: number
|
|
||||||
} | null
|
|
||||||
expect(mark).not.toBeNull()
|
|
||||||
expect(mark!.generation).toBeGreaterThan(0)
|
|
||||||
await brain.close()
|
|
||||||
|
|
||||||
const brain2 = await open(root)
|
|
||||||
const log = (brain2 as any).generationStore.getFactLog()
|
|
||||||
const scanSpy = vi.spyOn(log, 'scanFacts')
|
|
||||||
try {
|
|
||||||
await (brain2 as any).recoverPendingEmbedsFromLog()
|
|
||||||
expect(scanSpy).toHaveBeenCalledTimes(1)
|
|
||||||
const opts = scanSpy.mock.calls[0][0] as { fromGeneration?: number }
|
|
||||||
expect(opts.fromGeneration).toBeGreaterThanOrEqual(mark!.generation + 1)
|
|
||||||
} finally {
|
|
||||||
scanSpy.mockRestore()
|
|
||||||
await brain2.close()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a pending embed enqueued after the mark survives an unclean stop', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
await brain.add({ id: 'settled', data: 'lands before the mark', type: NounType.Thing })
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
await new Promise((r) => setTimeout(r, 50))
|
|
||||||
|
|
||||||
// A deferred write whose embed never lands: block the worker, then drop
|
|
||||||
// the instance without close() — the unclean-stop shape.
|
|
||||||
;(brain as any).kickEmbedWorker = () => {}
|
|
||||||
await brain.add({
|
|
||||||
id: 'orphan',
|
|
||||||
data: 'enqueued then abandoned',
|
|
||||||
type: NounType.Thing,
|
|
||||||
deferEmbedding: true
|
|
||||||
})
|
|
||||||
expect(brain.pendingEmbedCount()).toBeGreaterThan(0)
|
|
||||||
// No close(): simulate the crash by releasing only the writer lock so the
|
|
||||||
// next open can proceed.
|
|
||||||
await (brain as any).storage.releaseWriterLock()
|
|
||||||
|
|
||||||
const brain2 = await open(root)
|
|
||||||
expect(brain2.pendingEmbedCount()).toBeGreaterThan(0)
|
|
||||||
await brain2.awaitPendingEmbeds()
|
|
||||||
expect(brain2.pendingEmbedCount()).toBe(0)
|
|
||||||
await brain2.close()
|
|
||||||
// Reap the crashed instance: its fence is gone, so close() fails loudly —
|
|
||||||
// swallow that here; the point is clearing its watchers and registry entry.
|
|
||||||
await brain.close().catch(() => undefined)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a reopened brain has its pending set settled when open() returns', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing })
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
await brain.close()
|
|
||||||
|
|
||||||
const brain2 = await open(root)
|
|
||||||
// The crash-recovery contract: markers are re-armed by open itself —
|
|
||||||
// no latch, no background race. (Here the drain landed, so zero.)
|
|
||||||
expect(brain2.pendingEmbedCount()).toBe(0)
|
|
||||||
await brain2.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a clean close with an empty set writes the mark even if no drain happened', async () => {
|
|
||||||
const root = dir()
|
|
||||||
const brain = await open(root)
|
|
||||||
await brain.add({ id: 'r1', data: 'row one', type: NounType.Thing })
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
await brain.close()
|
|
||||||
// Read the mark back through the storage door (the adapter owns the
|
|
||||||
// on-disk encoding), on a fresh instance.
|
|
||||||
const brain2 = await open(root)
|
|
||||||
const mark = (await (brain2 as any).storage.readRawObject(LOWWATER_PATH)) as {
|
|
||||||
generation: number
|
|
||||||
} | null
|
|
||||||
expect(mark).not.toBeNull()
|
|
||||||
expect(mark!.generation).toBeGreaterThan(0)
|
|
||||||
await brain2.close()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/related-verb-array
|
|
||||||
* @description related() honours EVERY verb type in an array (10.4.9).
|
|
||||||
*
|
|
||||||
* The storage fast paths for `sourceId + verbType` and `verbType` collapsed a
|
|
||||||
* verb-type ARRAY to its first element — `related({ from, type: [a, b] })`
|
|
||||||
* silently returned only `a` edges, whichever order the array came in. The
|
|
||||||
* same quiet-loss class as the graph-first paging defect, one seam over.
|
|
||||||
* These pins seed a store where the SECOND requested type's edge must come
|
|
||||||
* back, on every path the collapse lived in.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
|
||||||
import { Brainy } from '../../src/brainy'
|
|
||||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
|
||||||
import { v5 } from '../../src/universal/uuid'
|
|
||||||
|
|
||||||
describe('related() with a verb-type array returns every requested type', () => {
|
|
||||||
let brain: Brainy<any>
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
||||||
await brain.init()
|
|
||||||
for (const id of ['a', 'b', 'c', 'd']) {
|
|
||||||
await brain.add({ id, data: `node ${id}`, type: NounType.Person })
|
|
||||||
}
|
|
||||||
await brain.relate({ from: 'a', to: 'b', type: VerbType.Supports })
|
|
||||||
await brain.relate({ from: 'a', to: 'c', type: VerbType.RelatedTo })
|
|
||||||
await brain.relate({ from: 'a', to: 'd', type: VerbType.Knows })
|
|
||||||
await brain.relate({ from: 'b', to: 'c', type: VerbType.RelatedTo })
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
brain = null as any
|
|
||||||
})
|
|
||||||
|
|
||||||
it('from + type array: the second type\'s edge comes back, both orders', async () => {
|
|
||||||
for (const types of [
|
|
||||||
[VerbType.Supports, VerbType.RelatedTo],
|
|
||||||
[VerbType.RelatedTo, VerbType.Supports]
|
|
||||||
]) {
|
|
||||||
const edges = await brain.related({ from: 'a', type: types })
|
|
||||||
const targets = new Set(edges.map((e) => e.to))
|
|
||||||
expect(targets.has(v5('b')), `types [${types}] missing Supports edge`).toBe(true)
|
|
||||||
expect(targets.has(v5('c')), `types [${types}] missing RelatedTo edge`).toBe(true)
|
|
||||||
expect(targets.has(v5('d'))).toBe(false)
|
|
||||||
expect(edges).toHaveLength(2)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a single-element array behaves exactly like the scalar', async () => {
|
|
||||||
const scalar = await brain.related({ from: 'a', type: VerbType.Supports })
|
|
||||||
const array = await brain.related({ from: 'a', type: [VerbType.Supports] })
|
|
||||||
expect(array.map((e) => e.id).sort()).toEqual(scalar.map((e) => e.id).sort())
|
|
||||||
expect(array).toHaveLength(1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('no duplicate edges when types overlap the same edge set', async () => {
|
|
||||||
const edges = await brain.related({
|
|
||||||
from: 'a',
|
|
||||||
type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows]
|
|
||||||
})
|
|
||||||
const ids = edges.map((e) => e.id)
|
|
||||||
expect(new Set(ids).size).toBe(ids.length)
|
|
||||||
expect(edges).toHaveLength(3)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('type-only asks (no anchor) honour the whole array too', async () => {
|
|
||||||
const edges = await brain.related({ type: [VerbType.Supports, VerbType.Knows] })
|
|
||||||
const verbs = new Set(edges.map((e) => e.type))
|
|
||||||
expect(verbs.has(VerbType.Supports)).toBe(true)
|
|
||||||
expect(verbs.has(VerbType.Knows)).toBe(true)
|
|
||||||
expect(edges).toHaveLength(2)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('to + type array: the target side honours every type too', async () => {
|
|
||||||
const edges = await brain.related({ to: 'c', type: [VerbType.RelatedTo, VerbType.Supports] })
|
|
||||||
const froms = new Set(edges.map((e) => e.from))
|
|
||||||
expect(froms.has(v5('a'))).toBe(true)
|
|
||||||
expect(froms.has(v5('b'))).toBe(true)
|
|
||||||
expect(edges).toHaveLength(2)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('pagination stays consistent across the union', async () => {
|
|
||||||
const page1 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2 })
|
|
||||||
const page2 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2, offset: 2 })
|
|
||||||
const all = [...page1, ...page2].map((e) => e.id)
|
|
||||||
expect(new Set(all).size).toBe(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
Reference in a new issue