soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): upsert + FindParams.includeVectors + removeMany adaptive chunking

Browse source 
Three additive ergonomics from the API-simplification audit (no behavior change
to existing call sites):

- AddParams.upsert: create-or-update in one call. With a custom id, an existing
  entity is MERGED via the update path (merges metadata, re-embeds changed data,
  bumps _rev, PRESERVES createdAt) instead of the destructive full overwrite a
  plain add() does. Mutually exclusive with ifAbsent (throws if both set);
  ignored when no id is supplied. Wired into add(), addMany (per-item flag
  propagation), and the transact add op (routes to planTxUpdate). Kills the
  get()-then-add() round-trip for idempotent writes.
- FindParams.includeVectors: mirror of GetOptions.includeVectors — find() returns
  stored vectors when set; default stays empty (the perf contract is preserved).
  Honored on both the query and metadata-only where paths, and in db.find().
- removeMany adaptive chunking: replaced the hardcoded chunkSize=10 with
  params.chunkSize ?? storageConfig.maxBatchSize, matching addMany/relateMany —
  one storage-adaptive batch policy across all *Many methods (no thrash on
  high-latency backends).

Tests: tests/unit/brainy/upsert.test.ts (insert/merge/createdAt-preserved/
re-embed/ifAbsent-conflict/no-id/addMany/transact), find-include-vectors.test.ts
(true/default/where-path), batch-operations.test.ts (removeMany >10 items).
This commit is contained in:
David Snelling 2026-06-20 16:34:20 -07:00
parent 1bc709d31b
commit 4cc2088aed
6 changed files with 481 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

116
src/brainy.ts
View file

@ -1257,6 +1257,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// ifAbsent check so a natural-key upsert is idempotent against the same key.
const { id, originalId } = coerceNewEntityId(params.id)
// ifAbsent and upsert are opposite resolutions of the same id collision —
// ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both
// would be ambiguous, so it is a hard error.
if (params.ifAbsent && params.upsert) {
throw new Error(
'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
)
}
// ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id
// is supplied; a freshly generated UUID can never collide. Returns the existing
// (canonical) id without writing if the entity is already present (no throw,
@ -1266,6 +1275,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (existing) return id
}
// upsert — by-ID create-or-update. Only meaningful when a custom id is supplied
// (a freshly generated UUID can never collide). When the id already exists,
// delegate to update() so the supplied fields MERGE into the existing entity
// (metadata merge, re-embed on changed data, _rev bump, createdAt preserved)
// instead of the default destructive overwrite. When the id is absent, fall
// through to the normal insert below.
if (params.id && params.upsert) {
const existing = await this.storage.getNounMetadata(id)
if (existing) {
await this.update({
id,
...(params.data !== undefined && { data: params.data }),
...(params.type !== undefined && { type: params.type }),
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined && { visibility: params.visibility }),
...(params.metadata !== undefined && { metadata: params.metadata }),
...(params.vector !== undefined && { vector: params.vector }),
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight })
})
return id
}
}
// Get or compute vector
const vector = params.vector || (await this.embed(params.data))
@ -4244,10 +4277,26 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return results.slice(finalOffset, finalOffset + limit)
})()
// includeVectors — opt-in vector hydration. Default (false) keeps the perf
// contract: every result path above builds entities via the metadata-only
// fast path, so `entity.vector` is the empty stub. When requested, fetch the
// stored vectors for the already-paginated page in ONE batch and attach them,
// mirroring get({ includeVectors: true }). Done once here so every find()
// path (metadata-only, vector/text/proximity, graph) honors it uniformly.
if (params.includeVectors && result.length > 0) {
const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id))
for (const r of result) {
const noun = nounsMap.get(r.id)
if (noun?.vector && r.entity) {
r.entity.vector = noun.vector
}
}
}
// Record performance for auto-tuning
const duration = Date.now() - startTime
recordQueryPerformance(duration, result.length)
return result
}
@ -4542,12 +4591,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const promises = chunk.map(async (item) => {
try {
// ifAbsent (7.31.0) — propagate batch-level flag to each item, but let
// per-item flag take precedence so callers can override individual rows.
const itemWithIfAbsent = params.ifAbsent && item.ifAbsent === undefined
? { ...item, ifAbsent: true }
: item
const id = await this.add(itemWithIfAbsent)
// ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every
// item, but let a per-item flag take precedence so callers can override
// individual rows. The two flags stay independent: per-item upsert handling
// (create-or-merge) and per-item ifAbsent handling (create-or-skip) both
// fire inside the delegated add() call, which also enforces their mutual
// exclusion when a single resolved item carries both.
const resolvedItem =
(params.ifAbsent && item.ifAbsent === undefined) ||
(params.upsert && item.upsert === undefined)
? {
...item,
...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }),
...(params.upsert && item.upsert === undefined && { upsert: true })
}
: item
const id = await this.add(resolvedItem)
result.successful.push(id)
} catch (error) {
result.failed.push({
@ -4641,9 +4700,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const startTime = Date.now()
// Batch deletes into chunks for 10x faster performance with proper error handling
// Single transaction per chunk (10 entities) = atomic within chunk, graceful failure across chunks
const chunkSize = 10
// Batch deletes into chunks for 10x faster performance with proper error handling.
// Single transaction per chunk = atomic within chunk, graceful failure across chunks.
// Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's
// maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it.
const storageConfig = this.storage.getBatchConfig()
const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize
for (let i = 0; i < idsToDelete.length; i += chunkSize) {
const chunk = idsToDelete.slice(i, i + chunkSize)
@ -6330,6 +6392,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// natural-key upsert is idempotent against the same key.
const { id, originalId } = coerceNewEntityId(params.id)
// ifAbsent and upsert resolve the same id collision in opposite ways
// (skip vs. merge) — mirror of add()'s hard error.
if (params.ifAbsent && params.upsert) {
throw new Error(
'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
)
}
// ifAbsent — idempotent by-id insert, resolved against batch + store.
if (params.id && params.ifAbsent) {
const existing = await this.planGetEntity(state, id)
@ -6338,6 +6408,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
// upsert — by-id create-or-update, resolved against batch + store. When the id
// already exists, delegate to planTxUpdate with a synthesized update op so the
// supplied fields MERGE (mirror of add()'s live upsert path) instead of
// overwriting. The id is already canonical here (coerceNewEntityId above).
if (params.id && params.upsert) {
const existing = await this.planGetEntity(state, id)
if (existing) {
return this.planTxUpdate(
{
op: 'update',
id,
...(params.data !== undefined && { data: params.data }),
...(params.type !== undefined && { type: params.type }),
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined && { visibility: params.visibility }),
...(params.metadata !== undefined && { metadata: params.metadata }),
...(params.vector !== undefined && { vector: params.vector }),
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight })
} as Extract<TxOperation<T>, { op: 'update' }>,
state,
plan
)
}
}
const vector = params.vector || (await this.embed(params.data))
if (!this.dimensions) {
this.dimensions = vector.length

4
src/db/db.ts
View file

@ -371,12 +371,12 @@ export class Db<T = any> {
entity = await this.host.entityFromRecord(
id,
{ metadata: resolved.metadata, vector: resolved.vector },
false
params.includeVectors ?? false
)
} else if (resolved.source === 'current') {
// Defensive: changed ids always resolve to a record or absent, but
// a 'current' answer is still served correctly from the live path.
entity = await this.host.get(id)
entity = await this.host.get(id, { includeVectors: params.includeVectors ?? false })
}
if (entity && entityMatchesFind(entity as Entity, params as FindParams)) {
merged.push(resultFromEntity(entity))

42
src/types/brainy.types.ts
View file

@ -346,6 +346,24 @@ export interface AddParams<T = any> {
* Ignored when `id` is omitted (a freshly generated UUID can never collide).
*/
ifAbsent?: boolean
/**
* Create-or-update in a single call. When `true` AND a custom `id` is supplied AND
* an entity with that `id` already exists, `add()` applies the supplied fields as an
* UPDATE instead of the default destructive overwrite: metadata is MERGED (not
* replaced), `data` re-embeds only when it changed, `_rev` is bumped, and the
* original `createdAt` is PRESERVED. When no entity with that `id` exists, `add()`
* inserts normally (a fresh `_rev` of 1). Ignored — behaves as a plain insert — when
* `id` is omitted, since a freshly generated UUID can never collide.
*
* Mutually exclusive with {@link AddParams.ifAbsent}: `ifAbsent` SKIPS an existing
* entity, `upsert` MERGES into it — supplying both throws.
*
* @example
* // First call inserts; a later call with the same id merges + bumps _rev.
* await brain.add({ id: 'customer-42', type: 'customer', data: 'Acme', upsert: true })
* await brain.add({ id: 'customer-42', type: 'customer', metadata: { tier: 'gold' }, upsert: true })
*/
upsert?: boolean
}
/**
@ -538,6 +556,14 @@ export interface FindParams<T = any> {
includeRelations?: boolean // Include entity relationships
excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included)
service?: string // Multi-tenancy filter
/**
* Return each result's stored embedding under `entity.vector`. Defaults to `false`,
* in which case `entity.vector` is an empty array (the perf default — vectors are
* large and most reads don't need them). Set `true` to get the persisted vector
* back from `find()` without a recompute, e.g. to feed downstream similarity math.
* Mirrors {@link GetOptions.includeVectors} on `get()`.
*/
includeVectors?: boolean
// Hybrid search options
searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid
@ -738,6 +764,14 @@ export interface AddManyParams<T = any> {
* on each item individually. Item-level `ifAbsent` overrides the batch flag.
*/
ifAbsent?: boolean
/**
* Create-or-update applied to every item. Equivalent to setting `upsert: true` on
* each item individually (see {@link AddParams.upsert}): an item whose custom `id`
* already exists is MERGED as an update rather than overwritten. Item-level `upsert`
* overrides the batch flag. Mutually exclusive with {@link AddManyParams.ifAbsent}
* at the item level — an item resolving to both throws.
*/
upsert?: boolean
}
/**
@ -759,6 +793,14 @@ export interface RemoveManyParams {
type?: NounType // Remove all of type
where?: any // Remove by metadata
limit?: number // Max to remove (safety)
/**
* Entities removed per transaction chunk. Each chunk is one atomic transaction —
* a chunk commits or rolls back together, and failures are isolated to their chunk.
* Defaults to the storage adapter's `maxBatchSize` (memory: 1000, S3/R2: 100,
* GCS: 50), so zero-config removals match the adapter's characteristics. Override
* only to tune throughput vs. transaction granularity.
*/
chunkSize?: number
onProgress?: (done: number, total: number) => void
continueOnError?: boolean // Continue processing if a removal fails
}