soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): upsert + FindParams.includeVectors + removeMany adaptive chunking

Browse source 
Three additive ergonomics from the API-simplification audit (no behavior change
to existing call sites):

- AddParams.upsert: create-or-update in one call. With a custom id, an existing
  entity is MERGED via the update path (merges metadata, re-embeds changed data,
  bumps _rev, PRESERVES createdAt) instead of the destructive full overwrite a
  plain add() does. Mutually exclusive with ifAbsent (throws if both set);
  ignored when no id is supplied. Wired into add(), addMany (per-item flag
  propagation), and the transact add op (routes to planTxUpdate). Kills the
  get()-then-add() round-trip for idempotent writes.
- FindParams.includeVectors: mirror of GetOptions.includeVectors — find() returns
  stored vectors when set; default stays empty (the perf contract is preserved).
  Honored on both the query and metadata-only where paths, and in db.find().
- removeMany adaptive chunking: replaced the hardcoded chunkSize=10 with
  params.chunkSize ?? storageConfig.maxBatchSize, matching addMany/relateMany —
  one storage-adaptive batch policy across all *Many methods (no thrash on
  high-latency backends).

Tests: tests/unit/brainy/upsert.test.ts (insert/merge/createdAt-preserved/
re-embed/ifAbsent-conflict/no-id/addMany/transact), find-include-vectors.test.ts
(true/default/where-path), batch-operations.test.ts (removeMany >10 items).
This commit is contained in:
David Snelling 2026-06-20 16:34:20 -07:00
parent 1bc709d31b
commit 4cc2088aed
6 changed files with 481 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

112
src/brainy.ts
View file

@ -1257,6 +1257,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// ifAbsent check so a natural-key upsert is idempotent against the same key.
const { id, originalId } = coerceNewEntityId(params.id)
// ifAbsent and upsert are opposite resolutions of the same id collision —
// ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both
// would be ambiguous, so it is a hard error.
if (params.ifAbsent && params.upsert) {
throw new Error(
'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
)
}
// ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id
// is supplied; a freshly generated UUID can never collide. Returns the existing
// (canonical) id without writing if the entity is already present (no throw,
@ -1266,6 +1275,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (existing) return id
}
// upsert — by-ID create-or-update. Only meaningful when a custom id is supplied
// (a freshly generated UUID can never collide). When the id already exists,
// delegate to update() so the supplied fields MERGE into the existing entity
// (metadata merge, re-embed on changed data, _rev bump, createdAt preserved)
// instead of the default destructive overwrite. When the id is absent, fall
// through to the normal insert below.
if (params.id && params.upsert) {
const existing = await this.storage.getNounMetadata(id)
if (existing) {
await this.update({
id,
...(params.data !== undefined && { data: params.data }),
...(params.type !== undefined && { type: params.type }),
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined && { visibility: params.visibility }),
...(params.metadata !== undefined && { metadata: params.metadata }),
...(params.vector !== undefined && { vector: params.vector }),
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight })
})
return id
}
}
// Get or compute vector
const vector = params.vector || (await this.embed(params.data))
@ -4244,6 +4277,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return results.slice(finalOffset, finalOffset + limit)
})()
// includeVectors — opt-in vector hydration. Default (false) keeps the perf
// contract: every result path above builds entities via the metadata-only
// fast path, so `entity.vector` is the empty stub. When requested, fetch the
// stored vectors for the already-paginated page in ONE batch and attach them,
// mirroring get({ includeVectors: true }). Done once here so every find()
// path (metadata-only, vector/text/proximity, graph) honors it uniformly.
if (params.includeVectors && result.length > 0) {
const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id))
for (const r of result) {
const noun = nounsMap.get(r.id)
if (noun?.vector && r.entity) {
r.entity.vector = noun.vector
}
}
}
// Record performance for auto-tuning
const duration = Date.now() - startTime
recordQueryPerformance(duration, result.length)
@ -4542,12 +4591,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const promises = chunk.map(async (item) => {
try {
// ifAbsent (7.31.0) — propagate batch-level flag to each item, but let
// per-item flag take precedence so callers can override individual rows.
const itemWithIfAbsent = params.ifAbsent && item.ifAbsent === undefined
? { ...item, ifAbsent: true }
// ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every
// item, but let a per-item flag take precedence so callers can override
// individual rows. The two flags stay independent: per-item upsert handling
// (create-or-merge) and per-item ifAbsent handling (create-or-skip) both
// fire inside the delegated add() call, which also enforces their mutual
// exclusion when a single resolved item carries both.
const resolvedItem =
(params.ifAbsent && item.ifAbsent === undefined) ||
(params.upsert && item.upsert === undefined)
? {
...item,
...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }),
...(params.upsert && item.upsert === undefined && { upsert: true })
}
: item
const id = await this.add(itemWithIfAbsent)
const id = await this.add(resolvedItem)
result.successful.push(id)
} catch (error) {
result.failed.push({
@ -4641,9 +4700,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const startTime = Date.now()
// Batch deletes into chunks for 10x faster performance with proper error handling
// Single transaction per chunk (10 entities) = atomic within chunk, graceful failure across chunks
const chunkSize = 10
// Batch deletes into chunks for 10x faster performance with proper error handling.
// Single transaction per chunk = atomic within chunk, graceful failure across chunks.
// Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's
// maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it.
const storageConfig = this.storage.getBatchConfig()
const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize
for (let i = 0; i < idsToDelete.length; i += chunkSize) {
const chunk = idsToDelete.slice(i, i + chunkSize)
@ -6330,6 +6392,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// natural-key upsert is idempotent against the same key.
const { id, originalId } = coerceNewEntityId(params.id)
// ifAbsent and upsert resolve the same id collision in opposite ways
// (skip vs. merge) — mirror of add()'s hard error.
if (params.ifAbsent && params.upsert) {
throw new Error(
'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
)
}
// ifAbsent — idempotent by-id insert, resolved against batch + store.
if (params.id && params.ifAbsent) {
const existing = await this.planGetEntity(state, id)
@ -6338,6 +6408,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
// upsert — by-id create-or-update, resolved against batch + store. When the id
// already exists, delegate to planTxUpdate with a synthesized update op so the
// supplied fields MERGE (mirror of add()'s live upsert path) instead of
// overwriting. The id is already canonical here (coerceNewEntityId above).
if (params.id && params.upsert) {
const existing = await this.planGetEntity(state, id)
if (existing) {
return this.planTxUpdate(
{
op: 'update',
id,
...(params.data !== undefined && { data: params.data }),
...(params.type !== undefined && { type: params.type }),
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined && { visibility: params.visibility }),
...(params.metadata !== undefined && { metadata: params.metadata }),
...(params.vector !== undefined && { vector: params.vector }),
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight })
} as Extract<TxOperation<T>, { op: 'update' }>,
state,
plan
)
}
}
const vector = params.vector || (await this.embed(params.data))
if (!this.dimensions) {
this.dimensions = vector.length

4
src/db/db.ts
View file

@ -371,12 +371,12 @@ export class Db<T = any> {
entity = await this.host.entityFromRecord(
id,
{ metadata: resolved.metadata, vector: resolved.vector },
false
params.includeVectors ?? false
)
} else if (resolved.source === 'current') {
// Defensive: changed ids always resolve to a record or absent, but
// a 'current' answer is still served correctly from the live path.
entity = await this.host.get(id)
entity = await this.host.get(id, { includeVectors: params.includeVectors ?? false })
}
if (entity && entityMatchesFind(entity as Entity, params as FindParams)) {
merged.push(resultFromEntity(entity))

42
src/types/brainy.types.ts
View file

@ -346,6 +346,24 @@ export interface AddParams<T = any> {
* Ignored when `id` is omitted (a freshly generated UUID can never collide).
*/
ifAbsent?: boolean
/**
* Create-or-update in a single call. When `true` AND a custom `id` is supplied AND
* an entity with that `id` already exists, `add()` applies the supplied fields as an
* UPDATE instead of the default destructive overwrite: metadata is MERGED (not
* replaced), `data` re-embeds only when it changed, `_rev` is bumped, and the
* original `createdAt` is PRESERVED. When no entity with that `id` exists, `add()`
* inserts normally (a fresh `_rev` of 1). Ignored — behaves as a plain insert — when
* `id` is omitted, since a freshly generated UUID can never collide.
*
* Mutually exclusive with {@link AddParams.ifAbsent}: `ifAbsent` SKIPS an existing
* entity, `upsert` MERGES into it — supplying both throws.
*
* @example
* // First call inserts; a later call with the same id merges + bumps _rev.
* await brain.add({ id: 'customer-42', type: 'customer', data: 'Acme', upsert: true })
* await brain.add({ id: 'customer-42', type: 'customer', metadata: { tier: 'gold' }, upsert: true })
*/
upsert?: boolean
}
/**
@ -538,6 +556,14 @@ export interface FindParams<T = any> {
includeRelations?: boolean // Include entity relationships
excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included)
service?: string // Multi-tenancy filter
/**
* Return each result's stored embedding under `entity.vector`. Defaults to `false`,
* in which case `entity.vector` is an empty array (the perf default — vectors are
* large and most reads don't need them). Set `true` to get the persisted vector
* back from `find()` without a recompute, e.g. to feed downstream similarity math.
* Mirrors {@link GetOptions.includeVectors} on `get()`.
*/
includeVectors?: boolean
// Hybrid search options
searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid
@ -738,6 +764,14 @@ export interface AddManyParams<T = any> {
* on each item individually. Item-level `ifAbsent` overrides the batch flag.
*/
ifAbsent?: boolean
/**
* Create-or-update applied to every item. Equivalent to setting `upsert: true` on
* each item individually (see {@link AddParams.upsert}): an item whose custom `id`
* already exists is MERGED as an update rather than overwritten. Item-level `upsert`
* overrides the batch flag. Mutually exclusive with {@link AddManyParams.ifAbsent}
* at the item level — an item resolving to both throws.
*/
upsert?: boolean
}
/**
@ -759,6 +793,14 @@ export interface RemoveManyParams {
type?: NounType // Remove all of type
where?: any // Remove by metadata
limit?: number // Max to remove (safety)
/**
* Entities removed per transaction chunk. Each chunk is one atomic transaction —
* a chunk commits or rolls back together, and failures are isolated to their chunk.
* Defaults to the storage adapter's `maxBatchSize` (memory: 1000, S3/R2: 100,
* GCS: 50), so zero-config removals match the adapter's characteristics. Override
* only to tune throughput vs. transaction granularity.
*/
chunkSize?: number
onProgress?: (done: number, total: number) => void
continueOnError?: boolean // Continue processing if a removal fails
}

43
tests/unit/brainy/batch-operations.test.ts
View file

@ -180,6 +180,49 @@ describe('Brainy Batch Operations', () => {
}
})
it('removes more than the legacy 10-item chunk in a single call (adaptive chunking)', async () => {
// Seed 25 entities — more than the old hardcoded chunk of 10 — to prove the
// storage-adaptive chunk size processes every chunk and nothing is capped.
const seed = await brain.addMany({
items: Array.from({ length: 25 }, (_, i) => ({
data: `Adaptive Chunk ${i}`,
type: NounType.Thing,
metadata: { batch: 'adaptive' }
}))
})
expect(seed.successful).toHaveLength(25)
const result = await brain.removeMany({ ids: seed.successful })
expect(result.successful).toHaveLength(25)
expect(result.failed).toHaveLength(0)
expect(result.total).toBe(25)
// Every entity is actually gone.
for (const id of seed.successful) {
expect(await brain.get(id)).toBeNull()
}
})
it('removes all entities across multiple chunks when chunkSize is overridden', async () => {
const seed = await brain.addMany({
items: Array.from({ length: 12 }, (_, i) => ({
data: `Override Chunk ${i}`,
type: NounType.Thing,
metadata: { batch: 'override' }
}))
})
expect(seed.successful).toHaveLength(12)
// chunkSize 5 → 3 chunks (5 + 5 + 2); all must be removed.
const result = await brain.removeMany({ ids: seed.successful, chunkSize: 5 })
expect(result.successful).toHaveLength(12)
expect(result.failed).toHaveLength(0)
for (const id of seed.successful) {
expect(await brain.get(id)).toBeNull()
}
})
it('should handle selective deletion', async () => {
// Delete only some
const toDelete = [testIds[0], testIds[2], testIds[4]]

94
tests/unit/brainy/find-include-vectors.test.ts Normal file
View file

@ -0,0 +1,94 @@
/**
* Unit tests for FindParams.includeVectors — opt-in vector hydration on find().
*
* Verifies that find({ includeVectors: true }) returns each result's stored
* embedding under entity.vector (matching get({ includeVectors: true })), that
* the default keeps the perf contract of an empty vector, and that the
* metadata-only find({ where }) path honors the flag too.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../../src/brainy'
import { createTestConfig } from '../../helpers/test-factory'
describe('FindParams.includeVectors', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
})
afterEach(async () => {
await brain.close()
})
it('returns the stored vector for query-based find when includeVectors is true', async () => {
const id = await brain.add({
type: 'document',
data: 'machine learning and neural networks',
metadata: { topic: 'ai' }
})
// Stored vector, as get() surfaces it — the reference for the find() result.
const stored = await brain.get(id, { includeVectors: true })
expect(stored!.vector.length).toBeGreaterThan(0)
const results = await brain.find({
query: 'machine learning and neural networks',
includeVectors: true,
limit: 10
})
const hit = results.find((r) => r.entity.id === id)
expect(hit).toBeDefined()
expect(hit!.entity.vector.length).toBeGreaterThan(0)
expect(hit!.entity.vector).toEqual(stored!.vector)
})
it('returns an empty vector by default (perf contract preserved)', async () => {
const id = await brain.add({
type: 'document',
data: 'default path should not hydrate vectors',
metadata: { topic: 'perf' }
})
const results = await brain.find({
query: 'default path should not hydrate vectors',
limit: 10
})
const hit = results.find((r) => r.entity.id === id)
expect(hit).toBeDefined()
expect(hit!.entity.vector.length).toBe(0)
})
it('honors includeVectors on the metadata-only where path', async () => {
const id = await brain.add({
type: 'thing',
data: 'metadata only filter target',
metadata: { category: 'filtered' }
})
const stored = await brain.get(id, { includeVectors: true })
const withVectors = await brain.find({
where: { category: 'filtered' },
includeVectors: true,
limit: 10
})
const hit = withVectors.find((r) => r.entity.id === id)
expect(hit).toBeDefined()
expect(hit!.entity.vector.length).toBeGreaterThan(0)
expect(hit!.entity.vector).toEqual(stored!.vector)
// And the same filter without the flag stays empty.
const withoutVectors = await brain.find({
where: { category: 'filtered' },
limit: 10
})
const plainHit = withoutVectors.find((r) => r.entity.id === id)
expect(plainHit).toBeDefined()
expect(plainHit!.entity.vector.length).toBe(0)
})
})

194
tests/unit/brainy/upsert.test.ts Normal file
View file

@ -0,0 +1,194 @@
/**
* Unit tests for AddParams.upsert — create-or-update in a single add() call.
*
* Covers the live add() path and the addMany batch flag. Verifies that upsert
* inserts on a fresh id, merges (not overwrites) into an existing id while
* preserving createdAt and bumping _rev, is mutually exclusive with ifAbsent,
* and is a plain insert when no id is supplied.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../../src/brainy'
import { createTestConfig } from '../../helpers/test-factory'
describe('AddParams.upsert', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
})
afterEach(async () => {
await brain.close()
})
it('inserts a new entity when the id does not exist (_rev 1)', async () => {
// 8.0 normalizes a natural string key to a stable canonical UUID; the entity is
// still reachable by the natural key via get()'s id resolution.
const id = await brain.add({
id: 'customer-1',
type: 'thing',
data: 'Acme Corp',
metadata: { tier: 'silver' },
upsert: true
})
expect(typeof id).toBe('string')
const entity = await brain.get('customer-1')
expect(entity).not.toBeNull()
expect(entity!.type).toBe('thing')
expect(entity!.metadata.tier).toBe('silver')
expect(entity!._rev).toBe(1)
})
it('merges into an existing entity, preserves createdAt, bumps _rev, applies changed data', async () => {
// Seed the entity.
await brain.add({
id: 'invoice-9',
type: 'thing',
data: 'Original line items',
metadata: { status: 'draft', amount: 100 }
})
const before = await brain.get('invoice-9')
expect(before).not.toBeNull()
const originalCreatedAt = before!.createdAt
expect(before!._rev).toBe(1)
// Upsert over the existing id with a partial metadata patch + new data.
await brain.add({
id: 'invoice-9',
type: 'thing',
data: 'Revised line items',
metadata: { status: 'sent' },
upsert: true
})
const after = await brain.get('invoice-9')
expect(after).not.toBeNull()
// Metadata MERGED, not replaced — the untouched 'amount' field survives.
expect(after!.metadata.status).toBe('sent')
expect(after!.metadata.amount).toBe(100)
// Changed data applied.
expect(after!.data).toBe('Revised line items')
// _rev bumped, createdAt preserved.
expect(after!._rev).toBe(2)
expect(after!.createdAt).toBe(originalCreatedAt)
})
it('re-embeds when data changes on upsert', async () => {
await brain.add({
id: 'doc-vec',
type: 'document',
data: 'cats and dogs and small furry animals'
})
const before = await brain.get('doc-vec', { includeVectors: true })
expect(before!.vector.length).toBeGreaterThan(0)
await brain.add({
id: 'doc-vec',
type: 'document',
data: 'quarterly financial revenue projections spreadsheet',
upsert: true
})
const after = await brain.get('doc-vec', { includeVectors: true })
expect(after!.vector.length).toBe(before!.vector.length)
// A genuinely different document re-embeds to a different vector.
expect(after!.vector).not.toEqual(before!.vector)
})
it('throws when both ifAbsent and upsert are true', async () => {
await expect(
brain.add({
id: 'conflict-1',
type: 'thing',
data: 'x',
ifAbsent: true,
upsert: true
})
).rejects.toThrow(/mutually exclusive/)
})
it('behaves like a plain insert when no id is supplied', async () => {
const id = await brain.add({
type: 'thing',
data: 'no id supplied',
metadata: { k: 'v' },
upsert: true
})
expect(typeof id).toBe('string')
const entity = await brain.get(id)
expect(entity).not.toBeNull()
expect(entity!._rev).toBe(1)
expect(entity!.metadata.k).toBe('v')
})
it('merges an existing item via the addMany batch upsert flag', async () => {
// Seed an item that the batch will later upsert into.
await brain.add({
id: 'order-1',
type: 'thing',
data: 'first',
metadata: { region: 'west', count: 1 }
})
const seeded = await brain.get('order-1')
const seededCreatedAt = seeded!.createdAt
const result = await brain.addMany({
upsert: true,
items: [
{ id: 'order-1', type: 'thing', data: 'updated', metadata: { count: 2 } },
{ id: 'order-2', type: 'thing', data: 'brand new', metadata: { region: 'east' } }
]
})
// Both items resolved (returned ids are the canonical UUIDs the natural keys map to).
expect(result.failed).toHaveLength(0)
expect(result.successful).toHaveLength(2)
// Existing item merged (region survived), _rev bumped, createdAt preserved.
const merged = await brain.get('order-1')
expect(merged!.metadata.region).toBe('west')
expect(merged!.metadata.count).toBe(2)
expect(merged!.data).toBe('updated')
expect(merged!._rev).toBe(2)
expect(merged!.createdAt).toBe(seededCreatedAt)
// New item inserted fresh.
const fresh = await brain.get('order-2')
expect(fresh!._rev).toBe(1)
expect(fresh!.metadata.region).toBe('east')
})
it('upserts inside a transact batch — existing id merges, new id inserts', async () => {
await brain.add({
id: 'acct-1',
type: 'thing',
data: 'seed',
metadata: { plan: 'free', seats: 1 }
})
const seeded = await brain.get('acct-1')
const seededCreatedAt = seeded!.createdAt
await brain.transact([
{ op: 'add', id: 'acct-1', type: 'thing', data: 'seed', metadata: { plan: 'pro' }, upsert: true },
{ op: 'add', id: 'acct-2', type: 'thing', data: 'new account', metadata: { plan: 'free' }, upsert: true }
])
const merged = await brain.get('acct-1')
expect(merged!.metadata.plan).toBe('pro')
expect(merged!.metadata.seats).toBe(1) // merged, not overwritten
expect(merged!._rev).toBe(2)
expect(merged!.createdAt).toBe(seededCreatedAt)
const fresh = await brain.get('acct-2')
expect(fresh!._rev).toBe(1)
expect(fresh!.metadata.plan).toBe('free')
})
})