feat(8.0): upsert + FindParams.includeVectors + removeMany adaptive chunking
Three additive ergonomics from the API-simplification audit (no behavior change to existing call sites): - AddParams.upsert: create-or-update in one call. With a custom id, an existing entity is MERGED via the update path (merges metadata, re-embeds changed data, bumps _rev, PRESERVES createdAt) instead of the destructive full overwrite a plain add() does. Mutually exclusive with ifAbsent (throws if both set); ignored when no id is supplied. Wired into add(), addMany (per-item flag propagation), and the transact add op (routes to planTxUpdate). Kills the get()-then-add() round-trip for idempotent writes. - FindParams.includeVectors: mirror of GetOptions.includeVectors — find() returns stored vectors when set; default stays empty (the perf contract is preserved). Honored on both the query and metadata-only where paths, and in db.find(). - removeMany adaptive chunking: replaced the hardcoded chunkSize=10 with params.chunkSize ?? storageConfig.maxBatchSize, matching addMany/relateMany — one storage-adaptive batch policy across all *Many methods (no thrash on high-latency backends). Tests: tests/unit/brainy/upsert.test.ts (insert/merge/createdAt-preserved/ re-embed/ifAbsent-conflict/no-id/addMany/transact), find-include-vectors.test.ts (true/default/where-path), batch-operations.test.ts (removeMany >10 items).
This commit is contained in:
parent
1bc709d31b
commit
4cc2088aed
6 changed files with 481 additions and 12 deletions
116
src/brainy.ts
116
src/brainy.ts
|
|
@ -1257,6 +1257,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// ifAbsent check so a natural-key upsert is idempotent against the same key.
|
||||
const { id, originalId } = coerceNewEntityId(params.id)
|
||||
|
||||
// ifAbsent and upsert are opposite resolutions of the same id collision —
|
||||
// ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both
|
||||
// would be ambiguous, so it is a hard error.
|
||||
if (params.ifAbsent && params.upsert) {
|
||||
throw new Error(
|
||||
'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
|
||||
)
|
||||
}
|
||||
|
||||
// ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id
|
||||
// is supplied; a freshly generated UUID can never collide. Returns the existing
|
||||
// (canonical) id without writing if the entity is already present (no throw,
|
||||
|
|
@ -1266,6 +1275,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
if (existing) return id
|
||||
}
|
||||
|
||||
// upsert — by-ID create-or-update. Only meaningful when a custom id is supplied
|
||||
// (a freshly generated UUID can never collide). When the id already exists,
|
||||
// delegate to update() so the supplied fields MERGE into the existing entity
|
||||
// (metadata merge, re-embed on changed data, _rev bump, createdAt preserved)
|
||||
// instead of the default destructive overwrite. When the id is absent, fall
|
||||
// through to the normal insert below.
|
||||
if (params.id && params.upsert) {
|
||||
const existing = await this.storage.getNounMetadata(id)
|
||||
if (existing) {
|
||||
await this.update({
|
||||
id,
|
||||
...(params.data !== undefined && { data: params.data }),
|
||||
...(params.type !== undefined && { type: params.type }),
|
||||
...(params.subtype !== undefined && { subtype: params.subtype }),
|
||||
...(params.visibility !== undefined && { visibility: params.visibility }),
|
||||
...(params.metadata !== undefined && { metadata: params.metadata }),
|
||||
...(params.vector !== undefined && { vector: params.vector }),
|
||||
...(params.confidence !== undefined && { confidence: params.confidence }),
|
||||
...(params.weight !== undefined && { weight: params.weight })
|
||||
})
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
// Get or compute vector
|
||||
const vector = params.vector || (await this.embed(params.data))
|
||||
|
||||
|
|
@ -4244,10 +4277,26 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return results.slice(finalOffset, finalOffset + limit)
|
||||
})()
|
||||
|
||||
// includeVectors — opt-in vector hydration. Default (false) keeps the perf
|
||||
// contract: every result path above builds entities via the metadata-only
|
||||
// fast path, so `entity.vector` is the empty stub. When requested, fetch the
|
||||
// stored vectors for the already-paginated page in ONE batch and attach them,
|
||||
// mirroring get({ includeVectors: true }). Done once here so every find()
|
||||
// path (metadata-only, vector/text/proximity, graph) honors it uniformly.
|
||||
if (params.includeVectors && result.length > 0) {
|
||||
const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id))
|
||||
for (const r of result) {
|
||||
const noun = nounsMap.get(r.id)
|
||||
if (noun?.vector && r.entity) {
|
||||
r.entity.vector = noun.vector
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Record performance for auto-tuning
|
||||
const duration = Date.now() - startTime
|
||||
recordQueryPerformance(duration, result.length)
|
||||
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
|
@ -4542,12 +4591,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
const promises = chunk.map(async (item) => {
|
||||
try {
|
||||
// ifAbsent (7.31.0) — propagate batch-level flag to each item, but let
|
||||
// per-item flag take precedence so callers can override individual rows.
|
||||
const itemWithIfAbsent = params.ifAbsent && item.ifAbsent === undefined
|
||||
? { ...item, ifAbsent: true }
|
||||
: item
|
||||
const id = await this.add(itemWithIfAbsent)
|
||||
// ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every
|
||||
// item, but let a per-item flag take precedence so callers can override
|
||||
// individual rows. The two flags stay independent: per-item upsert handling
|
||||
// (create-or-merge) and per-item ifAbsent handling (create-or-skip) both
|
||||
// fire inside the delegated add() call, which also enforces their mutual
|
||||
// exclusion when a single resolved item carries both.
|
||||
const resolvedItem =
|
||||
(params.ifAbsent && item.ifAbsent === undefined) ||
|
||||
(params.upsert && item.upsert === undefined)
|
||||
? {
|
||||
...item,
|
||||
...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }),
|
||||
...(params.upsert && item.upsert === undefined && { upsert: true })
|
||||
}
|
||||
: item
|
||||
const id = await this.add(resolvedItem)
|
||||
result.successful.push(id)
|
||||
} catch (error) {
|
||||
result.failed.push({
|
||||
|
|
@ -4641,9 +4700,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
const startTime = Date.now()
|
||||
|
||||
// Batch deletes into chunks for 10x faster performance with proper error handling
|
||||
// Single transaction per chunk (10 entities) = atomic within chunk, graceful failure across chunks
|
||||
const chunkSize = 10
|
||||
// Batch deletes into chunks for 10x faster performance with proper error handling.
|
||||
// Single transaction per chunk = atomic within chunk, graceful failure across chunks.
|
||||
// Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's
|
||||
// maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it.
|
||||
const storageConfig = this.storage.getBatchConfig()
|
||||
const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize
|
||||
|
||||
for (let i = 0; i < idsToDelete.length; i += chunkSize) {
|
||||
const chunk = idsToDelete.slice(i, i + chunkSize)
|
||||
|
|
@ -6330,6 +6392,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// natural-key upsert is idempotent against the same key.
|
||||
const { id, originalId } = coerceNewEntityId(params.id)
|
||||
|
||||
// ifAbsent and upsert resolve the same id collision in opposite ways
|
||||
// (skip vs. merge) — mirror of add()'s hard error.
|
||||
if (params.ifAbsent && params.upsert) {
|
||||
throw new Error(
|
||||
'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.'
|
||||
)
|
||||
}
|
||||
|
||||
// ifAbsent — idempotent by-id insert, resolved against batch + store.
|
||||
if (params.id && params.ifAbsent) {
|
||||
const existing = await this.planGetEntity(state, id)
|
||||
|
|
@ -6338,6 +6408,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// upsert — by-id create-or-update, resolved against batch + store. When the id
|
||||
// already exists, delegate to planTxUpdate with a synthesized update op so the
|
||||
// supplied fields MERGE (mirror of add()'s live upsert path) instead of
|
||||
// overwriting. The id is already canonical here (coerceNewEntityId above).
|
||||
if (params.id && params.upsert) {
|
||||
const existing = await this.planGetEntity(state, id)
|
||||
if (existing) {
|
||||
return this.planTxUpdate(
|
||||
{
|
||||
op: 'update',
|
||||
id,
|
||||
...(params.data !== undefined && { data: params.data }),
|
||||
...(params.type !== undefined && { type: params.type }),
|
||||
...(params.subtype !== undefined && { subtype: params.subtype }),
|
||||
...(params.visibility !== undefined && { visibility: params.visibility }),
|
||||
...(params.metadata !== undefined && { metadata: params.metadata }),
|
||||
...(params.vector !== undefined && { vector: params.vector }),
|
||||
...(params.confidence !== undefined && { confidence: params.confidence }),
|
||||
...(params.weight !== undefined && { weight: params.weight })
|
||||
} as Extract<TxOperation<T>, { op: 'update' }>,
|
||||
state,
|
||||
plan
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const vector = params.vector || (await this.embed(params.data))
|
||||
if (!this.dimensions) {
|
||||
this.dimensions = vector.length
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue