diff --git a/src/brainy.ts b/src/brainy.ts index b90194d1..e575c15e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1257,6 +1257,15 @@ export class Brainy implements BrainyInterface { // ifAbsent check so a natural-key upsert is idempotent against the same key. const { id, originalId } = coerceNewEntityId(params.id) + // ifAbsent and upsert are opposite resolutions of the same id collision — + // ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both + // would be ambiguous, so it is a hard error. + if (params.ifAbsent && params.upsert) { + throw new Error( + 'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.' + ) + } + // ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id // is supplied; a freshly generated UUID can never collide. Returns the existing // (canonical) id without writing if the entity is already present (no throw, @@ -1266,6 +1275,30 @@ export class Brainy implements BrainyInterface { if (existing) return id } + // upsert — by-ID create-or-update. Only meaningful when a custom id is supplied + // (a freshly generated UUID can never collide). When the id already exists, + // delegate to update() so the supplied fields MERGE into the existing entity + // (metadata merge, re-embed on changed data, _rev bump, createdAt preserved) + // instead of the default destructive overwrite. When the id is absent, fall + // through to the normal insert below. + if (params.id && params.upsert) { + const existing = await this.storage.getNounMetadata(id) + if (existing) { + await this.update({ + id, + ...(params.data !== undefined && { data: params.data }), + ...(params.type !== undefined && { type: params.type }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && { visibility: params.visibility }), + ...(params.metadata !== undefined && { metadata: params.metadata }), + ...(params.vector !== undefined && { vector: params.vector }), + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }) + }) + return id + } + } + // Get or compute vector const vector = params.vector || (await this.embed(params.data)) @@ -4244,10 +4277,26 @@ export class Brainy implements BrainyInterface { return results.slice(finalOffset, finalOffset + limit) })() + // includeVectors — opt-in vector hydration. Default (false) keeps the perf + // contract: every result path above builds entities via the metadata-only + // fast path, so `entity.vector` is the empty stub. When requested, fetch the + // stored vectors for the already-paginated page in ONE batch and attach them, + // mirroring get({ includeVectors: true }). Done once here so every find() + // path (metadata-only, vector/text/proximity, graph) honors it uniformly. + if (params.includeVectors && result.length > 0) { + const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id)) + for (const r of result) { + const noun = nounsMap.get(r.id) + if (noun?.vector && r.entity) { + r.entity.vector = noun.vector + } + } + } + // Record performance for auto-tuning const duration = Date.now() - startTime recordQueryPerformance(duration, result.length) - + return result } @@ -4542,12 +4591,22 @@ export class Brainy implements BrainyInterface { const promises = chunk.map(async (item) => { try { - // ifAbsent (7.31.0) — propagate batch-level flag to each item, but let - // per-item flag take precedence so callers can override individual rows. - const itemWithIfAbsent = params.ifAbsent && item.ifAbsent === undefined - ? { ...item, ifAbsent: true } - : item - const id = await this.add(itemWithIfAbsent) + // ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every + // item, but let a per-item flag take precedence so callers can override + // individual rows. The two flags stay independent: per-item upsert handling + // (create-or-merge) and per-item ifAbsent handling (create-or-skip) both + // fire inside the delegated add() call, which also enforces their mutual + // exclusion when a single resolved item carries both. + const resolvedItem = + (params.ifAbsent && item.ifAbsent === undefined) || + (params.upsert && item.upsert === undefined) + ? { + ...item, + ...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }), + ...(params.upsert && item.upsert === undefined && { upsert: true }) + } + : item + const id = await this.add(resolvedItem) result.successful.push(id) } catch (error) { result.failed.push({ @@ -4641,9 +4700,12 @@ export class Brainy implements BrainyInterface { const startTime = Date.now() - // Batch deletes into chunks for 10x faster performance with proper error handling - // Single transaction per chunk (10 entities) = atomic within chunk, graceful failure across chunks - const chunkSize = 10 + // Batch deletes into chunks for 10x faster performance with proper error handling. + // Single transaction per chunk = atomic within chunk, graceful failure across chunks. + // Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's + // maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it. + const storageConfig = this.storage.getBatchConfig() + const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize for (let i = 0; i < idsToDelete.length; i += chunkSize) { const chunk = idsToDelete.slice(i, i + chunkSize) @@ -6330,6 +6392,14 @@ export class Brainy implements BrainyInterface { // natural-key upsert is idempotent against the same key. const { id, originalId } = coerceNewEntityId(params.id) + // ifAbsent and upsert resolve the same id collision in opposite ways + // (skip vs. merge) — mirror of add()'s hard error. + if (params.ifAbsent && params.upsert) { + throw new Error( + 'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.' + ) + } + // ifAbsent — idempotent by-id insert, resolved against batch + store. if (params.id && params.ifAbsent) { const existing = await this.planGetEntity(state, id) @@ -6338,6 +6408,32 @@ export class Brainy implements BrainyInterface { } } + // upsert — by-id create-or-update, resolved against batch + store. When the id + // already exists, delegate to planTxUpdate with a synthesized update op so the + // supplied fields MERGE (mirror of add()'s live upsert path) instead of + // overwriting. The id is already canonical here (coerceNewEntityId above). + if (params.id && params.upsert) { + const existing = await this.planGetEntity(state, id) + if (existing) { + return this.planTxUpdate( + { + op: 'update', + id, + ...(params.data !== undefined && { data: params.data }), + ...(params.type !== undefined && { type: params.type }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && { visibility: params.visibility }), + ...(params.metadata !== undefined && { metadata: params.metadata }), + ...(params.vector !== undefined && { vector: params.vector }), + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }) + } as Extract, { op: 'update' }>, + state, + plan + ) + } + } + const vector = params.vector || (await this.embed(params.data)) if (!this.dimensions) { this.dimensions = vector.length diff --git a/src/db/db.ts b/src/db/db.ts index ac6e4a2e..041e9b95 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -371,12 +371,12 @@ export class Db { entity = await this.host.entityFromRecord( id, { metadata: resolved.metadata, vector: resolved.vector }, - false + params.includeVectors ?? false ) } else if (resolved.source === 'current') { // Defensive: changed ids always resolve to a record or absent, but // a 'current' answer is still served correctly from the live path. - entity = await this.host.get(id) + entity = await this.host.get(id, { includeVectors: params.includeVectors ?? false }) } if (entity && entityMatchesFind(entity as Entity, params as FindParams)) { merged.push(resultFromEntity(entity)) diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 5df12e11..69c04646 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -346,6 +346,24 @@ export interface AddParams { * Ignored when `id` is omitted (a freshly generated UUID can never collide). */ ifAbsent?: boolean + /** + * Create-or-update in a single call. When `true` AND a custom `id` is supplied AND + * an entity with that `id` already exists, `add()` applies the supplied fields as an + * UPDATE instead of the default destructive overwrite: metadata is MERGED (not + * replaced), `data` re-embeds only when it changed, `_rev` is bumped, and the + * original `createdAt` is PRESERVED. When no entity with that `id` exists, `add()` + * inserts normally (a fresh `_rev` of 1). Ignored — behaves as a plain insert — when + * `id` is omitted, since a freshly generated UUID can never collide. + * + * Mutually exclusive with {@link AddParams.ifAbsent}: `ifAbsent` SKIPS an existing + * entity, `upsert` MERGES into it — supplying both throws. + * + * @example + * // First call inserts; a later call with the same id merges + bumps _rev. + * await brain.add({ id: 'customer-42', type: 'customer', data: 'Acme', upsert: true }) + * await brain.add({ id: 'customer-42', type: 'customer', metadata: { tier: 'gold' }, upsert: true }) + */ + upsert?: boolean } /** @@ -538,6 +556,14 @@ export interface FindParams { includeRelations?: boolean // Include entity relationships excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included) service?: string // Multi-tenancy filter + /** + * Return each result's stored embedding under `entity.vector`. Defaults to `false`, + * in which case `entity.vector` is an empty array (the perf default — vectors are + * large and most reads don't need them). Set `true` to get the persisted vector + * back from `find()` without a recompute, e.g. to feed downstream similarity math. + * Mirrors {@link GetOptions.includeVectors} on `get()`. + */ + includeVectors?: boolean // Hybrid search options searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid @@ -738,6 +764,14 @@ export interface AddManyParams { * on each item individually. Item-level `ifAbsent` overrides the batch flag. */ ifAbsent?: boolean + /** + * Create-or-update applied to every item. Equivalent to setting `upsert: true` on + * each item individually (see {@link AddParams.upsert}): an item whose custom `id` + * already exists is MERGED as an update rather than overwritten. Item-level `upsert` + * overrides the batch flag. Mutually exclusive with {@link AddManyParams.ifAbsent} + * at the item level — an item resolving to both throws. + */ + upsert?: boolean } /** @@ -759,6 +793,14 @@ export interface RemoveManyParams { type?: NounType // Remove all of type where?: any // Remove by metadata limit?: number // Max to remove (safety) + /** + * Entities removed per transaction chunk. Each chunk is one atomic transaction — + * a chunk commits or rolls back together, and failures are isolated to their chunk. + * Defaults to the storage adapter's `maxBatchSize` (memory: 1000, S3/R2: 100, + * GCS: 50), so zero-config removals match the adapter's characteristics. Override + * only to tune throughput vs. transaction granularity. + */ + chunkSize?: number onProgress?: (done: number, total: number) => void continueOnError?: boolean // Continue processing if a removal fails } diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index 9bbef3d7..7af57ab4 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -180,6 +180,49 @@ describe('Brainy Batch Operations', () => { } }) + it('removes more than the legacy 10-item chunk in a single call (adaptive chunking)', async () => { + // Seed 25 entities — more than the old hardcoded chunk of 10 — to prove the + // storage-adaptive chunk size processes every chunk and nothing is capped. + const seed = await brain.addMany({ + items: Array.from({ length: 25 }, (_, i) => ({ + data: `Adaptive Chunk ${i}`, + type: NounType.Thing, + metadata: { batch: 'adaptive' } + })) + }) + expect(seed.successful).toHaveLength(25) + + const result = await brain.removeMany({ ids: seed.successful }) + expect(result.successful).toHaveLength(25) + expect(result.failed).toHaveLength(0) + expect(result.total).toBe(25) + + // Every entity is actually gone. + for (const id of seed.successful) { + expect(await brain.get(id)).toBeNull() + } + }) + + it('removes all entities across multiple chunks when chunkSize is overridden', async () => { + const seed = await brain.addMany({ + items: Array.from({ length: 12 }, (_, i) => ({ + data: `Override Chunk ${i}`, + type: NounType.Thing, + metadata: { batch: 'override' } + })) + }) + expect(seed.successful).toHaveLength(12) + + // chunkSize 5 → 3 chunks (5 + 5 + 2); all must be removed. + const result = await brain.removeMany({ ids: seed.successful, chunkSize: 5 }) + expect(result.successful).toHaveLength(12) + expect(result.failed).toHaveLength(0) + + for (const id of seed.successful) { + expect(await brain.get(id)).toBeNull() + } + }) + it('should handle selective deletion', async () => { // Delete only some const toDelete = [testIds[0], testIds[2], testIds[4]] diff --git a/tests/unit/brainy/find-include-vectors.test.ts b/tests/unit/brainy/find-include-vectors.test.ts new file mode 100644 index 00000000..38216692 --- /dev/null +++ b/tests/unit/brainy/find-include-vectors.test.ts @@ -0,0 +1,94 @@ +/** + * Unit tests for FindParams.includeVectors — opt-in vector hydration on find(). + * + * Verifies that find({ includeVectors: true }) returns each result's stored + * embedding under entity.vector (matching get({ includeVectors: true })), that + * the default keeps the perf contract of an empty vector, and that the + * metadata-only find({ where }) path honors the flag too. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { createTestConfig } from '../../helpers/test-factory' + +describe('FindParams.includeVectors', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('returns the stored vector for query-based find when includeVectors is true', async () => { + const id = await brain.add({ + type: 'document', + data: 'machine learning and neural networks', + metadata: { topic: 'ai' } + }) + + // Stored vector, as get() surfaces it — the reference for the find() result. + const stored = await brain.get(id, { includeVectors: true }) + expect(stored!.vector.length).toBeGreaterThan(0) + + const results = await brain.find({ + query: 'machine learning and neural networks', + includeVectors: true, + limit: 10 + }) + + const hit = results.find((r) => r.entity.id === id) + expect(hit).toBeDefined() + expect(hit!.entity.vector.length).toBeGreaterThan(0) + expect(hit!.entity.vector).toEqual(stored!.vector) + }) + + it('returns an empty vector by default (perf contract preserved)', async () => { + const id = await brain.add({ + type: 'document', + data: 'default path should not hydrate vectors', + metadata: { topic: 'perf' } + }) + + const results = await brain.find({ + query: 'default path should not hydrate vectors', + limit: 10 + }) + + const hit = results.find((r) => r.entity.id === id) + expect(hit).toBeDefined() + expect(hit!.entity.vector.length).toBe(0) + }) + + it('honors includeVectors on the metadata-only where path', async () => { + const id = await brain.add({ + type: 'thing', + data: 'metadata only filter target', + metadata: { category: 'filtered' } + }) + + const stored = await brain.get(id, { includeVectors: true }) + + const withVectors = await brain.find({ + where: { category: 'filtered' }, + includeVectors: true, + limit: 10 + }) + const hit = withVectors.find((r) => r.entity.id === id) + expect(hit).toBeDefined() + expect(hit!.entity.vector.length).toBeGreaterThan(0) + expect(hit!.entity.vector).toEqual(stored!.vector) + + // And the same filter without the flag stays empty. + const withoutVectors = await brain.find({ + where: { category: 'filtered' }, + limit: 10 + }) + const plainHit = withoutVectors.find((r) => r.entity.id === id) + expect(plainHit).toBeDefined() + expect(plainHit!.entity.vector.length).toBe(0) + }) +}) diff --git a/tests/unit/brainy/upsert.test.ts b/tests/unit/brainy/upsert.test.ts new file mode 100644 index 00000000..92e56b05 --- /dev/null +++ b/tests/unit/brainy/upsert.test.ts @@ -0,0 +1,194 @@ +/** + * Unit tests for AddParams.upsert — create-or-update in a single add() call. + * + * Covers the live add() path and the addMany batch flag. Verifies that upsert + * inserts on a fresh id, merges (not overwrites) into an existing id while + * preserving createdAt and bumping _rev, is mutually exclusive with ifAbsent, + * and is a plain insert when no id is supplied. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { createTestConfig } from '../../helpers/test-factory' + +describe('AddParams.upsert', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('inserts a new entity when the id does not exist (_rev 1)', async () => { + // 8.0 normalizes a natural string key to a stable canonical UUID; the entity is + // still reachable by the natural key via get()'s id resolution. + const id = await brain.add({ + id: 'customer-1', + type: 'thing', + data: 'Acme Corp', + metadata: { tier: 'silver' }, + upsert: true + }) + + expect(typeof id).toBe('string') + + const entity = await brain.get('customer-1') + expect(entity).not.toBeNull() + expect(entity!.type).toBe('thing') + expect(entity!.metadata.tier).toBe('silver') + expect(entity!._rev).toBe(1) + }) + + it('merges into an existing entity, preserves createdAt, bumps _rev, applies changed data', async () => { + // Seed the entity. + await brain.add({ + id: 'invoice-9', + type: 'thing', + data: 'Original line items', + metadata: { status: 'draft', amount: 100 } + }) + + const before = await brain.get('invoice-9') + expect(before).not.toBeNull() + const originalCreatedAt = before!.createdAt + expect(before!._rev).toBe(1) + + // Upsert over the existing id with a partial metadata patch + new data. + await brain.add({ + id: 'invoice-9', + type: 'thing', + data: 'Revised line items', + metadata: { status: 'sent' }, + upsert: true + }) + + const after = await brain.get('invoice-9') + expect(after).not.toBeNull() + + // Metadata MERGED, not replaced — the untouched 'amount' field survives. + expect(after!.metadata.status).toBe('sent') + expect(after!.metadata.amount).toBe(100) + + // Changed data applied. + expect(after!.data).toBe('Revised line items') + + // _rev bumped, createdAt preserved. + expect(after!._rev).toBe(2) + expect(after!.createdAt).toBe(originalCreatedAt) + }) + + it('re-embeds when data changes on upsert', async () => { + await brain.add({ + id: 'doc-vec', + type: 'document', + data: 'cats and dogs and small furry animals' + }) + const before = await brain.get('doc-vec', { includeVectors: true }) + expect(before!.vector.length).toBeGreaterThan(0) + + await brain.add({ + id: 'doc-vec', + type: 'document', + data: 'quarterly financial revenue projections spreadsheet', + upsert: true + }) + const after = await brain.get('doc-vec', { includeVectors: true }) + expect(after!.vector.length).toBe(before!.vector.length) + // A genuinely different document re-embeds to a different vector. + expect(after!.vector).not.toEqual(before!.vector) + }) + + it('throws when both ifAbsent and upsert are true', async () => { + await expect( + brain.add({ + id: 'conflict-1', + type: 'thing', + data: 'x', + ifAbsent: true, + upsert: true + }) + ).rejects.toThrow(/mutually exclusive/) + }) + + it('behaves like a plain insert when no id is supplied', async () => { + const id = await brain.add({ + type: 'thing', + data: 'no id supplied', + metadata: { k: 'v' }, + upsert: true + }) + + expect(typeof id).toBe('string') + const entity = await brain.get(id) + expect(entity).not.toBeNull() + expect(entity!._rev).toBe(1) + expect(entity!.metadata.k).toBe('v') + }) + + it('merges an existing item via the addMany batch upsert flag', async () => { + // Seed an item that the batch will later upsert into. + await brain.add({ + id: 'order-1', + type: 'thing', + data: 'first', + metadata: { region: 'west', count: 1 } + }) + const seeded = await brain.get('order-1') + const seededCreatedAt = seeded!.createdAt + + const result = await brain.addMany({ + upsert: true, + items: [ + { id: 'order-1', type: 'thing', data: 'updated', metadata: { count: 2 } }, + { id: 'order-2', type: 'thing', data: 'brand new', metadata: { region: 'east' } } + ] + }) + + // Both items resolved (returned ids are the canonical UUIDs the natural keys map to). + expect(result.failed).toHaveLength(0) + expect(result.successful).toHaveLength(2) + + // Existing item merged (region survived), _rev bumped, createdAt preserved. + const merged = await brain.get('order-1') + expect(merged!.metadata.region).toBe('west') + expect(merged!.metadata.count).toBe(2) + expect(merged!.data).toBe('updated') + expect(merged!._rev).toBe(2) + expect(merged!.createdAt).toBe(seededCreatedAt) + + // New item inserted fresh. + const fresh = await brain.get('order-2') + expect(fresh!._rev).toBe(1) + expect(fresh!.metadata.region).toBe('east') + }) + + it('upserts inside a transact batch — existing id merges, new id inserts', async () => { + await brain.add({ + id: 'acct-1', + type: 'thing', + data: 'seed', + metadata: { plan: 'free', seats: 1 } + }) + const seeded = await brain.get('acct-1') + const seededCreatedAt = seeded!.createdAt + + await brain.transact([ + { op: 'add', id: 'acct-1', type: 'thing', data: 'seed', metadata: { plan: 'pro' }, upsert: true }, + { op: 'add', id: 'acct-2', type: 'thing', data: 'new account', metadata: { plan: 'free' }, upsert: true } + ]) + + const merged = await brain.get('acct-1') + expect(merged!.metadata.plan).toBe('pro') + expect(merged!.metadata.seats).toBe(1) // merged, not overwritten + expect(merged!._rev).toBe(2) + expect(merged!.createdAt).toBe(seededCreatedAt) + + const fresh = await brain.get('acct-2') + expect(fresh!._rev).toBe(1) + expect(fresh!.metadata.plan).toBe('free') + }) +})