feat(8.0): upsert + FindParams.includeVectors + removeMany adaptive chunking

Three additive ergonomics from the API-simplification audit (no behavior change
to existing call sites):

- AddParams.upsert: create-or-update in one call. With a custom id, an existing
  entity is MERGED via the update path (merges metadata, re-embeds changed data,
  bumps _rev, PRESERVES createdAt) instead of the destructive full overwrite a
  plain add() does. Mutually exclusive with ifAbsent (throws if both set);
  ignored when no id is supplied. Wired into add(), addMany (per-item flag
  propagation), and the transact add op (routes to planTxUpdate). Kills the
  get()-then-add() round-trip for idempotent writes.
- FindParams.includeVectors: mirror of GetOptions.includeVectors — find() returns
  stored vectors when set; default stays empty (the perf contract is preserved).
  Honored on both the query and metadata-only where paths, and in db.find().
- removeMany adaptive chunking: replaced the hardcoded chunkSize=10 with
  params.chunkSize ?? storageConfig.maxBatchSize, matching addMany/relateMany —
  one storage-adaptive batch policy across all *Many methods (no thrash on
  high-latency backends).

Tests: tests/unit/brainy/upsert.test.ts (insert/merge/createdAt-preserved/
re-embed/ifAbsent-conflict/no-id/addMany/transact), find-include-vectors.test.ts
(true/default/where-path), batch-operations.test.ts (removeMany >10 items).

tests/unit/brainy/batch-operations.test.ts

@@ -180,6 +180,49 @@ describe('Brainy Batch Operations', () => {
       }
     })
     
+    it('removes more than the legacy 10-item chunk in a single call (adaptive chunking)', async () => {
+      // Seed 25 entities — more than the old hardcoded chunk of 10 — to prove the
+      // storage-adaptive chunk size processes every chunk and nothing is capped.
+      const seed = await brain.addMany({
+        items: Array.from({ length: 25 }, (_, i) => ({
+          data: `Adaptive Chunk ${i}`,
+          type: NounType.Thing,
+          metadata: { batch: 'adaptive' }
+        }))
+      })
+      expect(seed.successful).toHaveLength(25)
+
+      const result = await brain.removeMany({ ids: seed.successful })
+      expect(result.successful).toHaveLength(25)
+      expect(result.failed).toHaveLength(0)
+      expect(result.total).toBe(25)
+
+      // Every entity is actually gone.
+      for (const id of seed.successful) {
+        expect(await brain.get(id)).toBeNull()
+      }
+    })
+
+    it('removes all entities across multiple chunks when chunkSize is overridden', async () => {
+      const seed = await brain.addMany({
+        items: Array.from({ length: 12 }, (_, i) => ({
+          data: `Override Chunk ${i}`,
+          type: NounType.Thing,
+          metadata: { batch: 'override' }
+        }))
+      })
+      expect(seed.successful).toHaveLength(12)
+
+      // chunkSize 5 → 3 chunks (5 + 5 + 2); all must be removed.
+      const result = await brain.removeMany({ ids: seed.successful, chunkSize: 5 })
+      expect(result.successful).toHaveLength(12)
+      expect(result.failed).toHaveLength(0)
+
+      for (const id of seed.successful) {
+        expect(await brain.get(id)).toBeNull()
+      }
+    })
+
     it('should handle selective deletion', async () => {
       // Delete only some
       const toDelete = [testIds[0], testIds[2], testIds[4]]

tests/unit/brainy/find-include-vectors.test.ts

@@ -0,0 +1,94 @@
+/**
+ * Unit tests for FindParams.includeVectors — opt-in vector hydration on find().
+ *
+ * Verifies that find({ includeVectors: true }) returns each result's stored
+ * embedding under entity.vector (matching get({ includeVectors: true })), that
+ * the default keeps the perf contract of an empty vector, and that the
+ * metadata-only find({ where }) path honors the flag too.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../../src/brainy'
+import { createTestConfig } from '../../helpers/test-factory'
+
+describe('FindParams.includeVectors', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('returns the stored vector for query-based find when includeVectors is true', async () => {
+    const id = await brain.add({
+      type: 'document',
+      data: 'machine learning and neural networks',
+      metadata: { topic: 'ai' }
+    })
+
+    // Stored vector, as get() surfaces it — the reference for the find() result.
+    const stored = await brain.get(id, { includeVectors: true })
+    expect(stored!.vector.length).toBeGreaterThan(0)
+
+    const results = await brain.find({
+      query: 'machine learning and neural networks',
+      includeVectors: true,
+      limit: 10
+    })
+
+    const hit = results.find((r) => r.entity.id === id)
+    expect(hit).toBeDefined()
+    expect(hit!.entity.vector.length).toBeGreaterThan(0)
+    expect(hit!.entity.vector).toEqual(stored!.vector)
+  })
+
+  it('returns an empty vector by default (perf contract preserved)', async () => {
+    const id = await brain.add({
+      type: 'document',
+      data: 'default path should not hydrate vectors',
+      metadata: { topic: 'perf' }
+    })
+
+    const results = await brain.find({
+      query: 'default path should not hydrate vectors',
+      limit: 10
+    })
+
+    const hit = results.find((r) => r.entity.id === id)
+    expect(hit).toBeDefined()
+    expect(hit!.entity.vector.length).toBe(0)
+  })
+
+  it('honors includeVectors on the metadata-only where path', async () => {
+    const id = await brain.add({
+      type: 'thing',
+      data: 'metadata only filter target',
+      metadata: { category: 'filtered' }
+    })
+
+    const stored = await brain.get(id, { includeVectors: true })
+
+    const withVectors = await brain.find({
+      where: { category: 'filtered' },
+      includeVectors: true,
+      limit: 10
+    })
+    const hit = withVectors.find((r) => r.entity.id === id)
+    expect(hit).toBeDefined()
+    expect(hit!.entity.vector.length).toBeGreaterThan(0)
+    expect(hit!.entity.vector).toEqual(stored!.vector)
+
+    // And the same filter without the flag stays empty.
+    const withoutVectors = await brain.find({
+      where: { category: 'filtered' },
+      limit: 10
+    })
+    const plainHit = withoutVectors.find((r) => r.entity.id === id)
+    expect(plainHit).toBeDefined()
+    expect(plainHit!.entity.vector.length).toBe(0)
+  })
+})

tests/unit/brainy/upsert.test.ts

@@ -0,0 +1,194 @@
+/**
+ * Unit tests for AddParams.upsert — create-or-update in a single add() call.
+ *
+ * Covers the live add() path and the addMany batch flag. Verifies that upsert
+ * inserts on a fresh id, merges (not overwrites) into an existing id while
+ * preserving createdAt and bumping _rev, is mutually exclusive with ifAbsent,
+ * and is a plain insert when no id is supplied.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../../src/brainy'
+import { createTestConfig } from '../../helpers/test-factory'
+
+describe('AddParams.upsert', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('inserts a new entity when the id does not exist (_rev 1)', async () => {
+    // 8.0 normalizes a natural string key to a stable canonical UUID; the entity is
+    // still reachable by the natural key via get()'s id resolution.
+    const id = await brain.add({
+      id: 'customer-1',
+      type: 'thing',
+      data: 'Acme Corp',
+      metadata: { tier: 'silver' },
+      upsert: true
+    })
+
+    expect(typeof id).toBe('string')
+
+    const entity = await brain.get('customer-1')
+    expect(entity).not.toBeNull()
+    expect(entity!.type).toBe('thing')
+    expect(entity!.metadata.tier).toBe('silver')
+    expect(entity!._rev).toBe(1)
+  })
+
+  it('merges into an existing entity, preserves createdAt, bumps _rev, applies changed data', async () => {
+    // Seed the entity.
+    await brain.add({
+      id: 'invoice-9',
+      type: 'thing',
+      data: 'Original line items',
+      metadata: { status: 'draft', amount: 100 }
+    })
+
+    const before = await brain.get('invoice-9')
+    expect(before).not.toBeNull()
+    const originalCreatedAt = before!.createdAt
+    expect(before!._rev).toBe(1)
+
+    // Upsert over the existing id with a partial metadata patch + new data.
+    await brain.add({
+      id: 'invoice-9',
+      type: 'thing',
+      data: 'Revised line items',
+      metadata: { status: 'sent' },
+      upsert: true
+    })
+
+    const after = await brain.get('invoice-9')
+    expect(after).not.toBeNull()
+
+    // Metadata MERGED, not replaced — the untouched 'amount' field survives.
+    expect(after!.metadata.status).toBe('sent')
+    expect(after!.metadata.amount).toBe(100)
+
+    // Changed data applied.
+    expect(after!.data).toBe('Revised line items')
+
+    // _rev bumped, createdAt preserved.
+    expect(after!._rev).toBe(2)
+    expect(after!.createdAt).toBe(originalCreatedAt)
+  })
+
+  it('re-embeds when data changes on upsert', async () => {
+    await brain.add({
+      id: 'doc-vec',
+      type: 'document',
+      data: 'cats and dogs and small furry animals'
+    })
+    const before = await brain.get('doc-vec', { includeVectors: true })
+    expect(before!.vector.length).toBeGreaterThan(0)
+
+    await brain.add({
+      id: 'doc-vec',
+      type: 'document',
+      data: 'quarterly financial revenue projections spreadsheet',
+      upsert: true
+    })
+    const after = await brain.get('doc-vec', { includeVectors: true })
+    expect(after!.vector.length).toBe(before!.vector.length)
+    // A genuinely different document re-embeds to a different vector.
+    expect(after!.vector).not.toEqual(before!.vector)
+  })
+
+  it('throws when both ifAbsent and upsert are true', async () => {
+    await expect(
+      brain.add({
+        id: 'conflict-1',
+        type: 'thing',
+        data: 'x',
+        ifAbsent: true,
+        upsert: true
+      })
+    ).rejects.toThrow(/mutually exclusive/)
+  })
+
+  it('behaves like a plain insert when no id is supplied', async () => {
+    const id = await brain.add({
+      type: 'thing',
+      data: 'no id supplied',
+      metadata: { k: 'v' },
+      upsert: true
+    })
+
+    expect(typeof id).toBe('string')
+    const entity = await brain.get(id)
+    expect(entity).not.toBeNull()
+    expect(entity!._rev).toBe(1)
+    expect(entity!.metadata.k).toBe('v')
+  })
+
+  it('merges an existing item via the addMany batch upsert flag', async () => {
+    // Seed an item that the batch will later upsert into.
+    await brain.add({
+      id: 'order-1',
+      type: 'thing',
+      data: 'first',
+      metadata: { region: 'west', count: 1 }
+    })
+    const seeded = await brain.get('order-1')
+    const seededCreatedAt = seeded!.createdAt
+
+    const result = await brain.addMany({
+      upsert: true,
+      items: [
+        { id: 'order-1', type: 'thing', data: 'updated', metadata: { count: 2 } },
+        { id: 'order-2', type: 'thing', data: 'brand new', metadata: { region: 'east' } }
+      ]
+    })
+
+    // Both items resolved (returned ids are the canonical UUIDs the natural keys map to).
+    expect(result.failed).toHaveLength(0)
+    expect(result.successful).toHaveLength(2)
+
+    // Existing item merged (region survived), _rev bumped, createdAt preserved.
+    const merged = await brain.get('order-1')
+    expect(merged!.metadata.region).toBe('west')
+    expect(merged!.metadata.count).toBe(2)
+    expect(merged!.data).toBe('updated')
+    expect(merged!._rev).toBe(2)
+    expect(merged!.createdAt).toBe(seededCreatedAt)
+
+    // New item inserted fresh.
+    const fresh = await brain.get('order-2')
+    expect(fresh!._rev).toBe(1)
+    expect(fresh!.metadata.region).toBe('east')
+  })
+
+  it('upserts inside a transact batch — existing id merges, new id inserts', async () => {
+    await brain.add({
+      id: 'acct-1',
+      type: 'thing',
+      data: 'seed',
+      metadata: { plan: 'free', seats: 1 }
+    })
+    const seeded = await brain.get('acct-1')
+    const seededCreatedAt = seeded!.createdAt
+
+    await brain.transact([
+      { op: 'add', id: 'acct-1', type: 'thing', data: 'seed', metadata: { plan: 'pro' }, upsert: true },
+      { op: 'add', id: 'acct-2', type: 'thing', data: 'new account', metadata: { plan: 'free' }, upsert: true }
+    ])
+
+    const merged = await brain.get('acct-1')
+    expect(merged!.metadata.plan).toBe('pro')
+    expect(merged!.metadata.seats).toBe(1) // merged, not overwritten
+    expect(merged!._rev).toBe(2)
+    expect(merged!.createdAt).toBe(seededCreatedAt)
+
+    const fresh = await brain.get('acct-2')
+    expect(fresh!._rev).toBe(1)
+    expect(fresh!.metadata.plan).toBe('free')
+  })
+})