David Snelling
e5997a1516
Cleared ~60 rotted-test failures across 17 integration files: get() now passes
{includeVectors:true} where the vector is used; close() teardown added (cures
heartbeat-bleed timeouts); removed-API call-sites rewritten to the 8.0 surface
(addRelationship→relate, COW internals dropped); Result/Entity shape assertions
updated; deterministic-embedder semantic assertions rewritten as self-retrieval
(or moved to Tier-2 where irreducible); perf thresholds relaxed; 384-dim fixtures.
Adds the Tier-2 (real-model) scaffolding: tests/setup-semantic.ts +
tests/configs/vitest.semantic.config.ts + test:semantic; test:ci now runs
unit+integration (anti-rot gate, goes live once green).
Remaining 17 failures are REAL 8.0 library bugs the pass surfaced (fixed next,
not papered over): dual-bound where-filter dropping a bound; counts not
rehydrating after restart; related() offset pagination; relate() non-idempotent
updatedAt; unscoped VFS path-cache. Plus find-unified finish + a few stragglers.
518 lines
21 KiB
TypeScript
518 lines
21 KiB
TypeScript
/**
|
|
* COMPREHENSIVE Integration Tests for Brainy 8.0 (Tier 1)
|
|
*
|
|
* Exercises the full public surface end-to-end against real code paths:
|
|
* - find() vector / self-retrieval search
|
|
* - find() metadata-only filtering (where + range operators)
|
|
* - Triple Intelligence (getTripleIntelligence().find) — semantic + metadata + ranges
|
|
* - getStats() entity/relationship statistics
|
|
* - embed() vector generation
|
|
* - add() / get() round-trips and consistency
|
|
*
|
|
* Runs under the DETERMINISTIC embedder (tests/setup-integration.ts): embedding
|
|
* vectors are content-derived and stable, so self-identity cosine = 1.0 and two
|
|
* DIFFERENT texts sit at ~0.23 similarity. Cross-text SEMANTIC RELEVANCE is NOT
|
|
* meaningful here (that is covered by the Tier-2 real-model semantic suite); the
|
|
* reliable signal is SELF-RETRIEVAL — querying a thing by its OWN text returns it
|
|
* at the top. Assertions below are structural (counts, ids, shapes, metadata
|
|
* filters, self-retrieval, stats), which is exactly what Tier 1 validates.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
|
import { Brainy } from '../../src/brainy'
|
|
import { requiresMemory } from '../setup-integration.js'
|
|
|
|
describe('Brainy 8.0 Complete Feature Test (Tier 1, deterministic embedder)', () => {
|
|
let brain: Brainy
|
|
|
|
beforeAll(async () => {
|
|
requiresMemory(16)
|
|
|
|
// Full feature set; requireSubtype:false so plain add({type}) is allowed.
|
|
brain = new Brainy({
|
|
requireSubtype: false,
|
|
storage: { type: 'memory' }
|
|
})
|
|
|
|
await brain.init()
|
|
|
|
// Start with clean state
|
|
await brain.clear()
|
|
}, 300000)
|
|
|
|
afterAll(async () => {
|
|
if (brain) {
|
|
try {
|
|
await brain.clear()
|
|
await brain.close()
|
|
} catch (error) {
|
|
console.warn('Cleanup warning:', error)
|
|
}
|
|
}
|
|
|
|
if (global.gc) {
|
|
global.gc()
|
|
}
|
|
}, 60000)
|
|
|
|
describe('1. Core find() with vector search', () => {
|
|
const searchItems = [
|
|
'JavaScript is a programming language for web development',
|
|
'Python is excellent for machine learning and AI applications',
|
|
'React is a popular frontend framework for building user interfaces',
|
|
'Vue.js provides reactive data binding for modern web apps',
|
|
'Node.js enables server-side JavaScript development',
|
|
'TensorFlow is used for deep learning and neural networks',
|
|
'Docker containerizes applications for consistent deployment',
|
|
'Kubernetes orchestrates containerized applications at scale',
|
|
'PostgreSQL is a powerful relational database system',
|
|
'MongoDB stores documents in a flexible NoSQL format'
|
|
]
|
|
|
|
beforeAll(async () => {
|
|
for (const item of searchItems) {
|
|
await brain.add({ data: item, type: 'thing' })
|
|
}
|
|
})
|
|
|
|
it('should self-retrieve a document by its own text (deterministic embedder)', async () => {
|
|
// Self-retrieval: querying with an entity's OWN data text returns that entity
|
|
// at the top. We use searchMode:'semantic' to exercise the pure vector path
|
|
// (cosine); the default 'auto' mode is hybrid RRF, whose fused score is a
|
|
// small rank-derived value rather than the raw cosine, so it cannot assert
|
|
// the self-identity cosine = 1.0 property.
|
|
const target = 'React is a popular frontend framework for building user interfaces'
|
|
const results = await brain.find({ query: target, limit: 5, searchMode: 'semantic' })
|
|
|
|
expect(results).toHaveLength(5)
|
|
// Top hit is the exact item we queried for.
|
|
expect(results[0].entity.data).toBe(target)
|
|
// Self-identity cosine = 1.0 for the deterministic embedder.
|
|
expect(results[0].score).toBeGreaterThan(0.99)
|
|
|
|
// A second self-retrieval lands on its own item too.
|
|
const dbTarget = 'PostgreSQL is a powerful relational database system'
|
|
const dbResults = await brain.find({ query: dbTarget, limit: 3, searchMode: 'semantic' })
|
|
expect(dbResults).toHaveLength(3)
|
|
expect(dbResults[0].entity.data).toBe(dbTarget)
|
|
expect(dbResults[0].score).toBeGreaterThan(0.99)
|
|
})
|
|
|
|
it('should return well-formed results with proper scoring and structure', async () => {
|
|
const results = await brain.find({ query: searchItems[0], limit: 5 })
|
|
expect(results).toHaveLength(5)
|
|
|
|
// Every result has the documented Result shape.
|
|
results.forEach(result => {
|
|
expect(result).toHaveProperty('id')
|
|
expect(result).toHaveProperty('score')
|
|
expect(result).toHaveProperty('entity')
|
|
expect(typeof result.score).toBe('number')
|
|
})
|
|
|
|
// Scores are sorted descending.
|
|
for (let i = 0; i < results.length - 1; i++) {
|
|
expect(results[i].score).toBeGreaterThanOrEqual(results[i + 1].score)
|
|
}
|
|
})
|
|
|
|
it('should handle find() edge cases correctly', async () => {
|
|
// A query with no vector criteria still returns the documented array shape.
|
|
const limited = await brain.find({ query: searchItems[8], limit: 2 })
|
|
expect(limited).toHaveLength(2)
|
|
|
|
// Score ordering verification on a self-retrieval query.
|
|
const ordered = await brain.find({ query: searchItems[2], limit: 5 })
|
|
for (let i = 0; i < ordered.length - 1; i++) {
|
|
expect(ordered[i].score).toBeGreaterThanOrEqual(ordered[i + 1].score)
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('2. find() query-object form returns well-formed Result[]', () => {
|
|
it('should return well-formed Result[] for query-object searches', async () => {
|
|
// The query-OBJECT form ({ query }) is the deterministic, Tier-1 surface:
|
|
// it embeds the query string and runs vector/hybrid search directly.
|
|
const queries = [
|
|
'frontend frameworks',
|
|
'database technologies',
|
|
'programming languages',
|
|
'containerization and deployment'
|
|
]
|
|
|
|
for (const query of queries) {
|
|
const results = await brain.find({ query })
|
|
|
|
expect(results).toBeInstanceOf(Array)
|
|
expect(results.length).toBeGreaterThan(0)
|
|
|
|
results.forEach(result => {
|
|
expect(result).toHaveProperty('id')
|
|
expect(result).toHaveProperty('score')
|
|
expect(result).toHaveProperty('entity')
|
|
expect(typeof result.score).toBe('number')
|
|
})
|
|
}
|
|
})
|
|
|
|
it('should honor the limit argument on query-object find()', async () => {
|
|
const queries = [
|
|
'frameworks for building websites',
|
|
'tools for data analysis',
|
|
'languages for machine learning',
|
|
'databases for storing information'
|
|
]
|
|
|
|
for (const query of queries) {
|
|
const results = await brain.find({ query, limit: 3 })
|
|
expect(results).toHaveLength(3)
|
|
results.forEach(result => {
|
|
expect(typeof result.score).toBe('number')
|
|
})
|
|
}
|
|
})
|
|
|
|
// TIER2: semantic relevance (real-model suite)
|
|
//
|
|
// The natural-language STRING form `find('show me frontend frameworks')` runs
|
|
// the query through the NLP pattern library (220 embedded patterns) to infer
|
|
// intent (filters, graph traversal, semantic terms). That inference depends on
|
|
// the REAL embedding model — under the deterministic embedder the pattern
|
|
// vectors are meaningless, so the parser produces non-semantic results. Asserting
|
|
// that an imperative NL phrase returns the relevant entities is a real-model
|
|
// concern; the deterministic Tier-1 surface for the same intent is the
|
|
// query-object form exercised above.
|
|
it.skip('should understand imperative natural-language string queries', async () => {
|
|
const results = await brain.find('show me frontend frameworks')
|
|
expect(results.length).toBeGreaterThan(0)
|
|
expect(results.some(r => String(r.entity.data).toLowerCase().includes('react'))).toBe(true)
|
|
})
|
|
})
|
|
|
|
describe('3. Triple Intelligence: semantic + metadata + range', () => {
|
|
// NOTE: domain attributes live under non-reserved metadata keys. `type` is a
|
|
// RESERVED entity field (it classifies the noun), so the framework's
|
|
// frontend/backend kind is stored under `category` to remain queryable via
|
|
// `where`.
|
|
const frameworks = [
|
|
{ name: 'React', category: 'frontend', year: 2013, popularity: 95, language: 'JavaScript' },
|
|
{ name: 'Vue.js', category: 'frontend', year: 2014, popularity: 85, language: 'JavaScript' },
|
|
{ name: 'Angular', category: 'frontend', year: 2010, popularity: 75, language: 'TypeScript' },
|
|
{ name: 'Django', category: 'backend', year: 2005, popularity: 80, language: 'Python' },
|
|
{ name: 'FastAPI', category: 'backend', year: 2018, popularity: 70, language: 'Python' },
|
|
{ name: 'Express', category: 'backend', year: 2010, popularity: 90, language: 'JavaScript' }
|
|
]
|
|
|
|
beforeAll(async () => {
|
|
for (const fw of frameworks) {
|
|
await brain.add({
|
|
data: `${fw.name} framework for ${fw.category} development`,
|
|
type: 'thing',
|
|
metadata: fw
|
|
})
|
|
}
|
|
})
|
|
|
|
it('should combine semantic search with metadata + single-bound range filters (hard AND)', async () => {
|
|
// find({ query, where }) resolves `where` to candidate IDs FIRST and runs the
|
|
// vector search over only those candidates — so `where` is a hard AND filter
|
|
// (true intersection), which is the 8.0 surface for "semantic + structured"
|
|
// queries. searchMode:'semantic' keeps the score on the cosine path.
|
|
const results = await brain.find({
|
|
query: 'modern web development framework',
|
|
where: {
|
|
category: 'frontend',
|
|
popularity: { greaterThan: 80 },
|
|
year: { greaterThan: 2012 }
|
|
},
|
|
limit: 5,
|
|
searchMode: 'semantic'
|
|
})
|
|
|
|
expect(results.length).toBeGreaterThan(0)
|
|
expect(results.length).toBeLessThanOrEqual(5)
|
|
|
|
// Every result must satisfy ALL metadata filters. With these fixtures only
|
|
// React (95/2013) and Vue.js (85/2014) qualify.
|
|
results.forEach(result => {
|
|
expect(result.entity.metadata?.category).toBe('frontend')
|
|
expect(result.entity.metadata?.popularity).toBeGreaterThan(80)
|
|
expect(result.entity.metadata?.year).toBeGreaterThan(2012)
|
|
})
|
|
|
|
const names = results.map(r => r.entity.metadata?.name).sort()
|
|
expect(names).toEqual(['React', 'Vue.js'])
|
|
})
|
|
|
|
it('should apply dual-bound range filters (greaterThan + lessThan on one field)', async () => {
|
|
// year ∈ (2009, 2020) AND popularity ∈ (75, 95):
|
|
// React 2013/95 → 95 not < 95 → out
|
|
// Vue.js 2014/85 → in
|
|
// Angular 2010/75 → 75 not > 75 → out
|
|
// Django 2005/80 → 2005 not > 2009 → out
|
|
// FastAPI 2018/70 → 70 not > 75 → out
|
|
// Express 2010/90 → in
|
|
const results = await brain.find({
|
|
where: {
|
|
year: { greaterThan: 2009, lessThan: 2020 },
|
|
popularity: { greaterThan: 75, lessThan: 95 }
|
|
},
|
|
limit: 10
|
|
})
|
|
|
|
expect(results).toBeInstanceOf(Array)
|
|
results.forEach(result => {
|
|
expect(result.entity.metadata?.year).toBeGreaterThan(2009)
|
|
expect(result.entity.metadata?.year).toBeLessThan(2020)
|
|
expect(result.entity.metadata?.popularity).toBeGreaterThan(75)
|
|
expect(result.entity.metadata?.popularity).toBeLessThan(95)
|
|
})
|
|
|
|
const names = results.map(r => r.entity.metadata?.name).sort()
|
|
expect(names).toEqual(['Express', 'Vue.js'])
|
|
})
|
|
|
|
it('should expose Triple Intelligence RRF fusion (union ranking) shape', async () => {
|
|
// getTripleIntelligence().find() is a soft RRF-FUSION ranker: it UNIONs the
|
|
// vector ("like") and field ("where") signal sets and ranks by reciprocal
|
|
// rank — it is NOT a hard intersection. The documented contract is: returns a
|
|
// ranked array in which the field-matched entities are present.
|
|
const triple = brain.getTripleIntelligence()
|
|
const fused = await triple.find({
|
|
like: 'web framework',
|
|
where: { category: 'frontend' },
|
|
limit: 10
|
|
})
|
|
|
|
expect(fused).toBeInstanceOf(Array)
|
|
expect(fused.length).toBeGreaterThan(0)
|
|
|
|
// All three frontend frameworks (the field-matched set) appear in the fused output.
|
|
const names = new Set(fused.map(r => r.metadata?.name))
|
|
expect(names.has('React')).toBe(true)
|
|
expect(names.has('Vue.js')).toBe(true)
|
|
expect(names.has('Angular')).toBe(true)
|
|
|
|
// Every row carries a numeric fusion score and a full entity.
|
|
fused.forEach(r => {
|
|
expect(typeof r.score).toBe('number')
|
|
expect(r.entity).toBeTruthy()
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('4. Metadata filtering via find({ where })', () => {
|
|
it('should perform metadata-only filtering with exact matches', async () => {
|
|
// Metadata-only query (no vector search) — filters frameworks added above.
|
|
const backendPython = await brain.find({
|
|
where: { category: 'backend', language: 'Python' },
|
|
limit: 10
|
|
})
|
|
|
|
expect(backendPython).toBeInstanceOf(Array)
|
|
expect(backendPython.length).toBeGreaterThan(0)
|
|
backendPython.forEach(result => {
|
|
expect(result.entity.metadata?.category).toBe('backend')
|
|
expect(result.entity.metadata?.language).toBe('Python')
|
|
})
|
|
|
|
// Django + FastAPI are the only Python backends.
|
|
const names = backendPython.map(r => r.entity.metadata?.name).sort()
|
|
expect(names).toEqual(['Django', 'FastAPI'])
|
|
})
|
|
|
|
it('should handle nested metadata on add() and retrieve it intact', async () => {
|
|
const id = await brain.add({
|
|
data: 'Advanced framework test',
|
|
type: 'thing',
|
|
metadata: {
|
|
framework: {
|
|
name: 'Next.js',
|
|
version: '13.0',
|
|
features: ['SSR', 'API', 'Routing']
|
|
},
|
|
tech: {
|
|
language: 'JavaScript',
|
|
runtime: 'Node.js'
|
|
}
|
|
}
|
|
})
|
|
|
|
const retrieved = await brain.get(id)
|
|
expect(retrieved).toBeTruthy()
|
|
expect(retrieved?.metadata?.framework?.name).toBe('Next.js')
|
|
expect(retrieved?.metadata?.framework?.features).toEqual(['SSR', 'API', 'Routing'])
|
|
expect(retrieved?.metadata?.tech?.runtime).toBe('Node.js')
|
|
})
|
|
})
|
|
|
|
describe('5. Statistics and consistency', () => {
|
|
it('should report growing entity statistics as data is added', async () => {
|
|
const initialStats = await brain.getStats()
|
|
const initialTotal = initialStats.entities.total
|
|
|
|
// Add a batch and confirm the total grows by exactly that amount.
|
|
const batchData = Array.from({ length: 20 }, (_, i) =>
|
|
`Optimization test item ${i}: ${Math.random().toString(36).slice(2)}`
|
|
)
|
|
|
|
for (const item of batchData) {
|
|
await brain.add({
|
|
data: item,
|
|
type: 'thing',
|
|
metadata: { batch: 'optimization', index: Math.floor(Math.random() * 100) }
|
|
})
|
|
}
|
|
|
|
const finalStats = await brain.getStats()
|
|
expect(finalStats.entities.total).toBe(initialTotal + 20)
|
|
expect(finalStats.entities.total).toBeGreaterThan(initialTotal)
|
|
})
|
|
|
|
it('should round-trip an entity through add/get and a metadata filter', async () => {
|
|
// This brain already holds 80+ entities. Vector self-retrieval is exact only
|
|
// at small scale (covered in section 1 and tests/integration/vector-recall.test.ts);
|
|
// here we verify the DETERMINISTIC, exact round-trip paths: get() by id and the
|
|
// metadata index via find({ where }).
|
|
const data = 'Persistence test item with a unique phrase about consistency'
|
|
const uniqueTag = `persist-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
const testId = await brain.add({
|
|
data,
|
|
type: 'thing',
|
|
metadata: { test: 'persistence', tag: uniqueTag }
|
|
})
|
|
|
|
// Immediate metadata retrieval by id (exact).
|
|
const retrieved = await brain.get(testId)
|
|
expect(retrieved).toBeTruthy()
|
|
expect(retrieved?.metadata?.test).toBe('persistence')
|
|
expect(retrieved?.data).toBe(data)
|
|
|
|
// Metadata-index lookup by a unique tag returns exactly this entity (exact).
|
|
const byTag = await brain.find({ where: { tag: uniqueTag }, limit: 5 })
|
|
expect(byTag).toHaveLength(1)
|
|
expect(byTag[0].id).toBe(testId)
|
|
})
|
|
})
|
|
|
|
describe('6. Embedding generation', () => {
|
|
it('should produce a 384-dimension finite embedding vector', async () => {
|
|
const embedding = await brain.embed('test embedding generation')
|
|
expect(embedding).toBeInstanceOf(Array)
|
|
expect(embedding).toHaveLength(384)
|
|
|
|
// Tier-1 (deterministic embedder): assert each component is a finite number
|
|
// of reasonable magnitude. The normalized (-1, 1) unit-vector property is a
|
|
// real-model characteristic verified by the Tier-2 semantic suite, not by
|
|
// the content-derived deterministic stand-in.
|
|
embedding.forEach(val => {
|
|
expect(typeof val).toBe('number')
|
|
expect(Number.isFinite(val)).toBe(true)
|
|
expect(Math.abs(val)).toBeLessThan(2)
|
|
})
|
|
})
|
|
|
|
it('should produce identical embeddings for identical input (determinism)', async () => {
|
|
const a = await brain.embed('determinism check phrase')
|
|
const b = await brain.embed('determinism check phrase')
|
|
expect(a).toEqual(b)
|
|
})
|
|
})
|
|
|
|
describe('7. Bulk operations', () => {
|
|
it('should ingest a batch with metadata and remain queryable', async () => {
|
|
const performanceData = Array.from({ length: 50 }, (_, i) => ({
|
|
content: `Bulk test ${i}: ${Array.from({ length: 20 }, () =>
|
|
Math.random().toString(36).slice(2)).join(' ')}`,
|
|
category: ['frontend', 'backend', 'database', 'ai', 'devops'][i % 5],
|
|
priority: Math.floor(Math.random() * 100),
|
|
timestamp: Date.now() + i
|
|
}))
|
|
|
|
const beforeStats = await brain.getStats()
|
|
const ids: string[] = []
|
|
for (const item of performanceData) {
|
|
const id = await brain.add({
|
|
data: item.content,
|
|
type: 'thing',
|
|
metadata: {
|
|
category: item.category,
|
|
priority: item.priority,
|
|
timestamp: item.timestamp
|
|
}
|
|
})
|
|
ids.push(id)
|
|
}
|
|
expect(ids).toHaveLength(50)
|
|
|
|
// All 50 are counted (exact).
|
|
const afterStats = await brain.getStats()
|
|
expect(afterStats.entities.total).toBe(beforeStats.entities.total + 50)
|
|
|
|
// Each ingested item is retrievable by id and carries its metadata (exact,
|
|
// deterministic — independent of approximate vector recall at this scale).
|
|
const firstBack = await brain.get(ids[0])
|
|
expect(firstBack).toBeTruthy()
|
|
expect(firstBack?.data).toBe(performanceData[0].content)
|
|
expect(firstBack?.metadata?.category).toBe(performanceData[0].category)
|
|
|
|
const lastBack = await brain.get(ids[49])
|
|
expect(lastBack).toBeTruthy()
|
|
expect(lastBack?.data).toBe(performanceData[49].content)
|
|
|
|
// The batch is queryable by metadata: every 5th item is 'frontend'.
|
|
const frontendInBatch = await brain.find({
|
|
where: { category: 'frontend', priority: { greaterThanOrEqual: 0 } },
|
|
limit: 100
|
|
})
|
|
// At least the 10 frontend items from this batch (indices 0,5,10,...,45).
|
|
const frontendIds = new Set(frontendInBatch.map(r => r.id))
|
|
const batchFrontendIds = ids.filter((_, i) => i % 5 === 0)
|
|
batchFrontendIds.forEach(id => expect(frontendIds.has(id)).toBe(true))
|
|
})
|
|
})
|
|
|
|
describe('8. Final integration verification', () => {
|
|
it('should pass comprehensive cross-API verification', async () => {
|
|
// 1. find() vector search returns well-formed, score-sorted results. (Exact
|
|
// self-retrieval at small scale is covered in section 1; this brain holds
|
|
// 130+ entities, beyond the deterministic embedder's exact-recall envelope.)
|
|
const searchResults = await brain.find({ query: 'web development framework', limit: 5 })
|
|
expect(searchResults).toHaveLength(5)
|
|
searchResults.forEach(r => {
|
|
expect(typeof r.score).toBe('number')
|
|
expect(r.entity).toBeTruthy()
|
|
})
|
|
|
|
// 2. Exact lookup by unique metadata still resolves a known entity.
|
|
const vue = await brain.find({ where: { name: 'Vue.js' }, limit: 1 })
|
|
expect(vue).toHaveLength(1)
|
|
expect(vue[0].entity.metadata?.category).toBe('frontend')
|
|
|
|
// 3. find() query-object form returns the documented array shape.
|
|
const findResults = await brain.find({ query: 'frontend technologies', limit: 3 })
|
|
expect(findResults).toHaveLength(3)
|
|
|
|
// 4. find({ query, where }) hard-AND composition (semantic + metadata filter).
|
|
const frontend = await brain.find({
|
|
query: 'web framework',
|
|
where: { category: 'frontend' },
|
|
limit: 3,
|
|
searchMode: 'semantic'
|
|
})
|
|
expect(frontend).toBeInstanceOf(Array)
|
|
expect(frontend.length).toBeGreaterThan(0)
|
|
frontend.forEach(r => expect(r.entity.metadata?.category).toBe('frontend'))
|
|
|
|
// 5. Metadata-only filtering.
|
|
const backend = await brain.find({ where: { category: 'backend' }, limit: 50 })
|
|
expect(backend).toBeInstanceOf(Array)
|
|
expect(backend.length).toBeGreaterThan(0)
|
|
backend.forEach(r => expect(r.entity.metadata?.category).toBe('backend'))
|
|
|
|
// 6. Statistics.
|
|
const finalStats = await brain.getStats()
|
|
expect(finalStats.entities.total).toBeGreaterThan(50)
|
|
})
|
|
})
|
|
})
|