soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main'

Browse source 
# Conflicts:
#	CHANGES.md
#	package-lock.json
#	package.json
#	src/storage/adapters/fileSystemStorage.ts
This commit is contained in:
David Snelling 2025-07-28 10:04:58 -07:00
commit da675f0e5b
49 changed files with 5099 additions and 4391 deletions
Download patch file Download diff file Expand all files Collapse all files

129
tests/core.test.ts
View file

@ -5,6 +5,17 @@
import { describe, it, expect, beforeAll } from 'vitest'
/**
* Helper function to create a 512-dimensional vector for testing
* @param primaryIndex The index to set to 1.0, all other indices will be 0.0
* @returns A 512-dimensional vector with a single 1.0 value at the specified index
*/
function createTestVector(primaryIndex: number = 0): number[] {
const vector = new Array(512).fill(0)
vector[primaryIndex % 512] = 1.0
return vector
}
describe('Brainy Core Functionality', () => {
let brainy: any
@ -42,17 +53,14 @@ describe('Brainy Core Functionality', () => {
describe('BrainyData Configuration', () => {
it('should create instance with minimal configuration', () => {
const data = new brainy.BrainyData({
dimensions: 3
})
const data = new brainy.BrainyData({})
expect(data).toBeDefined()
expect(data.dimensions).toBe(3)
expect(data.dimensions).toBe(512)
})
it('should create instance with full configuration', () => {
const data = new brainy.BrainyData({
dimensions: 128,
metric: 'cosine',
maxConnections: 32,
efConstruction: 200,
@ -60,29 +68,28 @@ describe('Brainy Core Functionality', () => {
})
expect(data).toBeDefined()
expect(data.dimensions).toBe(128)
expect(data.dimensions).toBe(512)
})
it('should validate configuration parameters', () => {
it('should not throw with valid configuration parameters', () => {
// Dimensions are now fixed at 512 and not configurable
expect(() => {
new brainy.BrainyData({
dimensions: 0 // Invalid dimensions
metric: 'cosine'
})
}).toThrow()
}).not.toThrow()
expect(() => {
new brainy.BrainyData({
dimensions: -1 // Invalid dimensions
metric: 'euclidean'
})
}).toThrow()
}).not.toThrow()
})
it('should use default values for optional parameters', () => {
const data = new brainy.BrainyData({
dimensions: 10
})
const data = new brainy.BrainyData({})
expect(data.dimensions).toBe(10)
expect(data.dimensions).toBe(512)
// Should have reasonable defaults for other parameters
expect(data.maxConnections).toBeGreaterThan(0)
expect(data.efConstruction).toBeGreaterThan(0)
@ -92,20 +99,19 @@ describe('Brainy Core Functionality', () => {
describe('Vector Operations', () => {
it('should handle vector addition and search', async () => {
const data = new brainy.BrainyData({
dimensions: 3,
metric: 'euclidean'
})
await data.init()
await data.clear() // Clear any existing data
// Add vectors
await data.add([1, 0, 0], { id: 'v1', label: 'x-axis' })
await data.add([0, 1, 0], { id: 'v2', label: 'y-axis' })
await data.add([0, 0, 1], { id: 'v3', label: 'z-axis' })
// Add vectors using helper function
await data.add(createTestVector(0), { id: 'v1', label: 'x-axis' })
await data.add(createTestVector(1), { id: 'v2', label: 'y-axis' })
await data.add(createTestVector(2), { id: 'v3', label: 'z-axis' })
// Search for similar vector
const results = await data.search([1, 0, 0], 1)
const results = await data.search(createTestVector(0), 1)
expect(results).toBeDefined()
expect(results.length).toBe(1)
@ -114,7 +120,6 @@ describe('Brainy Core Functionality', () => {
it('should handle batch vector operations', async () => {
const data = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean'
})
@ -123,9 +128,9 @@ describe('Brainy Core Functionality', () => {
// Add multiple vectors
const vectors = [
{ vector: [1, 1], metadata: { id: 'batch1' } },
{ vector: [2, 2], metadata: { id: 'batch2' } },
{ vector: [3, 3], metadata: { id: 'batch3' } }
{ vector: createTestVector(10), metadata: { id: 'batch1' } },
{ vector: createTestVector(20), metadata: { id: 'batch2' } },
{ vector: createTestVector(30), metadata: { id: 'batch3' } }
]
for (const { vector, metadata } of vectors) {
@ -133,18 +138,16 @@ describe('Brainy Core Functionality', () => {
}
// Search should return results
const results = await data.search([1.5, 1.5], 3)
const results = await data.search(createTestVector(15), 3)
expect(results.length).toBe(3)
})
it('should handle different distance metrics', async () => {
const euclideanData = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean'
})
const cosineData = new brainy.BrainyData({
dimensions: 2,
metric: 'cosine'
})
@ -155,7 +158,7 @@ describe('Brainy Core Functionality', () => {
await euclideanData.clear()
await cosineData.clear()
const vector = [1, 1]
const vector = createTestVector(5)
const metadata = { id: 'test' }
await euclideanData.add(vector, metadata)
@ -237,7 +240,6 @@ describe('Brainy Core Functionality', () => {
describe('Error Handling', () => {
it('should handle invalid vector dimensions', async () => {
const data = new brainy.BrainyData({
dimensions: 3,
metric: 'euclidean'
})
@ -245,22 +247,20 @@ describe('Brainy Core Functionality', () => {
// Try to add vector with wrong dimensions
await expect(data.add([1, 2], { id: 'wrong' })).rejects.toThrow()
await expect(data.add([1, 2, 3, 4], { id: 'wrong' })).rejects.toThrow()
await expect(data.add(new Array(100).fill(0), { id: 'wrong' })).rejects.toThrow()
})
it('should handle search before initialization', async () => {
const data = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean'
})
// Try to search without initialization
await expect(data.search([1, 2], 1)).rejects.toThrow()
await expect(data.search(createTestVector(0), 1)).rejects.toThrow()
})
it('should handle empty search results gracefully', async () => {
const data = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean'
})
@ -268,7 +268,7 @@ describe('Brainy Core Functionality', () => {
await data.clear() // Clear any existing data
// Search in empty database
const results = await data.search([1, 2], 1)
const results = await data.search(createTestVector(0), 1)
expect(results).toBeDefined()
expect(Array.isArray(results)).toBe(true)
expect(results.length).toBe(0)
@ -278,7 +278,6 @@ describe('Brainy Core Functionality', () => {
describe('Performance and Scalability', () => {
it('should handle moderate number of vectors efficiently', async () => {
const data = new brainy.BrainyData({
dimensions: 10,
metric: 'euclidean'
})
@ -288,21 +287,14 @@ describe('Brainy Core Functionality', () => {
// Add 100 test vectors
for (let i = 0; i < 100; i++) {
const vector =
globalThis.testUtils?.createTestVector(10) ||
Array.from({ length: 10 }, (_, i) => (i + 1) / 10)
await data.add(vector, { id: `item_${i}`, index: i })
await data.add(createTestVector(i), { id: `item_${i}`, index: i })
}
const addTime = Date.now() - startTime
// Search should be fast
const searchStart = Date.now()
const results = await data.search(
globalThis.testUtils?.createTestVector(10) ||
Array.from({ length: 10 }, (_, i) => (i + 1) / 10),
10
)
const results = await data.search(createTestVector(50), 10)
const searchTime = Date.now() - searchStart
expect(results.length).toBeLessThanOrEqual(10)
@ -342,4 +334,51 @@ describe('Brainy Core Functionality', () => {
expect(results[0].metadata?.id).toBe('known')
})
})
describe('Database Statistics', () => {
it('should provide accurate statistics about the database', async () => {
const data = new brainy.BrainyData({
metric: 'euclidean'
})
await data.init()
await data.clear() // Clear any existing data
// Add some vectors (nouns)
await data.add(createTestVector(0), { id: 'v1', label: 'x-axis' })
await data.add(createTestVector(1), { id: 'v2', label: 'y-axis' })
await data.add(createTestVector(2), { id: 'v3', label: 'z-axis' })
// Add some connections (verbs)
await data.connect('v1', 'v2', 'related_to')
await data.connect('v2', 'v3', 'related_to')
// Get statistics
const stats = await data.getStatistics()
// Debug: Log all nouns in the database
const allNouns = await data.getAllNouns()
console.log('All nouns in database:', allNouns.map(n => n.id))
// Debug: Log all verbs in the database
const allVerbs = await data.getAllVerbs()
console.log('All verbs in database:', allVerbs.map(v => v.id))
// Debug: Log the verb IDs set used in getStatistics
const verbIds = new Set(allVerbs.map(verb => verb.id))
console.log('Verb IDs set:', Array.from(verbIds))
// Verify statistics
expect(stats).toBeDefined()
expect(stats).toHaveProperty('nounCount')
expect(stats).toHaveProperty('verbCount')
expect(stats).toHaveProperty('metadataCount')
expect(stats).toHaveProperty('hnswIndexSize')
// Verify counts
expect(stats.nounCount).toBe(3)
expect(stats.verbCount).toBe(2)
expect(stats.hnswIndexSize).toBe(3)
})
})
})

64
tests/database-operations.test.ts Normal file
View file

@ -0,0 +1,64 @@
import { describe, it, expect } from 'vitest'
import { BrainyData } from '../dist/brainyData.js'
describe('Database Operations', () => {
let db: BrainyData
beforeEach(async () => {
db = new BrainyData()
await db.init()
})
it('should initialize and return database status', async () => {
const status = await db.status()
expect(status).toBeDefined()
// The structure of status might vary, just check it exists
})
it('should return statistics', async () => {
const stats = await db.getStatistics()
expect(stats).toBeDefined()
// The structure of stats might vary, just check it exists
})
it('should retrieve all nouns', async () => {
const nouns = await db.getAllNouns()
expect(Array.isArray(nouns)).toBe(true)
})
it('should retrieve all verbs', async () => {
const verbs = await db.getAllVerbs()
expect(Array.isArray(verbs)).toBe(true)
})
it('should perform a search operation', async () => {
const searchResults = await db.searchText('test', 10)
expect(Array.isArray(searchResults)).toBe(true)
})
it('should add and retrieve an item', async () => {
// Add a test item
const testText = 'This is a test item for searching'
const metadata = { noun: 'Thing', category: 'test' }
const id = await db.add(testText, metadata)
// Verify the item was added
expect(id).toBeDefined()
// Retrieve the item
const noun = await db.get(id)
expect(noun).toBeDefined()
expect(noun.id).toBe(id)
// Check that the metadata contains our properties
// (The system might add additional properties)
expect(noun.metadata.category).toBe('test')
// Search for the item
const searchResults = await db.searchText('test', 10)
expect(searchResults.length).toBeGreaterThan(0)
// Clean up
await db.delete(id)
})
})

58
tests/dimension-standardization.test.ts Normal file
View file

@ -0,0 +1,58 @@
import { describe, it, expect } from 'vitest'
import { BrainyData } from '../dist/brainyData.js'
describe('Vector Dimension Standardization', () => {
it('should initialize BrainyData with 512 dimensions', async () => {
// Initialize BrainyData
const db = new BrainyData()
await db.init()
// Check the dimensions property
expect(db.dimensions).toBe(512)
})
it('should reject vectors with incorrect dimensions', async () => {
const db = new BrainyData()
await db.init()
// Test with a simple vector (this should throw an error because it's not 512 dimensions)
const smallVector = [0.1, 0.2, 0.3]
// Expect the add operation to throw an error
await expect(db.add(smallVector, { test: 'small-vector' }))
.rejects.toThrow()
})
it('should successfully embed text to 512 dimensions', async () => {
const db = new BrainyData()
await db.init()
// Test with text that will be embedded to 512 dimensions
const id = await db.add('This is a test text that will be embedded to 512 dimensions', { test: 'text-embedding' })
// Retrieve the vector and check its dimensions
const noun = await db.get(id)
expect(noun.vector.length).toBe(512)
})
it('should directly embed text to 512 dimensions', async () => {
const db = new BrainyData()
await db.init()
// Test direct embedding
const vector = await db.embed('Another test text')
expect(vector.length).toBe(512)
})
it('should use the configured dimensions', async () => {
// Create a BrainyData instance with a specific dimension
const customDimension = 300
const db = new BrainyData({
dimensions: customDimension
})
await db.init()
// The API appears to respect the configured dimensions
expect(db.dimensions).toBe(customDimension)
})
})

38
tests/environment.browser.test.ts
View file

@ -6,6 +6,17 @@
import { describe, it, expect, beforeAll, vi } from 'vitest'
/**
* Helper function to create a 512-dimensional vector for testing
* @param primaryIndex The index to set to 1.0, all other indices will be 0.0
* @returns A 512-dimensional vector with a single 1.0 value at the specified index
*/
function createTestVector(primaryIndex: number = 0): number[] {
const vector = new Array(512).fill(0)
vector[primaryIndex % 512] = 1.0
return vector
}
describe('Brainy in Browser Environment', () => {
let brainy: any
@ -53,7 +64,6 @@ describe('Brainy in Browser Environment', () => {
describe('Core Functionality - Add Data and Search', () => {
it('should create database and add vector data', async () => {
const db = new brainy.BrainyData({
dimensions: 3,
metric: 'euclidean',
storage: {
forceMemoryStorage: true
@ -63,12 +73,12 @@ describe('Brainy in Browser Environment', () => {
await db.init()
// Add some test vectors
await db.add([1, 0, 0], { id: 'item1', label: 'x-axis' })
await db.add([0, 1, 0], { id: 'item2', label: 'y-axis' })
await db.add([0, 0, 1], { id: 'item3', label: 'z-axis' })
await db.add(createTestVector(0), { id: 'item1', label: 'x-axis' })
await db.add(createTestVector(1), { id: 'item2', label: 'y-axis' })
await db.add(createTestVector(2), { id: 'item3', label: 'z-axis' })
// Search should work
const results = await db.search([1, 0, 0], 1)
const results = await db.search(createTestVector(0), 1)
expect(results).toBeDefined()
expect(results.length).toBe(1)
expect(results[0].metadata.id).toBe('item1')
@ -102,7 +112,6 @@ describe('Brainy in Browser Environment', () => {
it('should handle multiple data types', async () => {
const db = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean',
storage: {
forceMemoryStorage: true
@ -113,9 +122,9 @@ describe('Brainy in Browser Environment', () => {
// Add different types of data
const testData = [
{ vector: [1, 1], metadata: { type: 'point', name: 'A' } },
{ vector: [2, 2], metadata: { type: 'point', name: 'B' } },
{ vector: [3, 3], metadata: { type: 'point', name: 'C' } }
{ vector: createTestVector(10), metadata: { type: 'point', name: 'A' } },
{ vector: createTestVector(20), metadata: { type: 'point', name: 'B' } },
{ vector: createTestVector(30), metadata: { type: 'point', name: 'C' } }
]
for (const item of testData) {
@ -123,7 +132,7 @@ describe('Brainy in Browser Environment', () => {
}
// Search should return relevant results
const results = await db.search([1.5, 1.5], 2)
const results = await db.search(createTestVector(15), 2)
expect(results.length).toBe(2)
expect(
results.every(
@ -134,15 +143,14 @@ describe('Brainy in Browser Environment', () => {
})
describe('Error Handling', () => {
it('should handle invalid configurations gracefully', () => {
it('should not throw with valid configuration', () => {
expect(() => {
new brainy.BrainyData({ dimensions: 0 })
}).toThrow()
new brainy.BrainyData({ metric: 'euclidean' })
}).not.toThrow()
})
it('should handle search on empty database', async () => {
const db = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean',
storage: {
forceMemoryStorage: true
@ -151,7 +159,7 @@ describe('Brainy in Browser Environment', () => {
await db.init()
const results = await db.search([1, 2], 5)
const results = await db.search(createTestVector(0), 5)
expect(results).toBeDefined()
expect(Array.isArray(results)).toBe(true)
expect(results.length).toBe(0)

38
tests/environment.node.test.ts
View file

@ -5,6 +5,17 @@
import { describe, it, expect, beforeAll } from 'vitest'
/**
* Helper function to create a 512-dimensional vector for testing
* @param primaryIndex The index to set to 1.0, all other indices will be 0.0
* @returns A 512-dimensional vector with a single 1.0 value at the specified index
*/
function createTestVector(primaryIndex: number = 0): number[] {
const vector = new Array(512).fill(0)
vector[primaryIndex % 512] = 1.0
return vector
}
describe('Brainy in Node.js Environment', () => {
let brainy: any
@ -53,7 +64,6 @@ describe('Brainy in Node.js Environment', () => {
return
}
const db = new brainy.BrainyData({
dimensions: 3,
metric: 'euclidean',
storage: {
forceMemoryStorage: true
@ -64,12 +74,12 @@ describe('Brainy in Node.js Environment', () => {
await db.clear() // Clear any existing data
// Add some test vectors
await db.add([1, 0, 0], { id: 'item1', label: 'x-axis' })
await db.add([0, 1, 0], { id: 'item2', label: 'y-axis' })
await db.add([0, 0, 1], { id: 'item3', label: 'z-axis' })
await db.add(createTestVector(0), { id: 'item1', label: 'x-axis' })
await db.add(createTestVector(1), { id: 'item2', label: 'y-axis' })
await db.add(createTestVector(2), { id: 'item3', label: 'z-axis' })
// Search should work
const results = await db.search([1, 0, 0], 1)
const results = await db.search(createTestVector(0), 1)
expect(results).toBeDefined()
expect(results.length).toBe(1)
expect(results[0].metadata.id).toBe('item1')
@ -114,7 +124,6 @@ describe('Brainy in Node.js Environment', () => {
return
}
const db = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean',
storage: {
forceMemoryStorage: true
@ -126,9 +135,9 @@ describe('Brainy in Node.js Environment', () => {
// Add different types of data
const testData = [
{ vector: [1, 1], metadata: { type: 'point', name: 'A' } },
{ vector: [2, 2], metadata: { type: 'point', name: 'B' } },
{ vector: [3, 3], metadata: { type: 'point', name: 'C' } }
{ vector: createTestVector(10), metadata: { type: 'point', name: 'A' } },
{ vector: createTestVector(20), metadata: { type: 'point', name: 'B' } },
{ vector: createTestVector(30), metadata: { type: 'point', name: 'C' } }
]
for (const item of testData) {
@ -136,7 +145,7 @@ describe('Brainy in Node.js Environment', () => {
}
// Search should return relevant results
const results = await db.search([1.5, 1.5], 2)
const results = await db.search(createTestVector(15), 2)
expect(results.length).toBe(2)
expect(
results.every(
@ -147,14 +156,14 @@ describe('Brainy in Node.js Environment', () => {
})
describe('Error Handling', () => {
it('should handle invalid configurations gracefully', () => {
it('should not throw with valid configuration', () => {
if (brainy === null) {
console.warn('Skipping test due to TensorFlow.js initialization issue')
return
}
expect(() => {
new brainy.BrainyData({ dimensions: 0 })
}).toThrow()
new brainy.BrainyData({ metric: 'euclidean' })
}).not.toThrow()
})
it('should handle search on empty database', async () => {
@ -163,7 +172,6 @@ describe('Brainy in Node.js Environment', () => {
return
}
const db = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean',
storage: {
forceMemoryStorage: true
@ -173,7 +181,7 @@ describe('Brainy in Node.js Environment', () => {
await db.init()
await db.clear() // Clear any existing data
const results = await db.search([1, 2], 5)
const results = await db.search(createTestVector(0), 5)
expect(results).toBeDefined()
expect(Array.isArray(results)).toBe(true)
expect(results.length).toBe(0)

30
tests/opfs-storage.test.ts
View file

@ -120,15 +120,25 @@ describe('OPFSStorage', () => {
// Create test verb
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const timestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
const testVerb = {
id: 'test-verb-1',
vector: testVector,
connections: new Map(),
sourceId: 'source-noun-1',
targetId: 'target-noun-1',
type: 'test-relation',
source: 'source-noun-1',
target: 'target-noun-1',
verb: 'test-relation',
weight: 0.75,
metadata: { description: 'Test relation' }
metadata: { description: 'Test relation' },
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
augmentation: 'test-service',
version: '1.0'
}
}
// Save the verb
@ -141,11 +151,17 @@ describe('OPFSStorage', () => {
expect(retrievedVerb).toBeDefined()
expect(retrievedVerb?.id).toBe('test-verb-1')
expect(retrievedVerb?.vector).toEqual(testVector)
expect(retrievedVerb?.sourceId).toBe('source-noun-1')
expect(retrievedVerb?.targetId).toBe('target-noun-1')
expect(retrievedVerb?.type).toBe('test-relation')
expect(retrievedVerb?.source).toBe('source-noun-1')
expect(retrievedVerb?.target).toBe('target-noun-1')
expect(retrievedVerb?.verb).toBe('test-relation')
expect(retrievedVerb?.weight).toBe(0.75)
expect(retrievedVerb?.metadata).toEqual({ description: 'Test relation' })
expect(retrievedVerb?.createdAt).toEqual(timestamp)
expect(retrievedVerb?.updatedAt).toEqual(timestamp)
expect(retrievedVerb?.createdBy).toEqual({
augmentation: 'test-service',
version: '1.0'
})
// Test getAllVerbs
const allVerbs = await opfsStorage.getAllVerbs()

198
tests/package-size-limit.test.ts
View file

@ -3,64 +3,64 @@
* Tests the predicted npm package size to ensure it stays within acceptable limits
*/
import { describe, expect, it } from 'vitest'
import { execSync } from 'child_process'
import {describe, expect, it} from 'vitest'
import {execSync} from 'child_process'
const CURRENT_UNPACKED_SIZE_MB = 10.4
const CURRENT_PACKED_SIZE_MB = 1.9
const CURRENT_UNPACKED_SIZE_MB = 11.1
const CURRENT_PACKED_SIZE_MB = 2.5
const ALLOWED_SIZE_INCREASE_PERCENTAGE = 5 // 5% increase threshold
/**
* Parses npm pack --dry-run output to extract package size information
*/
function parseNpmPackOutput(output: string): {
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
} {
const packageSizeMatch = output.match(
/npm notice package size:\s*([\d.]+)\s*([KMGT]?B)/
)
const unpackedSizeMatch = output.match(
/npm notice unpacked size:\s*([\d.]+)\s*([KMGT]?B)/
)
const totalFilesMatch = output.match(/npm notice total files:\s*(\d+)/)
const packageSizeMatch = output.match(
/npm notice package size:\s*([\d.]+)\s*([KMGT]?B)/
)
const unpackedSizeMatch = output.match(
/npm notice unpacked size:\s*([\d.]+)\s*([KMGT]?B)/
)
const totalFilesMatch = output.match(/npm notice total files:\s*(\d+)/)
const convertToMB = (size: number, unit: string): number => {
switch (unit) {
case 'B':
return size / (1024 * 1024)
case 'KB':
return size / 1024
case 'MB':
return size
case 'GB':
return size * 1024
default:
return size / (1024 * 1024) // assume bytes
const convertToMB = (size: number, unit: string): number => {
switch (unit) {
case 'B':
return size / (1024 * 1024)
case 'KB':
return size / 1024
case 'MB':
return size
case 'GB':
return size * 1024
default:
return size / (1024 * 1024) // assume bytes
}
}
}
const packedSizeMB = packageSizeMatch
? convertToMB(parseFloat(packageSizeMatch[1]), packageSizeMatch[2])
: 0
const packedSizeMB = packageSizeMatch
? convertToMB(parseFloat(packageSizeMatch[1]), packageSizeMatch[2])
: 0
const unpackedSizeMB = unpackedSizeMatch
? convertToMB(parseFloat(unpackedSizeMatch[1]), unpackedSizeMatch[2])
: 0
const unpackedSizeMB = unpackedSizeMatch
? convertToMB(parseFloat(unpackedSizeMatch[1]), unpackedSizeMatch[2])
: 0
const totalFiles = totalFilesMatch ? parseInt(totalFilesMatch[1], 10) : 0
const totalFiles = totalFilesMatch ? parseInt(totalFilesMatch[1], 10) : 0
return { packedSizeMB, unpackedSizeMB, totalFiles }
return {packedSizeMB, unpackedSizeMB, totalFiles}
}
/**
* Cached npm package size result to avoid multiple expensive npm pack calls
*/
let cachedPackageSize: {
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
} | null = null
/**
@ -68,79 +68,79 @@ let cachedPackageSize: {
* Results are cached to avoid multiple expensive executions
*/
async function getNpmPackageSize(): Promise<{
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
}> {
// Return cached result if available
if (cachedPackageSize) {
return cachedPackageSize
}
// Return cached result if available
if (cachedPackageSize) {
return cachedPackageSize
}
try {
// Use 2>&1 to capture both stdout and stderr in one command
const output = execSync('npm pack --dry-run 2>&1', {
encoding: 'utf8',
cwd: process.cwd(),
timeout: 45000 // 45 second timeout to prevent hanging
})
try {
// Use 2>&1 to capture both stdout and stderr in one command
const output = execSync('npm pack --dry-run 2>&1', {
encoding: 'utf8',
cwd: process.cwd(),
timeout: 45000 // 45 second timeout to prevent hanging
})
const result = parseNpmPackOutput(output)
// Cache the result for subsequent calls
cachedPackageSize = result
return result
} catch (error) {
throw new Error(`Failed to get npm package size: ${error}`)
}
const result = parseNpmPackOutput(output)
// Cache the result for subsequent calls
cachedPackageSize = result
return result
} catch (error) {
throw new Error(`Failed to get npm package size: ${error}`)
}
}
describe('Package Size Limits', () => {
it('should not exceed unpacked size threshold for npm package', async () => {
const { unpackedSizeMB } = await getNpmPackageSize()
const maxAllowedSize =
CURRENT_UNPACKED_SIZE_MB * (1 + ALLOWED_SIZE_INCREASE_PERCENTAGE / 100)
it('should not exceed unpacked size threshold for npm package', async () => {
const {unpackedSizeMB} = await getNpmPackageSize()
const maxAllowedSize =
CURRENT_UNPACKED_SIZE_MB * (1 + ALLOWED_SIZE_INCREASE_PERCENTAGE / 100)
console.log(`Current unpacked package size: ${unpackedSizeMB.toFixed(2)}MB`)
console.log(`Maximum allowed unpacked size: ${maxAllowedSize.toFixed(2)}MB`)
console.log(`Current unpacked package size: ${unpackedSizeMB.toFixed(2)}MB`)
console.log(`Maximum allowed unpacked size: ${maxAllowedSize.toFixed(2)}MB`)
expect(
unpackedSizeMB,
`Unpacked package size (${unpackedSizeMB.toFixed(2)}MB) exceeds maximum allowed size (${maxAllowedSize.toFixed(2)}MB)`
).toBeLessThanOrEqual(maxAllowedSize)
})
expect(
unpackedSizeMB,
`Unpacked package size (${unpackedSizeMB.toFixed(2)}MB) exceeds maximum allowed size (${maxAllowedSize.toFixed(2)}MB)`
).toBeLessThanOrEqual(maxAllowedSize)
})
it('should not exceed packed size threshold for npm package', async () => {
const { packedSizeMB } = await getNpmPackageSize()
const maxAllowedSize =
CURRENT_PACKED_SIZE_MB * (1 + ALLOWED_SIZE_INCREASE_PERCENTAGE / 100)
it('should not exceed packed size threshold for npm package', async () => {
const {packedSizeMB} = await getNpmPackageSize()
const maxAllowedSize =
CURRENT_PACKED_SIZE_MB * (1 + ALLOWED_SIZE_INCREASE_PERCENTAGE / 100)
console.log(`Current packed package size: ${packedSizeMB.toFixed(2)}MB`)
console.log(`Maximum allowed packed size: ${maxAllowedSize.toFixed(2)}MB`)
console.log(`Current packed package size: ${packedSizeMB.toFixed(2)}MB`)
console.log(`Maximum allowed packed size: ${maxAllowedSize.toFixed(2)}MB`)
expect(
packedSizeMB,
`Packed package size (${packedSizeMB.toFixed(2)}MB) exceeds maximum allowed size (${maxAllowedSize.toFixed(2)}MB)`
).toBeLessThanOrEqual(maxAllowedSize)
})
expect(
packedSizeMB,
`Packed package size (${packedSizeMB.toFixed(2)}MB) exceeds maximum allowed size (${maxAllowedSize.toFixed(2)}MB)`
).toBeLessThanOrEqual(maxAllowedSize)
})
it('should report package composition details', async () => {
const { packedSizeMB, unpackedSizeMB, totalFiles } =
await getNpmPackageSize()
it('should report package composition details', async () => {
const {packedSizeMB, unpackedSizeMB, totalFiles} =
await getNpmPackageSize()
console.log(`\nPackage composition:`)
console.log(`- Total files: ${totalFiles}`)
console.log(`- Packed size: ${packedSizeMB.toFixed(2)}MB`)
console.log(`- Unpacked size: ${unpackedSizeMB.toFixed(2)}MB`)
console.log(
`- Compression ratio: ${((1 - packedSizeMB / unpackedSizeMB) * 100).toFixed(1)}%`
)
console.log(`\nPackage composition:`)
console.log(`- Total files: ${totalFiles}`)
console.log(`- Packed size: ${packedSizeMB.toFixed(2)}MB`)
console.log(`- Unpacked size: ${unpackedSizeMB.toFixed(2)}MB`)
console.log(
`- Compression ratio: ${((1 - packedSizeMB / unpackedSizeMB) * 100).toFixed(1)}%`
)
// Basic sanity checks
expect(totalFiles).toBeGreaterThan(0)
expect(packedSizeMB).toBeGreaterThan(0)
expect(unpackedSizeMB).toBeGreaterThan(0)
expect(packedSizeMB).toBeLessThan(unpackedSizeMB)
})
// Basic sanity checks
expect(totalFiles).toBeGreaterThan(0)
expect(packedSizeMB).toBeGreaterThan(0)
expect(unpackedSizeMB).toBeGreaterThan(0)
expect(packedSizeMB).toBeLessThan(unpackedSizeMB)
})
})

18
tests/s3-storage.test.ts
View file

@ -222,15 +222,25 @@ describe('S3CompatibleStorage', () => {
// Create test verb
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const timestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
const testVerb = {
id: 'test-verb-1',
vector: testVector,
connections: new Map(),
sourceId: 'source-noun-1',
targetId: 'target-noun-1',
type: 'test-relation',
source: 'source-noun-1',
target: 'target-noun-1',
verb: 'test-relation',
weight: 0.75,
metadata: { description: 'Test relation' }
metadata: { description: 'Test relation' },
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
augmentation: 'test-service',
version: '1.0'
}
}
// Save the verb

158
tests/statistics-storage.test.ts Normal file
View file

@ -0,0 +1,158 @@
/**
* Test script for the statistics storage implementation
*
* This script tests:
* 1. Saving statistics data
* 2. Retrieving statistics data
* 3. Verifying that the data is correctly saved and retrieved
* 4. Checking that time-based partitioning works correctly
* 5. Checking that backward compatibility is maintained
*/
// Import required modules
// @ts-expect-error - dotenv doesn't have TypeScript types
import { config } from 'dotenv'
import { setTimeout } from 'timers/promises'
import { describe, it, expect, beforeAll, beforeEach } from 'vitest'
import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3'
import * as process from 'process'
// Define types for statistics data
interface ServiceStatistics {
nounCount: number
verbCount: number
metadataCount: number
}
interface StatisticsData {
nounCount: Record<string, number>
verbCount: Record<string, number>
metadataCount: Record<string, number>
hnswIndexSize: number
lastUpdated: string
}
// Define types for storage configuration
interface S3StorageConfig {
endpoint: string
region: string
bucketName: string
accessKeyId: string
secretAccessKey: string
prefix: string
serviceType?: string
sessionToken?: string
accountId?: string
}
// Load environment variables
config()
// Create test statistics data
const testStatistics: StatisticsData = {
nounCount: { 'test-service': 100, 'another-service': 50 },
verbCount: { 'test-service': 75, 'another-service': 25 },
metadataCount: { 'test-service': 100, 'another-service': 50 },
hnswIndexSize: 150,
lastUpdated: new Date().toISOString()
}
// Test configuration
const storageConfig: S3StorageConfig = {
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
region: process.env.S3_REGION || 'us-east-1',
bucketName: process.env.S3_BUCKET || 'test-bucket',
accessKeyId: process.env.S3_ACCESS_KEY,
secretAccessKey: process.env.S3_SECRET_KEY,
prefix: 'test-statistics/'
}
// Check if required S3 credentials are available
const hasS3Credentials = !!process.env.S3_ACCESS_KEY && !!process.env.S3_SECRET_KEY;
// Use conditional describe to skip all tests if credentials are missing
(hasS3Credentials ? describe : describe.skip)('Statistics Storage', () => {
let storage: any
let s3Client: S3Client
beforeAll(async () => {
if (!hasS3Credentials) {
console.log('Skipping S3 storage tests: S3_ACCESS_KEY or S3_SECRET_KEY environment variables not set')
return
}
try {
// Import S3CompatibleStorage dynamically to avoid issues with dynamic imports
const { S3CompatibleStorage } = await import('../dist/storage/adapters/s3CompatibleStorage')
// Create storage instance
storage = new S3CompatibleStorage(storageConfig)
await storage.init()
// Initialize S3 client for checking files
s3Client = new S3Client({
endpoint: storageConfig.endpoint,
region: storageConfig.region,
credentials: {
accessKeyId: storageConfig.accessKeyId,
secretAccessKey: storageConfig.secretAccessKey
}
})
} catch (error) {
console.log('Error initializing S3 storage:', error)
throw error // Let the test fail with a clear error message
}
})
it('should save statistics data', async () => {
await storage.saveStatistics(testStatistics)
expect(true).toBe(true) // If no error is thrown, the test passes
})
it('should retrieve statistics data after batch update completes', async () => {
// Wait for the batch update to complete (longer than MAX_FLUSH_DELAY_MS)
await setTimeout(35000)
const retrievedStats = await storage.getStatistics()
expect(retrievedStats).not.toBeNull()
// Check that all properties match
expect(JSON.stringify(retrievedStats.nounCount)).toBe(JSON.stringify(testStatistics.nounCount))
expect(JSON.stringify(retrievedStats.verbCount)).toBe(JSON.stringify(testStatistics.verbCount))
expect(JSON.stringify(retrievedStats.metadataCount)).toBe(JSON.stringify(testStatistics.metadataCount))
expect(retrievedStats.hnswIndexSize).toBe(testStatistics.hnswIndexSize)
})
it('should store statistics in time-partitioned files', async () => {
// Get current date in YYYYMMDD format
const now = new Date()
const year = now.getUTCFullYear()
const month = String(now.getUTCMonth() + 1).padStart(2, '0')
const day = String(now.getUTCDate()).padStart(2, '0')
const dateStr = `${year}${month}${day}`
// Check if the file exists in the expected location
const listResponse = await s3Client.send(new ListObjectsV2Command({
Bucket: storageConfig.bucketName,
Prefix: `${storageConfig.prefix}index/statistics_${dateStr}`
}))
expect(listResponse.Contents).toBeDefined()
expect(listResponse.Contents?.length).toBeGreaterThan(0)
})
it('should maintain backward compatibility with legacy statistics file', async () => {
// Check if the legacy file exists
const legacyListResponse = await s3Client.send(new ListObjectsV2Command({
Bucket: storageConfig.bucketName,
Prefix: `${storageConfig.prefix}index/statistics.json`
}))
// This test is informational - the legacy file may not exist if the 10% random update didn't trigger
if (legacyListResponse.Contents && legacyListResponse.Contents.length > 0) {
expect(legacyListResponse.Contents.length).toBeGreaterThan(0)
} else {
console.log('Legacy statistics file not found. This is expected if the 10% random update didn\'t trigger.')
}
})
})

132
tests/statistics.test.ts Normal file
View file

@ -0,0 +1,132 @@
/**
* Statistics Functionality Tests
* Tests the getStatistics function as a consumer would use it
*/
import { describe, it, expect, beforeAll } from 'vitest'
/**
* Helper function to create a 512-dimensional vector for testing
* @param primaryIndex The index to set to 1.0, all other indices will be 0.0
* @returns A 512-dimensional vector with a single 1.0 value at the specified index
*/
function createTestVector(primaryIndex: number = 0): number[] {
const vector = new Array(512).fill(0)
vector[primaryIndex % 512] = 1.0
return vector
}
describe('Brainy Statistics Functionality', () => {
let brainy: any
beforeAll(async () => {
// Load brainy library as a consumer would
brainy = await import('../dist/unified.js')
})
describe('Library Exports', () => {
it('should export getStatistics function at the root level', () => {
expect(brainy.getStatistics).toBeDefined()
expect(typeof brainy.getStatistics).toBe('function')
})
})
describe('getStatistics Functionality', () => {
it('should retrieve statistics from a BrainyData instance', async () => {
// Create a BrainyData instance
const data = new brainy.BrainyData({
metric: 'euclidean'
})
await data.init()
await data.clear() // Clear any existing data
// Add some test data
await data.add(createTestVector(0), { id: 'v1', label: 'x-axis' })
await data.add(createTestVector(1), { id: 'v2', label: 'y-axis' })
await data.add(createTestVector(2), { id: 'v3', label: 'z-axis' })
// Add a verb
await data.addVerb('v1', 'v2', createTestVector(3), { type: 'connected_to' })
// Get statistics using the standalone function
const stats = await brainy.getStatistics(data)
// Verify statistics
expect(stats).toBeDefined()
expect(stats.nounCount).toBe(3)
expect(stats.verbCount).toBe(1)
expect(stats.metadataCount).toBe(3) // Each noun has metadata
expect(stats.hnswIndexSize).toBe(3)
})
it('should throw an error when no instance is provided', async () => {
await expect(brainy.getStatistics()).rejects.toThrow('BrainyData instance must be provided')
})
it('should match the instance method results', async () => {
// Create a BrainyData instance
const data = new brainy.BrainyData({})
await data.init()
// Add some test data
await data.add(createTestVector(5), { id: 'test1' })
// Get statistics using both methods
const instanceStats = await data.getStatistics()
const functionStats = await brainy.getStatistics(data)
// Verify they match
expect(functionStats).toEqual(instanceStats)
})
it('should track statistics by service', async () => {
// Create a BrainyData instance
const data = new brainy.BrainyData({
metric: 'euclidean'
})
await data.init()
await data.clear() // Clear any existing data
// Add data from different services
await data.add(createTestVector(10), { id: 'v1', label: 'service1-item' }, { service: 'service1' })
await data.add(createTestVector(20), { id: 'v2', label: 'service1-item' }, { service: 'service1' })
await data.add(createTestVector(30), { id: 'v3', label: 'service2-item' }, { service: 'service2' })
// Add verbs from different services
await data.addVerb('v1', 'v2', undefined, { type: 'related_to', service: 'service1' })
await data.addVerb('v2', 'v3', undefined, { type: 'related_to', service: 'service2' })
// Get statistics for all services
const allStats = await data.getStatistics()
// Verify total counts
expect(allStats.nounCount).toBe(3)
expect(allStats.verbCount).toBe(2)
expect(allStats.metadataCount).toBe(3)
// Verify service breakdown exists
expect(allStats.serviceBreakdown).toBeDefined()
// Verify service1 statistics
const service1Stats = await data.getStatistics({ service: 'service1' })
expect(service1Stats.nounCount).toBe(2)
expect(service1Stats.verbCount).toBe(1)
expect(service1Stats.metadataCount).toBe(2)
// Verify service2 statistics
const service2Stats = await data.getStatistics({ service: 'service2' })
expect(service2Stats.nounCount).toBe(1)
expect(service2Stats.verbCount).toBe(1)
expect(service2Stats.metadataCount).toBe(1)
// Verify multiple services filter
const combinedStats = await data.getStatistics({ service: ['service1', 'service2'] })
expect(combinedStats.nounCount).toBe(3)
expect(combinedStats.verbCount).toBe(2)
expect(combinedStats.metadataCount).toBe(3)
})
})
})

37
tests/vector-operations.test.ts
View file

@ -1,6 +1,17 @@
import { describe, it, expect } from 'vitest'
import { euclideanDistance } from '../src/utils/distance.js'
/**
* Helper function to create a 512-dimensional vector for testing
* @param primaryIndex The index to set to 1.0, all other indices will be 0.0
* @returns A 512-dimensional vector with a single 1.0 value at the specified index
*/
function createTestVector(primaryIndex: number = 0): number[] {
const vector = new Array(512).fill(0)
vector[primaryIndex % 512] = 1.0
return vector
}
describe('Vector Operations', () => {
it('should load brainy library successfully', async () => {
const brainy = await import('../dist/unified.js')
@ -14,23 +25,21 @@ describe('Vector Operations', () => {
const brainy = await import('../dist/unified.js')
const db = new brainy.BrainyData({
dimensions: 3,
distanceFunction: euclideanDistance
})
expect(db).toBeDefined()
expect(db.dimensions).toBe(3)
expect(db.dimensions).toBe(512)
await db.init()
// If we get here without throwing, initialization was successful
expect(true).toBe(true)
})
it('should handle simple 2D vector operations', async () => {
it('should handle simple vector operations', async () => {
const brainy = await import('../dist/unified.js')
const db = new brainy.BrainyData({
dimensions: 2,
distanceFunction: euclideanDistance
})
@ -38,10 +47,11 @@ describe('Vector Operations', () => {
await db.clear() // Clear any existing data
// Add a simple vector
await db.add([1, 2], { id: 'test' })
const testVector = createTestVector(1)
await db.add(testVector, { id: 'test' })
// Search for the same vector
const results = await db.search([1, 2], 1)
const results = await db.search(testVector, 1)
expect(results).toBeDefined()
expect(results.length).toBeGreaterThan(0)
@ -52,7 +62,6 @@ describe('Vector Operations', () => {
const brainy = await import('../dist/unified.js')
const db = new brainy.BrainyData({
dimensions: 3,
distanceFunction: euclideanDistance
})
@ -60,13 +69,17 @@ describe('Vector Operations', () => {
await db.clear() // Clear any existing data
// Add multiple vectors
await db.add([1, 0, 0], { id: 'vec1', type: 'unit' })
await db.add([0, 1, 0], { id: 'vec2', type: 'unit' })
await db.add([0, 0, 1], { id: 'vec3', type: 'unit' })
await db.add([0.5, 0.5, 0], { id: 'vec4', type: 'mixed' })
await db.add(createTestVector(0), { id: 'vec1', type: 'unit' })
await db.add(createTestVector(1), { id: 'vec2', type: 'unit' })
await db.add(createTestVector(2), { id: 'vec3', type: 'unit' })
// Create a mixed vector with two non-zero elements
const mixedVector = createTestVector(3)
mixedVector[4] = 0.5
await db.add(mixedVector, { id: 'vec4', type: 'mixed' })
// Search for multiple results
const results = await db.search([1, 0, 0], 3)
const results = await db.search(createTestVector(0), 3)
expect(results).toBeDefined()
expect(results.length).toBeGreaterThanOrEqual(1)