**feat(core): enhance logging, storage options, and testing coverage**

- **Core Improvements**:
  - Refactored logging functions into a unified `logger` method for consistent output across the library.
  - Enabled the `forceMemoryStorage` option in `BrainyData` initialization for improved storage flexibility in tests and specific use cases.

- **TensorFlow.js and Environment Updates**:
  - Clarified the dependency structure in `README.md` to emphasize bundled dependencies and remove legacy peer dependency instructions.
  - Simplified and reformatted environment detection logic for better maintainability and readability.

- **Testing Enhancements**:
  - Added `tests/package-size-limit.test.ts` to monitor and validate npm package size against defined thresholds.
  - Updated `tests/environment.node.test.ts` and core tests to leverage `forceMemoryStorage` for better test setup standardization.
  - Improved test isolation with expanded `globalThis` utility definitions and cleanup logic.

- **Documentation**:
  - Added detailed best practices for debugging and organizing tests in `DEVELOPERS.md`.
  - Removed outdated installation hints from `package.json` and streamlined scripts by including `test:size` for package size validation.

**Purpose**: These changes unify core logging mechanisms, expand configurability of storage options, and improve testing reliability and coverage. Documentation and clarity are enhanced to align with updated functionality and best practices.
This commit is contained in:
David Snelling 2025-07-17 10:00:28 -07:00
parent 99ed56c1f1
commit e6514b10cf
8 changed files with 238 additions and 39 deletions

View file

@ -182,7 +182,10 @@ describe('Brainy Core Functionality', () => {
const data = new brainy.BrainyData({
embeddingFunction,
dimensions: 512, // Universal Sentence Encoder produces 512-dimensional vectors
metric: 'cosine'
metric: 'cosine',
storage: {
forceMemoryStorage: true
}
})
await data.init()

View file

@ -54,7 +54,10 @@ describe('Brainy in Node.js Environment', () => {
}
const db = new brainy.BrainyData({
dimensions: 3,
metric: 'euclidean'
metric: 'euclidean',
storage: {
forceMemoryStorage: true
}
})
await db.init()
@ -83,7 +86,10 @@ describe('Brainy in Node.js Environment', () => {
}
const db = new brainy.BrainyData({
embeddingFunction: brainy.createEmbeddingFunction(),
metric: 'cosine'
metric: 'cosine',
storage: {
forceMemoryStorage: true
}
})
await db.init()
@ -109,7 +115,10 @@ describe('Brainy in Node.js Environment', () => {
}
const db = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean'
metric: 'euclidean',
storage: {
forceMemoryStorage: true
}
})
await db.init()
@ -155,7 +164,10 @@ describe('Brainy in Node.js Environment', () => {
}
const db = new brainy.BrainyData({
dimensions: 2,
metric: 'euclidean'
metric: 'euclidean',
storage: {
forceMemoryStorage: true
}
})
await db.init()

View file

@ -0,0 +1,146 @@
/**
* Package Size Limit Tests
* Tests the predicted npm package size to ensure it stays within acceptable limits
*/
import { describe, expect, it } from 'vitest'
import { execSync } from 'child_process'
const CURRENT_UNPACKED_SIZE_MB = 10.4
const CURRENT_PACKED_SIZE_MB = 1.9
const ALLOWED_SIZE_INCREASE_PERCENTAGE = 5 // 5% increase threshold
/**
* Parses npm pack --dry-run output to extract package size information
*/
function parseNpmPackOutput(output: string): {
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
} {
const packageSizeMatch = output.match(
/npm notice package size:\s*([\d.]+)\s*([KMGT]?B)/
)
const unpackedSizeMatch = output.match(
/npm notice unpacked size:\s*([\d.]+)\s*([KMGT]?B)/
)
const totalFilesMatch = output.match(/npm notice total files:\s*(\d+)/)
const convertToMB = (size: number, unit: string): number => {
switch (unit) {
case 'B':
return size / (1024 * 1024)
case 'KB':
return size / 1024
case 'MB':
return size
case 'GB':
return size * 1024
default:
return size / (1024 * 1024) // assume bytes
}
}
const packedSizeMB = packageSizeMatch
? convertToMB(parseFloat(packageSizeMatch[1]), packageSizeMatch[2])
: 0
const unpackedSizeMB = unpackedSizeMatch
? convertToMB(parseFloat(unpackedSizeMatch[1]), unpackedSizeMatch[2])
: 0
const totalFiles = totalFilesMatch ? parseInt(totalFilesMatch[1], 10) : 0
return { packedSizeMB, unpackedSizeMB, totalFiles }
}
/**
* Cached npm package size result to avoid multiple expensive npm pack calls
*/
let cachedPackageSize: {
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
} | null = null
/**
* Gets the predicted npm package size using npm pack --dry-run
* Results are cached to avoid multiple expensive executions
*/
async function getNpmPackageSize(): Promise<{
packedSizeMB: number
unpackedSizeMB: number
totalFiles: number
}> {
// Return cached result if available
if (cachedPackageSize) {
return cachedPackageSize
}
try {
// Use 2>&1 to capture both stdout and stderr in one command
const output = execSync('npm pack --dry-run 2>&1', {
encoding: 'utf8',
cwd: process.cwd(),
timeout: 45000 // 45 second timeout to prevent hanging
})
const result = parseNpmPackOutput(output)
// Cache the result for subsequent calls
cachedPackageSize = result
return result
} catch (error) {
throw new Error(`Failed to get npm package size: ${error}`)
}
}
describe('Package Size Limits', () => {
it('should not exceed unpacked size threshold for npm package', async () => {
const { unpackedSizeMB } = await getNpmPackageSize()
const maxAllowedSize =
CURRENT_UNPACKED_SIZE_MB * (1 + ALLOWED_SIZE_INCREASE_PERCENTAGE / 100)
console.log(`Current unpacked package size: ${unpackedSizeMB.toFixed(2)}MB`)
console.log(`Maximum allowed unpacked size: ${maxAllowedSize.toFixed(2)}MB`)
expect(
unpackedSizeMB,
`Unpacked package size (${unpackedSizeMB.toFixed(2)}MB) exceeds maximum allowed size (${maxAllowedSize.toFixed(2)}MB)`
).toBeLessThanOrEqual(maxAllowedSize)
})
it('should not exceed packed size threshold for npm package', async () => {
const { packedSizeMB } = await getNpmPackageSize()
const maxAllowedSize =
CURRENT_PACKED_SIZE_MB * (1 + ALLOWED_SIZE_INCREASE_PERCENTAGE / 100)
console.log(`Current packed package size: ${packedSizeMB.toFixed(2)}MB`)
console.log(`Maximum allowed packed size: ${maxAllowedSize.toFixed(2)}MB`)
expect(
packedSizeMB,
`Packed package size (${packedSizeMB.toFixed(2)}MB) exceeds maximum allowed size (${maxAllowedSize.toFixed(2)}MB)`
).toBeLessThanOrEqual(maxAllowedSize)
})
it('should report package composition details', async () => {
const { packedSizeMB, unpackedSizeMB, totalFiles } =
await getNpmPackageSize()
console.log(`\nPackage composition:`)
console.log(`- Total files: ${totalFiles}`)
console.log(`- Packed size: ${packedSizeMB.toFixed(2)}MB`)
console.log(`- Unpacked size: ${unpackedSizeMB.toFixed(2)}MB`)
console.log(
`- Compression ratio: ${((1 - packedSizeMB / unpackedSizeMB) * 100).toFixed(1)}%`
)
// Basic sanity checks
expect(totalFiles).toBeGreaterThan(0)
expect(packedSizeMB).toBeGreaterThan(0)
expect(unpackedSizeMB).toBeGreaterThan(0)
expect(packedSizeMB).toBeLessThan(unpackedSizeMB)
})
})

View file

@ -5,27 +5,39 @@
import { beforeEach } from 'vitest'
// Extend global type definitions
// Define the test utilities type for reuse
type TestUtilsType = {
createTestVector: (dimensions: number) => number[]
timeout: number
}
// Extend global type definitions for both global and globalThis
declare global {
let testUtils:
| {
createTestVector: (dimensions: number) => number[]
timeout: number
}
| undefined
let testUtils: TestUtilsType | undefined
let __ENV__: any
}
// Explicitly declare globalThis interface to ensure TypeScript recognizes these properties
declare global {
interface globalThis {
testUtils?: TestUtilsType | undefined
__ENV__?: any
}
}
// Clean up between tests
beforeEach(() => {
// Clear any global state that might interfere with tests
if (typeof globalThis !== 'undefined' && globalThis.__ENV__) {
delete globalThis.__ENV__
}
if (typeof global !== 'undefined' && global.__ENV__) {
delete global.__ENV__
}
})
// Add simple test utilities
global.testUtils = {
// Add simple test utilities to both global and globalThis for compatibility
const testUtilsObject = {
// Create a simple test vector with predictable values
createTestVector: (dimensions: number): number[] => {
return Array.from({ length: dimensions }, (_, i) => (i + 1) / dimensions)
@ -34,3 +46,6 @@ global.testUtils = {
// Standard timeout for async operations
timeout: 30000
}
global.testUtils = testUtilsObject
globalThis.testUtils = testUtilsObject