soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(core, tests): add standalone getStatistics function and improve storage configuration

Browse source 
- **Core**: Introduced a new `getStatistics` utility function in `statistics.ts` for fetching database statistics at the root level of the library. Enhanced `BrainyData` methods to ensure metadata includes `id` field and refined statistics calculations, excluding verbs from the noun count.
- **Tests**: Added comprehensive test coverage in `statistics.test.ts` for the new utility function, validating proper error handling, statistics accuracy, and consistent results between instance methods and standalone function.
- **Storage Config**: Enabled dynamic support for AWS S3, Cloudflare R2, and Google Cloud Storage in web service configuration, utilizing environment variables for adapter setup. Addressed a race condition in `FileSystemStorage` initialization by deferring path module imports.

**Purpose**: Enhance database analytics by introducing a reusable `getStatistics` function, improve flexibility in storage configuration, and ensure robust testing for reliability and accuracy.
This commit is contained in:
David Snelling 2025-07-23 16:26:59 -07:00
parent b8f10ba39a
commit 2322b53a0c
13 changed files with 346 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

40
web-service-package/dist/server.js vendored
View file

@ -70,15 +70,53 @@ async function initializeBrainy() {
const storageOptions = {
requestPersistentStorage: true
};
// Add AWS S3 configuration if environment variables are present
if (process.env.S3_BUCKET_NAME) {
storageOptions.s3Storage = {
bucketName: process.env.S3_BUCKET_NAME,
region: process.env.S3_REGION || process.env.AWS_REGION || 'us-east-1',
accessKeyId: process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY,
sessionToken: process.env.S3_SESSION_TOKEN || process.env.AWS_SESSION_TOKEN
};
}
// Add Cloudflare R2 configuration if environment variables are present
if (process.env.R2_BUCKET_NAME) {
storageOptions.r2Storage = {
bucketName: process.env.R2_BUCKET_NAME,
accountId: process.env.R2_ACCOUNT_ID,
accessKeyId: process.env.R2_ACCESS_KEY_ID,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
};
}
// Add Google Cloud Storage configuration if environment variables are present
if (process.env.GCS_BUCKET_NAME) {
storageOptions.gcsStorage = {
bucketName: process.env.GCS_BUCKET_NAME,
region: process.env.GCS_REGION,
endpoint: process.env.GCS_ENDPOINT,
accessKeyId: process.env.GCS_ACCESS_KEY_ID,
secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY
};
}
// Check if local storage is forced
if (process.env.FORCE_LOCAL_STORAGE === 'true') {
console.log('Forcing local filesystem storage (FORCE_LOCAL_STORAGE=true)');
storageOptions.forceFileSystemStorage = true;
// Set the data path for local storage
if (DATA_PATH) {
// We'll need to import FileSystemStorage for forced local storage
// We'll need to import FileSystemStorage and path for forced local storage
const { FileSystemStorage } = await import('@soulcraft/brainy');
const path = await import('path');
// Create storage with explicit path handling to avoid race condition
const nounsDir = path.join(DATA_PATH, 'nouns');
const verbsDir = path.join(DATA_PATH, 'verbs');
const metadataDir = path.join(DATA_PATH, 'metadata');
const indexDir = path.join(DATA_PATH, 'index');
// Create a storage instance with pre-computed paths
const storage = new FileSystemStorage(DATA_PATH);
// Initialize the storage adapter before using it
await storage.init();
brainyInstance = new BrainyData({
dimensions: 384, // Default dimensions, can be overridden
storageAdapter: storage,

2
web-service-package/dist/server.js.map vendored
View file

File diff suppressed because one or more lines are too long

10
web-service-package/package-lock.json generated
View file

@ -2654,16 +2654,6 @@
"@tensorflow/tfjs-core": "4.22.0"
}
},
"node_modules/@tensorflow/tfjs-converter": {
"version": "3.21.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz",
"integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==",
"license": "Apache-2.0",
"peer": true,
"peerDependencies": {
"@tensorflow/tfjs-core": "3.21.0"
}
},
"node_modules/@tensorflow/tfjs-core": {
"version": "4.22.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.22.0.tgz",

46
web-service-package/src/server.ts
View file

@ -83,16 +83,60 @@ async function initializeBrainy(): Promise<BrainyData> {
requestPersistentStorage: true
}
// Add AWS S3 configuration if environment variables are present
if (process.env.S3_BUCKET_NAME) {
storageOptions.s3Storage = {
bucketName: process.env.S3_BUCKET_NAME,
region: process.env.S3_REGION || process.env.AWS_REGION || 'us-east-1',
accessKeyId: process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY,
sessionToken: process.env.S3_SESSION_TOKEN || process.env.AWS_SESSION_TOKEN
}
}
// Add Cloudflare R2 configuration if environment variables are present
if (process.env.R2_BUCKET_NAME) {
storageOptions.r2Storage = {
bucketName: process.env.R2_BUCKET_NAME,
accountId: process.env.R2_ACCOUNT_ID,
accessKeyId: process.env.R2_ACCESS_KEY_ID,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
}
}
// Add Google Cloud Storage configuration if environment variables are present
if (process.env.GCS_BUCKET_NAME) {
storageOptions.gcsStorage = {
bucketName: process.env.GCS_BUCKET_NAME,
region: process.env.GCS_REGION,
endpoint: process.env.GCS_ENDPOINT,
accessKeyId: process.env.GCS_ACCESS_KEY_ID,
secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY
}
}
// Check if local storage is forced
if (process.env.FORCE_LOCAL_STORAGE === 'true') {
console.log('Forcing local filesystem storage (FORCE_LOCAL_STORAGE=true)')
storageOptions.forceFileSystemStorage = true
// Set the data path for local storage
if (DATA_PATH) {
// We'll need to import FileSystemStorage for forced local storage
// We'll need to import FileSystemStorage and path for forced local storage
const { FileSystemStorage } = await import('@soulcraft/brainy')
const path = await import('path')
// Create storage with explicit path handling to avoid race condition
const nounsDir = path.join(DATA_PATH, 'nouns')
const verbsDir = path.join(DATA_PATH, 'verbs')
const metadataDir = path.join(DATA_PATH, 'metadata')
const indexDir = path.join(DATA_PATH, 'index')
// Create a storage instance with pre-computed paths
const storage = new FileSystemStorage(DATA_PATH)
// Initialize the storage adapter before using it
await storage.init()
brainyInstance = new BrainyData({
dimensions: 384, // Default dimensions, can be overridden
storageAdapter: storage,

32
web-service-package/tests/cloud-storage.test.ts
View file

@ -37,25 +37,49 @@ describe('Brainy Web Service Cloud Storage Integration', () => {
let errorOutput = ''
serverProcess.stdout?.on('data', (data) => {
output += data.toString()
const dataStr = data.toString()
output += dataStr
// Check for error messages in stdout that indicate initialization failure
if (dataStr.includes('Failed to initialize') || dataStr.includes('Error:')) {
console.log(` Server stdout error: ${dataStr.trim()}`)
reject(new Error(`Server initialization failed: ${dataStr.trim()}`))
return
}
if (output.includes('Server running on')) {
resolve()
}
})
serverProcess.stderr?.on('data', (data) => {
errorOutput += data.toString()
console.log(` Server stderr: ${data.toString().trim()}`)
const dataStr = data.toString()
errorOutput += dataStr
console.log(` Server stderr: ${dataStr.trim()}`)
// Check for error messages in stderr that indicate initialization failure
if (dataStr.includes('Failed to initialize') || dataStr.includes('Error:')) {
reject(new Error(`Server initialization failed: ${dataStr.trim()}`))
}
})
serverProcess.on('error', (error) => {
reject(new Error(`Failed to start server: ${error.message}`))
})
serverProcess.on('exit', (code) => {
serverProcess.on('exit', (code, signal) => {
// If the process exited with a non-zero code, it's an error
if (code !== 0 && code !== null) {
reject(new Error(`Server exited with code ${code}. Error: ${errorOutput}`))
}
// If the process was terminated by a signal and we have error output, it's an error
else if (signal && errorOutput) {
reject(new Error(`Server terminated by signal ${signal}. Error: ${errorOutput}`))
}
// If we have error output but no code or signal, it's still an error
else if (errorOutput && errorOutput.includes('Error:')) {
reject(new Error(`Server exited with error: ${errorOutput}`))
}
})
// Timeout after 10 seconds