**chore: remove unused CLI and utility files**
- **Codebase Cleanup**:
- Removed `cli.ts` and `textEncoding.ts` from `cli-package/src`:
- Deleted outdated CLI logic and text encoding utilities no longer actively used or maintained.
- **Purpose**:
- Simplify the repository by eliminating unused and redundant code, reducing maintenance overhead.
This commit is contained in:
parent
b846342681
commit
a6ac8b791e
27 changed files with 6 additions and 13665 deletions
|
|
@ -1,54 +0,0 @@
|
||||||
# @soulcraft/brainy-cli
|
|
||||||
|
|
||||||
Command-line interface for the [Brainy vector graph database](https://github.com/soulcraft-research/brainy).
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install globally
|
|
||||||
npm install -g @soulcraft/brainy-cli
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
Once installed, you can use the `brainy` command from anywhere:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Show help
|
|
||||||
brainy --help
|
|
||||||
|
|
||||||
# Initialize a new database
|
|
||||||
brainy init
|
|
||||||
|
|
||||||
# Add data
|
|
||||||
brainy add "Cats are independent pets" '{"noun":"Thing","category":"animal"}'
|
|
||||||
|
|
||||||
# Search
|
|
||||||
brainy search "feline pets" --limit 5
|
|
||||||
|
|
||||||
# Add relationships
|
|
||||||
brainy addVerb id1 id2 RelatedTo '{"description":"Both are pets"}'
|
|
||||||
|
|
||||||
# Visualize the graph
|
|
||||||
brainy visualize
|
|
||||||
brainy visualize --root <id> --depth 3
|
|
||||||
|
|
||||||
# Generate random test data
|
|
||||||
brainy generate-random-graph --noun-count 20 --verb-count 30 --clear
|
|
||||||
```
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Full access to all Brainy database functionality from the command line
|
|
||||||
- Autocomplete support for commands and options
|
|
||||||
- Visualization of graph data
|
|
||||||
- Import/export capabilities
|
|
||||||
- Augmentation pipeline testing
|
|
||||||
|
|
||||||
## Requirements
|
|
||||||
|
|
||||||
- Node.js >= 24.4.0
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
MIT
|
|
||||||
|
|
@ -1,97 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Brainy CLI Wrapper
|
|
||||||
* This script patches the global object to fix TextEncoder issues before loading the CLI
|
|
||||||
*/
|
|
||||||
|
|
||||||
console.log('Brainy running in Node.js environment')
|
|
||||||
|
|
||||||
// Define a custom PlatformNode class that doesn't rely on this.util.TextEncoder
|
|
||||||
if (
|
|
||||||
typeof global !== 'undefined' &&
|
|
||||||
typeof process !== 'undefined' &&
|
|
||||||
process.versions &&
|
|
||||||
process.versions.node
|
|
||||||
) {
|
|
||||||
try {
|
|
||||||
// Define a PlatformNode class that uses the global TextEncoder/TextDecoder directly
|
|
||||||
class PlatformNode {
|
|
||||||
constructor() {
|
|
||||||
// Create a util object with necessary methods
|
|
||||||
this.util = {
|
|
||||||
// Add isFloat32Array and isTypedArray directly to util
|
|
||||||
isFloat32Array: (arr) => {
|
|
||||||
return !!(
|
|
||||||
arr instanceof Float32Array ||
|
|
||||||
(arr &&
|
|
||||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
||||||
)
|
|
||||||
},
|
|
||||||
isTypedArray: (arr) => {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
},
|
|
||||||
// Instead of using constructors directly, create a utility object with constructors
|
|
||||||
TextEncoder,
|
|
||||||
TextDecoder
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize TextEncoder/TextDecoder instances
|
|
||||||
this.textEncoder = new TextEncoder()
|
|
||||||
this.textDecoder = new TextDecoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define isFloat32Array directly on the instance
|
|
||||||
isFloat32Array(arr) {
|
|
||||||
return !!(
|
|
||||||
arr instanceof Float32Array ||
|
|
||||||
(arr &&
|
|
||||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define isTypedArray directly on the instance
|
|
||||||
isTypedArray(arr) {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assign the PlatformNode class to the global object
|
|
||||||
global.PlatformNode = PlatformNode
|
|
||||||
|
|
||||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
|
||||||
global.platformNode = new PlatformNode()
|
|
||||||
|
|
||||||
// Ensure global.util exists and has the necessary methods
|
|
||||||
// This is needed because TensorFlow.js might look for these methods in global.util
|
|
||||||
if (!global.util) {
|
|
||||||
global.util = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add isFloat32Array method if it doesn't exist
|
|
||||||
if (!global.util.isFloat32Array) {
|
|
||||||
global.util.isFloat32Array = (arr) => {
|
|
||||||
return !!(
|
|
||||||
arr instanceof Float32Array ||
|
|
||||||
(arr &&
|
|
||||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add isTypedArray method if it doesn't exist
|
|
||||||
if (!global.util.isTypedArray) {
|
|
||||||
global.util.isTypedArray = (arr) => {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Failed to define global PlatformNode class:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now load and run the actual CLI
|
|
||||||
import('./dist/cli.js').catch((err) => {
|
|
||||||
console.error('Error loading CLI:', err)
|
|
||||||
process.exit(1)
|
|
||||||
})
|
|
||||||
|
|
@ -1,113 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CLI Wrapper Script for @soulcraft/brainy-cli
|
|
||||||
*
|
|
||||||
* This script serves as a wrapper for the Brainy CLI, ensuring that command-line arguments
|
|
||||||
* are properly passed to the CLI when invoked through the globally installed package.
|
|
||||||
*/
|
|
||||||
|
|
||||||
// CRITICAL: Apply TensorFlow.js environment patch before importing any other modules
|
|
||||||
// This prevents the "TextEncoder is not a constructor" error in Node.js environments
|
|
||||||
// by ensuring the global.PlatformNode class is defined before TensorFlow.js loads
|
|
||||||
function applyTensorFlowPatch() {
|
|
||||||
try {
|
|
||||||
// Define a custom Platform class that works in Node.js environments
|
|
||||||
class Platform {
|
|
||||||
constructor() {
|
|
||||||
// Create a util object with necessary methods and constructors
|
|
||||||
this.util = {
|
|
||||||
// Use native TextEncoder and TextDecoder constructors
|
|
||||||
TextEncoder: global.TextEncoder || TextEncoder,
|
|
||||||
TextDecoder: global.TextDecoder || TextDecoder
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize using native constructors directly
|
|
||||||
this.textEncoder = new TextEncoder()
|
|
||||||
this.textDecoder = new TextDecoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define isTypedArray directly on the instance
|
|
||||||
isTypedArray(arr) {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assign the Platform class to the global object as PlatformNode
|
|
||||||
global.PlatformNode = Platform
|
|
||||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
|
||||||
global.platformNode = new Platform()
|
|
||||||
|
|
||||||
console.log('Applied TensorFlow.js platform patch in CLI wrapper')
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Failed to apply TensorFlow.js platform patch:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply the patch immediately
|
|
||||||
applyTensorFlowPatch()
|
|
||||||
|
|
||||||
import { spawn } from 'child_process'
|
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
import { dirname, join } from 'path'
|
|
||||||
import fs from 'fs'
|
|
||||||
|
|
||||||
// Node.js v24+ compatibility patches are now applied above,
|
|
||||||
// before any imports, to ensure TensorFlow.js can correctly
|
|
||||||
// detect and use the TextEncoder/TextDecoder in the environment.
|
|
||||||
|
|
||||||
// Get the directory of the current module
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
|
||||||
const __dirname = dirname(__filename)
|
|
||||||
|
|
||||||
// Find the main package
|
|
||||||
const mainPackagePath = join(__dirname, 'node_modules', '@soulcraft', 'brainy')
|
|
||||||
|
|
||||||
// Path to the actual CLI script in this package
|
|
||||||
const cliPath = join(__dirname, 'dist', 'cli.js')
|
|
||||||
|
|
||||||
// Check if the CLI script exists
|
|
||||||
if (!fs.existsSync(cliPath)) {
|
|
||||||
console.error(`Error: CLI script not found at ${cliPath}`)
|
|
||||||
console.error(
|
|
||||||
'This is likely because the CLI was not built during package installation.'
|
|
||||||
)
|
|
||||||
console.error('Please reinstall the package with:')
|
|
||||||
console.error('npm uninstall -g @soulcraft/brainy-cli')
|
|
||||||
console.error('npm install -g @soulcraft/brainy-cli')
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Special handling for version flags
|
|
||||||
if (process.argv.includes('--version') || process.argv.includes('-V')) {
|
|
||||||
// Read version directly from package.json to ensure it's always correct
|
|
||||||
try {
|
|
||||||
const packageJsonPath = join(__dirname, 'package.json')
|
|
||||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
|
|
||||||
console.log(packageJson.version)
|
|
||||||
process.exit(0)
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error loading version information:', error.message)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Forward all arguments to the CLI script
|
|
||||||
const args = process.argv.slice(2)
|
|
||||||
|
|
||||||
// Check if npm is passing --force flag
|
|
||||||
// When npm runs with --force, it sets the npm_config_force environment variable
|
|
||||||
if (
|
|
||||||
process.env.npm_config_force === 'true' &&
|
|
||||||
args.includes('clear') &&
|
|
||||||
!args.includes('--force') &&
|
|
||||||
!args.includes('-f')
|
|
||||||
) {
|
|
||||||
args.push('--force')
|
|
||||||
}
|
|
||||||
|
|
||||||
const cli = spawn('node', [cliPath, ...args], { stdio: 'inherit' })
|
|
||||||
|
|
||||||
cli.on('close', (code) => {
|
|
||||||
process.exit(code)
|
|
||||||
})
|
|
||||||
3437
cli-package/package-lock.json
generated
3437
cli-package/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,67 +0,0 @@
|
||||||
{
|
|
||||||
"name": "@soulcraft/brainy-cli",
|
|
||||||
"version": "0.19.0",
|
|
||||||
"description": "Command-line interface for the Brainy vector graph database",
|
|
||||||
"type": "module",
|
|
||||||
"bin": {
|
|
||||||
"brainy": "cli-wrapper.js"
|
|
||||||
},
|
|
||||||
"files": [
|
|
||||||
"cli-wrapper.js",
|
|
||||||
"README.md",
|
|
||||||
"dist/cli.js",
|
|
||||||
"dist/cli.js.map"
|
|
||||||
],
|
|
||||||
"scripts": {
|
|
||||||
"build": "rollup -c rollup.config.js",
|
|
||||||
"prepare": "npm run build",
|
|
||||||
"postinstall": "node cli-wrapper.js --version",
|
|
||||||
"_version": "echo 'Version updated in package.json'",
|
|
||||||
"_version:patch": "npm version patch",
|
|
||||||
"_version:minor": "npm version minor",
|
|
||||||
"_version:major": "npm version major",
|
|
||||||
"_deploy": "npm run build && npm publish",
|
|
||||||
"_dry-run": "npm pack --dry-run"
|
|
||||||
},
|
|
||||||
"keywords": [
|
|
||||||
"vector-database",
|
|
||||||
"hnsw",
|
|
||||||
"cli",
|
|
||||||
"browser",
|
|
||||||
"container",
|
|
||||||
"graph-database"
|
|
||||||
],
|
|
||||||
"author": "David Snelling (david@soulcraft.com)",
|
|
||||||
"license": "MIT",
|
|
||||||
"private": false,
|
|
||||||
"publishConfig": {
|
|
||||||
"access": "public"
|
|
||||||
},
|
|
||||||
"homepage": "https://github.com/soulcraft-research/brainy",
|
|
||||||
"bugs": {
|
|
||||||
"url": "https://github.com/soulcraft-research/brainy/issues"
|
|
||||||
},
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "git+https://github.com/soulcraft-research/brainy.git"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"@soulcraft/brainy": "^0.24.0",
|
|
||||||
"commander": "^14.0.0",
|
|
||||||
"omelette": "^0.4.17"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@rollup/plugin-commonjs": "^25.0.7",
|
|
||||||
"@rollup/plugin-json": "^6.1.0",
|
|
||||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
|
||||||
"@rollup/plugin-typescript": "^11.1.6",
|
|
||||||
"@types/node": "^20.11.30",
|
|
||||||
"@types/omelette": "^0.4.5",
|
|
||||||
"rollup": "^4.13.0",
|
|
||||||
"rollup-plugin-terser": "^7.0.2",
|
|
||||||
"typescript": "^5.4.5"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=24.4.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
import typescript from '@rollup/plugin-typescript'
|
|
||||||
import resolve from '@rollup/plugin-node-resolve'
|
|
||||||
import commonjs from '@rollup/plugin-commonjs'
|
|
||||||
import json from '@rollup/plugin-json'
|
|
||||||
import { terser } from 'rollup-plugin-terser'
|
|
||||||
|
|
||||||
// CLI configuration
|
|
||||||
export default {
|
|
||||||
input: 'src/cli.ts',
|
|
||||||
context: 'this', // Preserve 'this' context to fix TensorFlow.js issue
|
|
||||||
output: {
|
|
||||||
dir: 'dist',
|
|
||||||
entryFileNames: 'cli.js',
|
|
||||||
format: 'es',
|
|
||||||
sourcemap: true,
|
|
||||||
inlineDynamicImports: true
|
|
||||||
},
|
|
||||||
plugins: [
|
|
||||||
resolve({
|
|
||||||
browser: false,
|
|
||||||
preferBuiltins: true
|
|
||||||
}),
|
|
||||||
commonjs({
|
|
||||||
transformMixedEsModules: true
|
|
||||||
}),
|
|
||||||
json(),
|
|
||||||
typescript({
|
|
||||||
tsconfig: './tsconfig.json',
|
|
||||||
declaration: false,
|
|
||||||
declarationMap: false
|
|
||||||
})
|
|
||||||
],
|
|
||||||
external: [
|
|
||||||
// External dependencies that should not be bundled
|
|
||||||
'@soulcraft/brainy',
|
|
||||||
'commander',
|
|
||||||
'omelette',
|
|
||||||
'fs',
|
|
||||||
'path',
|
|
||||||
'url',
|
|
||||||
'child_process',
|
|
||||||
'worker_threads'
|
|
||||||
]
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,12 +0,0 @@
|
||||||
/**
|
|
||||||
* This file is imported for its side effects to patch the environment
|
|
||||||
* for TensorFlow.js before any other library code runs.
|
|
||||||
*
|
|
||||||
* It ensures that by the time TensorFlow.js is imported by any other
|
|
||||||
* module, the necessary compatibility fixes for the current Node.js
|
|
||||||
* environment are already in place.
|
|
||||||
*/
|
|
||||||
import { applyTensorFlowPatch } from './utils/textEncoding.js'
|
|
||||||
|
|
||||||
// Apply the TensorFlow.js platform patch if needed
|
|
||||||
applyTensorFlowPatch()
|
|
||||||
|
|
@ -1,98 +0,0 @@
|
||||||
/**
|
|
||||||
* Unified Text Encoding Utilities
|
|
||||||
*
|
|
||||||
* This module provides a consistent way to handle text encoding/decoding across all environments
|
|
||||||
* using the native TextEncoder/TextDecoder APIs.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a text encoder that works in the current environment
|
|
||||||
* @returns A TextEncoder instance
|
|
||||||
*/
|
|
||||||
export function getTextEncoder(): TextEncoder {
|
|
||||||
return new TextEncoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a text decoder that works in the current environment
|
|
||||||
* @returns A TextDecoder instance
|
|
||||||
*/
|
|
||||||
export function getTextDecoder(): TextDecoder {
|
|
||||||
return new TextDecoder()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply the TensorFlow.js platform patch if needed
|
|
||||||
* This function patches the global object to provide a PlatformNode class
|
|
||||||
* that uses native TextEncoder/TextDecoder
|
|
||||||
*/
|
|
||||||
export function applyTensorFlowPatch(): void {
|
|
||||||
try {
|
|
||||||
// Define a custom Platform class that works in both Node.js and browser environments
|
|
||||||
class Platform {
|
|
||||||
util: any
|
|
||||||
textEncoder: TextEncoder
|
|
||||||
textDecoder: TextDecoder
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
// Create a util object with necessary methods and constructors
|
|
||||||
// Store the actual constructor functions, not just references
|
|
||||||
const TextEncoderConstructor = globalThis.TextEncoder || TextEncoder
|
|
||||||
const TextDecoderConstructor = globalThis.TextDecoder || TextDecoder
|
|
||||||
|
|
||||||
this.util = {
|
|
||||||
// Use native TextEncoder and TextDecoder constructors
|
|
||||||
TextEncoder: TextEncoderConstructor,
|
|
||||||
TextDecoder: TextDecoderConstructor
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize using native constructors directly
|
|
||||||
this.textEncoder = new TextEncoderConstructor()
|
|
||||||
this.textDecoder = new TextDecoderConstructor()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define isFloat32Array directly on the instance
|
|
||||||
isFloat32Array(arr: any) {
|
|
||||||
return !!(
|
|
||||||
arr instanceof Float32Array ||
|
|
||||||
(arr &&
|
|
||||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define isTypedArray directly on the instance
|
|
||||||
isTypedArray(arr: any) {
|
|
||||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the global object in a way that works in both Node.js and browser
|
|
||||||
const globalObj =
|
|
||||||
typeof global !== 'undefined'
|
|
||||||
? global
|
|
||||||
: typeof window !== 'undefined'
|
|
||||||
? window
|
|
||||||
: typeof self !== 'undefined'
|
|
||||||
? self
|
|
||||||
: {}
|
|
||||||
|
|
||||||
// Only apply in Node.js environment
|
|
||||||
if (
|
|
||||||
typeof process !== 'undefined' &&
|
|
||||||
process.versions &&
|
|
||||||
process.versions.node
|
|
||||||
) {
|
|
||||||
// Assign the Platform class to the global object as PlatformNode for Node.js
|
|
||||||
;(globalObj as any).PlatformNode = Platform
|
|
||||||
// Also create an instance and assign it to global.platformNode (lowercase p)
|
|
||||||
;(globalObj as any).platformNode = new Platform()
|
|
||||||
} else if (typeof window !== 'undefined' || typeof self !== 'undefined') {
|
|
||||||
// In browser environments, we might need to provide similar functionality
|
|
||||||
// but we'll use a different name to avoid conflicts
|
|
||||||
;(globalObj as any).PlatformBrowser = Platform
|
|
||||||
;(globalObj as any).platformBrowser = new Platform()
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('Failed to apply TensorFlow.js platform patch:', error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
"target": "ES2020",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleResolution": "node",
|
|
||||||
"esModuleInterop": true,
|
|
||||||
"strict": true,
|
|
||||||
"noImplicitAny": false,
|
|
||||||
"outDir": "dist",
|
|
||||||
"declaration": true,
|
|
||||||
"sourceMap": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"paths": {
|
|
||||||
"@soulcraft/brainy": ["../dist/unified.d.ts"]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"include": ["src/**/*", "types.d.ts"],
|
|
||||||
"exclude": ["node_modules", "dist"]
|
|
||||||
}
|
|
||||||
98
cli-package/types.d.ts
vendored
98
cli-package/types.d.ts
vendored
|
|
@ -1,98 +0,0 @@
|
||||||
// Type declarations for @soulcraft/brainy
|
|
||||||
declare module '@soulcraft/brainy' {
|
|
||||||
// Core types
|
|
||||||
export class BrainyData {
|
|
||||||
constructor(config?: any)
|
|
||||||
|
|
||||||
init(): Promise<void>
|
|
||||||
|
|
||||||
add(text: string, metadata?: any): Promise<string>
|
|
||||||
|
|
||||||
get(id: string): Promise<any>
|
|
||||||
|
|
||||||
delete(id: string): Promise<void>
|
|
||||||
|
|
||||||
search(query: string, limit?: number, options?: any): Promise<any[]>
|
|
||||||
|
|
||||||
searchText(query: string, limit?: number, options?: any): Promise<any[]>
|
|
||||||
|
|
||||||
embed(data: string | string[]): Promise<number[]>
|
|
||||||
|
|
||||||
calculateSimilarity(
|
|
||||||
a: number[] | string | string[],
|
|
||||||
b: number[] | string | string[],
|
|
||||||
options?: { forceEmbed?: boolean, distanceFunction?: any }
|
|
||||||
): Promise<number>
|
|
||||||
|
|
||||||
addVerb(
|
|
||||||
sourceId: string,
|
|
||||||
targetId: string,
|
|
||||||
text?: string,
|
|
||||||
options?: any
|
|
||||||
): Promise<string>
|
|
||||||
|
|
||||||
getVerbsBySource(sourceId: string): Promise<any[]>
|
|
||||||
|
|
||||||
getVerbsByTarget(targetId: string): Promise<any[]>
|
|
||||||
|
|
||||||
status(): Promise<any>
|
|
||||||
|
|
||||||
clear(): Promise<void>
|
|
||||||
|
|
||||||
backup(): Promise<any>
|
|
||||||
|
|
||||||
restore(data: any, options?: any): Promise<any>
|
|
||||||
|
|
||||||
importSparseData(data: any, options?: any): Promise<any>
|
|
||||||
|
|
||||||
generateRandomGraph(options?: any): Promise<any>
|
|
||||||
}
|
|
||||||
|
|
||||||
export class FileSystemStorage {
|
|
||||||
constructor(dataDir: string)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pipelines
|
|
||||||
export const sequentialPipeline: any
|
|
||||||
export const augmentationPipeline: any
|
|
||||||
|
|
||||||
// Enums
|
|
||||||
export enum NounType {
|
|
||||||
Person = 'Person',
|
|
||||||
Place = 'Place',
|
|
||||||
Thing = 'Thing',
|
|
||||||
Event = 'Event',
|
|
||||||
Concept = 'Concept',
|
|
||||||
Content = 'Content'
|
|
||||||
}
|
|
||||||
|
|
||||||
export enum VerbType {
|
|
||||||
RelatedTo = 'RelatedTo',
|
|
||||||
PartOf = 'PartOf',
|
|
||||||
HasA = 'HasA',
|
|
||||||
UsedFor = 'UsedFor',
|
|
||||||
CapableOf = 'CapableOf',
|
|
||||||
AtLocation = 'AtLocation',
|
|
||||||
Causes = 'Causes',
|
|
||||||
HasProperty = 'HasProperty',
|
|
||||||
Owns = 'Owns',
|
|
||||||
CreatedBy = 'CreatedBy'
|
|
||||||
}
|
|
||||||
|
|
||||||
export enum ExecutionMode {
|
|
||||||
SEQUENTIAL = 'sequential',
|
|
||||||
PARALLEL = 'parallel',
|
|
||||||
THREADED = 'threaded'
|
|
||||||
}
|
|
||||||
|
|
||||||
export enum AugmentationType {
|
|
||||||
SENSE = 'sense',
|
|
||||||
MEMORY = 'memory',
|
|
||||||
COGNITION = 'cognition',
|
|
||||||
CONDUIT = 'conduit',
|
|
||||||
ACTIVATION = 'activation',
|
|
||||||
PERCEPTION = 'perception',
|
|
||||||
DIALOG = 'dialog',
|
|
||||||
WEBSOCKET = 'websocket'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -1,75 +1,12 @@
|
||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Script to build and publish both the main package and CLI package
|
* DEPRECATED: Script to build and publish both the main package and CLI package
|
||||||
*
|
|
||||||
* This script:
|
|
||||||
* 1. Ensures versions are in sync by running generate-version.js
|
|
||||||
* 2. Builds the main package
|
|
||||||
* 3. Builds the CLI package
|
|
||||||
* 4. Publishes the main package
|
|
||||||
* 5. Publishes the CLI package
|
|
||||||
*
|
*
|
||||||
* This ensures both packages are always published together with the same version.
|
* This script is no longer functional as the CLI package has been removed.
|
||||||
|
* Use 'npm publish' directly to publish the main package only.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { execSync } from 'child_process'
|
console.error('This script is deprecated. The CLI package has been removed.')
|
||||||
import fs from 'fs'
|
console.error('To publish the main package only, use: npm publish')
|
||||||
import path from 'path'
|
process.exit(1)
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
|
|
||||||
// Get the directory of the current module
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
|
||||||
const __dirname = path.dirname(__filename)
|
|
||||||
const rootDir = path.join(__dirname, '..')
|
|
||||||
const cliPackageDir = path.join(rootDir, 'cli-package')
|
|
||||||
|
|
||||||
// Ensure the CLI package directory exists
|
|
||||||
if (!fs.existsSync(cliPackageDir)) {
|
|
||||||
console.error(`Error: CLI package directory not found at ${cliPackageDir}`)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Step 1: Ensure versions are in sync
|
|
||||||
console.log('Ensuring versions are in sync...')
|
|
||||||
execSync('node scripts/generate-version.js', { stdio: 'inherit', cwd: rootDir })
|
|
||||||
|
|
||||||
// Step 2: Build the main package
|
|
||||||
console.log('Building main package...')
|
|
||||||
execSync('npm run build', { stdio: 'inherit', cwd: rootDir })
|
|
||||||
|
|
||||||
// Step 3: Publish the main package
|
|
||||||
console.log('Publishing main package...')
|
|
||||||
execSync('npm publish', { stdio: 'inherit', cwd: rootDir })
|
|
||||||
|
|
||||||
// Step 4: Wait a moment to ensure the package is available
|
|
||||||
console.log('Waiting for package to be available...')
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 5000))
|
|
||||||
|
|
||||||
// Step 5: Build the CLI package
|
|
||||||
console.log('Building CLI package...')
|
|
||||||
execSync('npm run build', { stdio: 'inherit', cwd: cliPackageDir })
|
|
||||||
|
|
||||||
// Step 6: Verify the CLI was built successfully
|
|
||||||
const cliPath = path.join(cliPackageDir, 'dist', 'cli.js')
|
|
||||||
if (!fs.existsSync(cliPath)) {
|
|
||||||
console.error(`Error: CLI build failed. File not found at ${cliPath}`)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 7: Publish the CLI package
|
|
||||||
console.log('Publishing CLI package...')
|
|
||||||
execSync('npm publish', { stdio: 'inherit', cwd: cliPackageDir })
|
|
||||||
|
|
||||||
console.log('Both packages published successfully!')
|
|
||||||
|
|
||||||
// Step 8: Create GitHub release
|
|
||||||
console.log('Creating GitHub release...')
|
|
||||||
execSync('node scripts/create-github-release.js', { stdio: 'inherit', cwd: rootDir })
|
|
||||||
|
|
||||||
console.log('Deployment complete!')
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error:', error.message)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,728 +0,0 @@
|
||||||
# Brainy Web Service
|
|
||||||
|
|
||||||
A secure, read-only web service wrapper for the Brainy vector graph database. This service exposes Brainy's search capabilities through RESTful API endpoints, designed specifically as a search service with comprehensive security measures.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- 🔒 **Read-only by design** - Only safe search operations are exposed
|
|
||||||
- 🚀 **RESTful API** - Clean, well-documented HTTP endpoints
|
|
||||||
- 🛡️ **Security-first** - Rate limiting, input validation, CORS, and security headers
|
|
||||||
- 📊 **Comprehensive search** - Vector search, text search, similarity search
|
|
||||||
- 🔍 **Flexible querying** - Support for noun types, verb filtering, and pagination
|
|
||||||
- 📈 **Production-ready** - Proper error handling, logging, and graceful shutdown
|
|
||||||
- 🌐 **CORS enabled** - Ready for browser-based applications
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm install -g @soulcraft/brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
Or run directly:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx @soulcraft/brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
1. **Start the service:**
|
|
||||||
```bash
|
|
||||||
brainy-server
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Check if it's running:**
|
|
||||||
```bash
|
|
||||||
curl http://localhost:3000/health
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **View API documentation:**
|
|
||||||
```bash
|
|
||||||
curl http://localhost:3000/api
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🚀 Easy Deployment
|
|
||||||
|
|
||||||
### Option 1: Direct NPM Deployment
|
|
||||||
```bash
|
|
||||||
# Install globally and run
|
|
||||||
npm install -g @soulcraft/brainy-web-service
|
|
||||||
brainy-server
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: One-liner with npx
|
|
||||||
```bash
|
|
||||||
# Run without installation
|
|
||||||
npx @soulcraft/brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 3: Docker Deployment
|
|
||||||
```bash
|
|
||||||
# Create a simple Dockerfile
|
|
||||||
cat > Dockerfile << EOF
|
|
||||||
FROM node:24-alpine
|
|
||||||
RUN npm install -g @soulcraft/brainy-web-service
|
|
||||||
EXPOSE 3000
|
|
||||||
CMD ["brainy-server"]
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Build and run
|
|
||||||
docker build -t brainy-web-service .
|
|
||||||
docker run -p 3000:3000 brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 4: Cloud Platform Deployment
|
|
||||||
|
|
||||||
#### Google Cloud Platform (App Engine)
|
|
||||||
```bash
|
|
||||||
# Create package.json
|
|
||||||
echo '{"scripts":{"start":"brainy-server"},"dependencies":{"@soulcraft/brainy-web-service":"latest"}}' > package.json
|
|
||||||
|
|
||||||
# Create app.yaml for App Engine
|
|
||||||
cat > app.yaml << EOF
|
|
||||||
runtime: nodejs20
|
|
||||||
env: standard
|
|
||||||
automatic_scaling:
|
|
||||||
min_instances: 1
|
|
||||||
max_instances: 10
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Deploy to App Engine
|
|
||||||
gcloud app deploy
|
|
||||||
```
|
|
||||||
|
|
||||||
#### AWS (App Runner)
|
|
||||||
```bash
|
|
||||||
# Create package.json
|
|
||||||
echo '{"scripts":{"start":"brainy-server"},"dependencies":{"@soulcraft/brainy-web-service":"latest"}}' > package.json
|
|
||||||
|
|
||||||
# Create apprunner.yaml
|
|
||||||
cat > apprunner.yaml << EOF
|
|
||||||
version: 1.0
|
|
||||||
runtime: nodejs20
|
|
||||||
build:
|
|
||||||
commands:
|
|
||||||
build:
|
|
||||||
- npm install
|
|
||||||
run:
|
|
||||||
runtime-version: 20
|
|
||||||
command: npm start
|
|
||||||
network:
|
|
||||||
port: 3000
|
|
||||||
env: PORT
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Deploy via AWS App Runner console or CLI
|
|
||||||
aws apprunner create-service --service-name brainy-web-service --source-configuration '{...}'
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Microsoft Azure (App Service)
|
|
||||||
```bash
|
|
||||||
# Create package.json
|
|
||||||
echo '{"scripts":{"start":"brainy-server"},"dependencies":{"@soulcraft/brainy-web-service":"latest"}}' > package.json
|
|
||||||
|
|
||||||
# Deploy using Azure CLI
|
|
||||||
az webapp create --resource-group myResourceGroup --plan myAppServicePlan --name brainy-web-service --runtime "NODE:20-lts"
|
|
||||||
az webapp deployment source config-zip --resource-group myResourceGroup --name brainy-web-service --src deployment.zip
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Cloudflare Workers
|
|
||||||
```bash
|
|
||||||
# Create package.json
|
|
||||||
echo '{"scripts":{"start":"brainy-server"},"dependencies":{"@soulcraft/brainy-web-service":"latest"}}' > package.json
|
|
||||||
|
|
||||||
# Create wrangler.toml
|
|
||||||
cat > wrangler.toml << EOF
|
|
||||||
name = "brainy-web-service"
|
|
||||||
main = "src/worker.js"
|
|
||||||
compatibility_date = "2024-01-01"
|
|
||||||
|
|
||||||
[env.production]
|
|
||||||
name = "brainy-web-service"
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Deploy with Wrangler
|
|
||||||
npx wrangler deploy
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 5: Serverless Deployment
|
|
||||||
```javascript
|
|
||||||
// For serverless platforms, wrap the service:
|
|
||||||
const express = require('express');
|
|
||||||
const app = require('@soulcraft/brainy-web-service');
|
|
||||||
|
|
||||||
module.exports = app;
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Configure the service using environment variables:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Server configuration
|
|
||||||
PORT=3000 # Server port (default: 3000)
|
|
||||||
HOST=0.0.0.0 # Server host (default: 0.0.0.0)
|
|
||||||
|
|
||||||
# Local Storage configuration (fallback)
|
|
||||||
BRAINY_DATA_PATH=/path/to/data # Path to Brainy database files (used when no cloud storage configured)
|
|
||||||
FORCE_LOCAL_STORAGE=true # Force local filesystem storage (ignores cloud storage config)
|
|
||||||
|
|
||||||
# Security configuration
|
|
||||||
CORS_ORIGIN=* # CORS origin (default: *)
|
|
||||||
RATE_LIMIT_WINDOW=900000 # Rate limit window in ms (default: 15 minutes)
|
|
||||||
RATE_LIMIT_MAX=100 # Max requests per window (default: 100)
|
|
||||||
|
|
||||||
# Environment
|
|
||||||
NODE_ENV=production # Environment mode
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloud Storage Configuration
|
|
||||||
|
|
||||||
The service automatically detects and uses cloud storage when configured. If no cloud storage is configured, it falls back to local filesystem storage.
|
|
||||||
|
|
||||||
#### AWS S3
|
|
||||||
```bash
|
|
||||||
S3_BUCKET_NAME=my-brainy-bucket
|
|
||||||
S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE # or AWS_ACCESS_KEY_ID
|
|
||||||
S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY # or AWS_SECRET_ACCESS_KEY
|
|
||||||
S3_REGION=us-west-2 # or AWS_REGION
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Cloudflare R2
|
|
||||||
```bash
|
|
||||||
R2_BUCKET_NAME=my-brainy-bucket
|
|
||||||
R2_ACCOUNT_ID=your-account-id
|
|
||||||
R2_ACCESS_KEY_ID=your-r2-access-key
|
|
||||||
R2_SECRET_ACCESS_KEY=your-r2-secret-key
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Google Cloud Storage (S3-compatible)
|
|
||||||
```bash
|
|
||||||
GCS_BUCKET_NAME=my-brainy-bucket
|
|
||||||
GCS_ACCESS_KEY_ID=your-gcs-access-key
|
|
||||||
GCS_SECRET_ACCESS_KEY=your-gcs-secret-key
|
|
||||||
GCS_ENDPOINT=https://storage.googleapis.com # Optional, defaults to GCS endpoint
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Azure Blob Storage (S3-compatible)
|
|
||||||
For Azure, you can use the S3-compatible interface with a custom endpoint:
|
|
||||||
```bash
|
|
||||||
S3_BUCKET_NAME=my-brainy-container
|
|
||||||
S3_ACCESS_KEY_ID=your-storage-account-name
|
|
||||||
S3_SECRET_ACCESS_KEY=your-storage-account-key
|
|
||||||
S3_REGION=auto
|
|
||||||
# Note: Azure Blob Storage S3 compatibility may require additional configuration
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Custom S3-Compatible Storage
|
|
||||||
```bash
|
|
||||||
S3_BUCKET_NAME=my-brainy-bucket
|
|
||||||
S3_ACCESS_KEY_ID=your-access-key
|
|
||||||
S3_SECRET_ACCESS_KEY=your-secret-key
|
|
||||||
S3_REGION=us-east-1
|
|
||||||
# Set custom endpoint via application configuration
|
|
||||||
```
|
|
||||||
|
|
||||||
## API Endpoints
|
|
||||||
|
|
||||||
### Health & Status
|
|
||||||
|
|
||||||
#### `GET /health`
|
|
||||||
Health check endpoint.
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"status": "healthy",
|
|
||||||
"timestamp": "2025-07-22T09:33:00.000Z",
|
|
||||||
"service": "brainy-web-service",
|
|
||||||
"version": "0.12.0"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `GET /api`
|
|
||||||
API documentation and endpoint listing.
|
|
||||||
|
|
||||||
#### `GET /api/status`
|
|
||||||
Database status and statistics.
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"size": 1000,
|
|
||||||
"readOnly": true,
|
|
||||||
"timestamp": "2025-07-22T09:33:00.000Z"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Search Operations
|
|
||||||
|
|
||||||
#### `POST /api/search`
|
|
||||||
Search using a vector.
|
|
||||||
|
|
||||||
**Request Body:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"vector": [0.1, 0.2, 0.3, ...],
|
|
||||||
"k": 10,
|
|
||||||
"nounTypes": ["document", "image"],
|
|
||||||
"includeVerbs": false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"results": [
|
|
||||||
{
|
|
||||||
"id": "item-1",
|
|
||||||
"score": 0.95,
|
|
||||||
"vector": [0.1, 0.2, 0.3, ...],
|
|
||||||
"metadata": {
|
|
||||||
"title": "Example Document",
|
|
||||||
"type": "document"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"query": {
|
|
||||||
"vectorLength": 384,
|
|
||||||
"k": 10,
|
|
||||||
"nounTypes": ["document"],
|
|
||||||
"includeVerbs": false
|
|
||||||
},
|
|
||||||
"timestamp": "2025-07-22T09:33:00.000Z"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `POST /api/search/text`
|
|
||||||
Search using text query (automatically vectorized).
|
|
||||||
|
|
||||||
**Request Body:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"query": "machine learning algorithms",
|
|
||||||
"k": 5,
|
|
||||||
"nounTypes": ["document"],
|
|
||||||
"includeVerbs": false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Response:** Same format as vector search.
|
|
||||||
|
|
||||||
### Data Retrieval
|
|
||||||
|
|
||||||
#### `GET /api/item/:id`
|
|
||||||
Get a specific item by ID.
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"item": {
|
|
||||||
"id": "item-1",
|
|
||||||
"vector": [0.1, 0.2, 0.3, ...],
|
|
||||||
"metadata": {
|
|
||||||
"title": "Example Document",
|
|
||||||
"type": "document"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"timestamp": "2025-07-22T09:33:00.000Z"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `GET /api/items?page=1&limit=20`
|
|
||||||
Get all items with pagination.
|
|
||||||
|
|
||||||
**Query Parameters:**
|
|
||||||
- `page` (optional): Page number (default: 1)
|
|
||||||
- `limit` (optional): Items per page (default: 20, max: 100)
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"items": [...],
|
|
||||||
"pagination": {
|
|
||||||
"page": 1,
|
|
||||||
"limit": 20,
|
|
||||||
"total": 1000,
|
|
||||||
"totalPages": 50,
|
|
||||||
"hasNext": true,
|
|
||||||
"hasPrev": false
|
|
||||||
},
|
|
||||||
"timestamp": "2025-07-22T09:33:00.000Z"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `POST /api/similar/:id`
|
|
||||||
Find items similar to a given item.
|
|
||||||
|
|
||||||
**Request Body:**
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"k": 10,
|
|
||||||
"includeVerbs": false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Response:** Same format as search results.
|
|
||||||
|
|
||||||
## Usage Examples
|
|
||||||
|
|
||||||
### JavaScript/Node.js
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const axios = require('axios');
|
|
||||||
|
|
||||||
const API_BASE = 'http://localhost:3000/api';
|
|
||||||
|
|
||||||
// Text search
|
|
||||||
async function searchText(query) {
|
|
||||||
const response = await axios.post(`${API_BASE}/search/text`, {
|
|
||||||
query: query,
|
|
||||||
k: 10
|
|
||||||
});
|
|
||||||
return response.data.results;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Vector search
|
|
||||||
async function searchVector(vector) {
|
|
||||||
const response = await axios.post(`${API_BASE}/search`, {
|
|
||||||
vector: vector,
|
|
||||||
k: 5
|
|
||||||
});
|
|
||||||
return response.data.results;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get item by ID
|
|
||||||
async function getItem(id) {
|
|
||||||
const response = await axios.get(`${API_BASE}/item/${id}`);
|
|
||||||
return response.data.item;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Usage
|
|
||||||
searchText('artificial intelligence').then(results => {
|
|
||||||
console.log('Search results:', results);
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
### Python
|
|
||||||
|
|
||||||
```python
|
|
||||||
import requests
|
|
||||||
|
|
||||||
API_BASE = 'http://localhost:3000/api'
|
|
||||||
|
|
||||||
def search_text(query, k=10):
|
|
||||||
response = requests.post(f'{API_BASE}/search/text', json={
|
|
||||||
'query': query,
|
|
||||||
'k': k
|
|
||||||
})
|
|
||||||
return response.json()['results']
|
|
||||||
|
|
||||||
def search_vector(vector, k=10):
|
|
||||||
response = requests.post(f'{API_BASE}/search', json={
|
|
||||||
'vector': vector,
|
|
||||||
'k': k
|
|
||||||
})
|
|
||||||
return response.json()['results']
|
|
||||||
|
|
||||||
def get_item(item_id):
|
|
||||||
response = requests.get(f'{API_BASE}/item/{item_id}')
|
|
||||||
return response.json()['item']
|
|
||||||
|
|
||||||
# Usage
|
|
||||||
results = search_text('machine learning')
|
|
||||||
print(f'Found {len(results)} results')
|
|
||||||
```
|
|
||||||
|
|
||||||
### cURL
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Text search
|
|
||||||
curl -X POST http://localhost:3000/api/search/text \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"query": "artificial intelligence", "k": 5}'
|
|
||||||
|
|
||||||
# Vector search
|
|
||||||
curl -X POST http://localhost:3000/api/search \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"vector": [0.1, 0.2, 0.3], "k": 10}'
|
|
||||||
|
|
||||||
# Get item
|
|
||||||
curl http://localhost:3000/api/item/example-id
|
|
||||||
|
|
||||||
# Get items with pagination
|
|
||||||
curl "http://localhost:3000/api/items?page=1&limit=10"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Security Features
|
|
||||||
|
|
||||||
### Read-Only Protection
|
|
||||||
- Database is automatically set to read-only mode on startup
|
|
||||||
- No write operations (add, delete, update) are exposed
|
|
||||||
- Only safe search and retrieval operations are available
|
|
||||||
|
|
||||||
### Rate Limiting
|
|
||||||
- Default: 100 requests per 15-minute window per IP
|
|
||||||
- Configurable via environment variables
|
|
||||||
- Returns 429 status code when limit exceeded
|
|
||||||
|
|
||||||
### Input Validation
|
|
||||||
- All inputs are validated using express-validator
|
|
||||||
- Vector dimensions and types are checked
|
|
||||||
- String lengths are limited to prevent abuse
|
|
||||||
- Numeric ranges are enforced
|
|
||||||
|
|
||||||
### Security Headers
|
|
||||||
- Helmet.js provides security headers
|
|
||||||
- Content Security Policy configured
|
|
||||||
- CORS properly configured
|
|
||||||
- Compression enabled for performance
|
|
||||||
|
|
||||||
### Error Handling
|
|
||||||
- Detailed error messages in development
|
|
||||||
- Generic error messages in production
|
|
||||||
- All errors are logged for monitoring
|
|
||||||
- Graceful shutdown on SIGTERM/SIGINT
|
|
||||||
|
|
||||||
## Deployment
|
|
||||||
|
|
||||||
### Docker
|
|
||||||
|
|
||||||
Create a `Dockerfile`:
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
FROM node:24-alpine
|
|
||||||
|
|
||||||
# Create non-root user for security
|
|
||||||
RUN addgroup -g 1001 -S brainy && \
|
|
||||||
adduser -S brainy -u 1001
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Copy package files
|
|
||||||
COPY package*.json ./
|
|
||||||
RUN npm ci --only=production
|
|
||||||
|
|
||||||
# Copy application
|
|
||||||
COPY . .
|
|
||||||
RUN npm run build
|
|
||||||
|
|
||||||
# Create data directory and set permissions
|
|
||||||
RUN mkdir -p /app/data && chown -R brainy:brainy /app
|
|
||||||
|
|
||||||
# Switch to non-root user
|
|
||||||
USER brainy
|
|
||||||
|
|
||||||
# Expose port
|
|
||||||
EXPOSE 3000
|
|
||||||
|
|
||||||
# Set environment
|
|
||||||
ENV NODE_ENV=production
|
|
||||||
|
|
||||||
# Start service
|
|
||||||
CMD ["npm", "start"]
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Local Storage Deployment
|
|
||||||
```bash
|
|
||||||
docker build -t brainy-web-service .
|
|
||||||
docker run -p 3000:3000 \
|
|
||||||
-v /host/data:/app/data \
|
|
||||||
-e BRAINY_DATA_PATH=/app/data \
|
|
||||||
brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
#### AWS S3 Deployment
|
|
||||||
```bash
|
|
||||||
docker run -p 3000:3000 \
|
|
||||||
-e S3_BUCKET_NAME=my-brainy-bucket \
|
|
||||||
-e S3_ACCESS_KEY_ID=your-access-key \
|
|
||||||
-e S3_SECRET_ACCESS_KEY=your-secret-key \
|
|
||||||
-e S3_REGION=us-west-2 \
|
|
||||||
brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Cloudflare R2 Deployment
|
|
||||||
```bash
|
|
||||||
docker run -p 3000:3000 \
|
|
||||||
-e R2_BUCKET_NAME=my-brainy-bucket \
|
|
||||||
-e R2_ACCOUNT_ID=your-account-id \
|
|
||||||
-e R2_ACCESS_KEY_ID=your-r2-access-key \
|
|
||||||
-e R2_SECRET_ACCESS_KEY=your-r2-secret-key \
|
|
||||||
brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Google Cloud Storage Deployment
|
|
||||||
```bash
|
|
||||||
docker run -p 3000:3000 \
|
|
||||||
-e GCS_BUCKET_NAME=my-brainy-bucket \
|
|
||||||
-e GCS_ACCESS_KEY_ID=your-gcs-access-key \
|
|
||||||
-e GCS_SECRET_ACCESS_KEY=your-gcs-secret-key \
|
|
||||||
brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
### Docker Compose
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
version: '3.8'
|
|
||||||
services:
|
|
||||||
brainy-web-service:
|
|
||||||
build: .
|
|
||||||
ports:
|
|
||||||
- "3000:3000"
|
|
||||||
volumes:
|
|
||||||
- ./data:/app/data
|
|
||||||
environment:
|
|
||||||
- NODE_ENV=production
|
|
||||||
- BRAINY_DATA_PATH=/app/data
|
|
||||||
- RATE_LIMIT_MAX=200
|
|
||||||
restart: unless-stopped
|
|
||||||
```
|
|
||||||
|
|
||||||
### Systemd Service
|
|
||||||
|
|
||||||
Create `/etc/systemd/system/brainy-web-service.service`:
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[Unit]
|
|
||||||
Description=Brainy Web Service
|
|
||||||
After=network.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=brainy
|
|
||||||
WorkingDirectory=/opt/brainy-web-service
|
|
||||||
ExecStart=/usr/bin/node dist/server.js
|
|
||||||
Restart=always
|
|
||||||
RestartSec=10
|
|
||||||
Environment=NODE_ENV=production
|
|
||||||
Environment=BRAINY_DATA_PATH=/opt/brainy-web-service/data
|
|
||||||
Environment=PORT=3000
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
```
|
|
||||||
|
|
||||||
Enable and start:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo systemctl enable brainy-web-service
|
|
||||||
sudo systemctl start brainy-web-service
|
|
||||||
```
|
|
||||||
|
|
||||||
### Reverse Proxy (Nginx)
|
|
||||||
|
|
||||||
```nginx
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
server_name your-domain.com;
|
|
||||||
|
|
||||||
location / {
|
|
||||||
proxy_pass http://localhost:3000;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
|
||||||
proxy_set_header Connection 'upgrade';
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_cache_bypass $http_upgrade;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
### Setup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git clone https://github.com/soulcraft-research/brainy.git
|
|
||||||
cd brainy/web-service-package
|
|
||||||
npm install
|
|
||||||
```
|
|
||||||
|
|
||||||
### Development Mode
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
### Building
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run build
|
|
||||||
```
|
|
||||||
|
|
||||||
### Testing
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm test
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monitoring
|
|
||||||
|
|
||||||
### Health Checks
|
|
||||||
|
|
||||||
The service provides health check endpoints for monitoring:
|
|
||||||
|
|
||||||
- `GET /health` - Basic health check
|
|
||||||
- `GET /api/status` - Detailed database status
|
|
||||||
|
|
||||||
### Logging
|
|
||||||
|
|
||||||
All requests and errors are logged to console. In production, consider using a logging service like Winston or Bunyan.
|
|
||||||
|
|
||||||
### Metrics
|
|
||||||
|
|
||||||
Consider integrating with monitoring solutions:
|
|
||||||
- Prometheus metrics
|
|
||||||
- New Relic
|
|
||||||
- DataDog
|
|
||||||
- Custom metrics via the status endpoint
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Common Issues
|
|
||||||
|
|
||||||
1. **Service won't start**
|
|
||||||
- Check if the data directory exists and is readable
|
|
||||||
- Verify Node.js version (requires >= 24.4.0)
|
|
||||||
- Check port availability
|
|
||||||
|
|
||||||
2. **Database not found**
|
|
||||||
- Ensure `BRAINY_DATA_PATH` points to valid Brainy database files
|
|
||||||
- Check file permissions
|
|
||||||
|
|
||||||
3. **Rate limiting issues**
|
|
||||||
- Adjust `RATE_LIMIT_MAX` and `RATE_LIMIT_WINDOW`
|
|
||||||
- Consider implementing IP whitelisting
|
|
||||||
|
|
||||||
4. **Memory issues**
|
|
||||||
- Monitor memory usage with large databases
|
|
||||||
- Consider implementing pagination for large result sets
|
|
||||||
|
|
||||||
### Debug Mode
|
|
||||||
|
|
||||||
Enable debug logging:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
NODE_ENV=development npm start
|
|
||||||
```
|
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
1. Fork the repository
|
|
||||||
2. Create a feature branch
|
|
||||||
3. Make your changes
|
|
||||||
4. Add tests
|
|
||||||
5. Submit a pull request
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
MIT License - see the main Brainy project for details.
|
|
||||||
|
|
||||||
## Support
|
|
||||||
|
|
||||||
- GitHub Issues: https://github.com/soulcraft-research/brainy/issues
|
|
||||||
- Documentation: https://github.com/soulcraft-research/brainy
|
|
||||||
- Email: david@soulcraft.com
|
|
||||||
3
web-service-package/dist/server.d.ts
vendored
3
web-service-package/dist/server.d.ts
vendored
|
|
@ -1,3 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
export {};
|
|
||||||
//# sourceMappingURL=server.d.ts.map
|
|
||||||
1
web-service-package/dist/server.d.ts.map
vendored
1
web-service-package/dist/server.d.ts.map
vendored
|
|
@ -1 +0,0 @@
|
||||||
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":""}
|
|
||||||
429
web-service-package/dist/server.js
vendored
429
web-service-package/dist/server.js
vendored
|
|
@ -1,429 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
#!/usr/bin/env node
|
|
||||||
import express from 'express';
|
|
||||||
import cors from 'cors';
|
|
||||||
import helmet from 'helmet';
|
|
||||||
import compression from 'compression';
|
|
||||||
import rateLimit from 'express-rate-limit';
|
|
||||||
import { body, param, query, validationResult } from 'express-validator';
|
|
||||||
import { BrainyData, cosineDistance, createStorage } from '@soulcraft/brainy';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
import { dirname, join } from 'path';
|
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url);
|
|
||||||
const __dirname = dirname(__filename);
|
|
||||||
// Configuration
|
|
||||||
const PORT = process.env.PORT || 3000;
|
|
||||||
const HOST = process.env.HOST || '0.0.0.0';
|
|
||||||
const DATA_PATH = process.env.BRAINY_DATA_PATH || join(__dirname, '..', 'data');
|
|
||||||
const CORS_ORIGIN = process.env.CORS_ORIGIN || '*';
|
|
||||||
const RATE_LIMIT_WINDOW = parseInt(process.env.RATE_LIMIT_WINDOW || '900000'); // 15 minutes
|
|
||||||
const RATE_LIMIT_MAX = parseInt(process.env.RATE_LIMIT_MAX || '100'); // 100 requests per window
|
|
||||||
// Cloud Storage Configuration
|
|
||||||
// The createStorage function will automatically detect and use these environment variables:
|
|
||||||
// AWS S3: S3_BUCKET_NAME, S3_ACCESS_KEY_ID (or AWS_ACCESS_KEY_ID), S3_SECRET_ACCESS_KEY (or AWS_SECRET_ACCESS_KEY), S3_REGION (or AWS_REGION)
|
|
||||||
// Cloudflare R2: R2_BUCKET_NAME, R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY
|
|
||||||
// Google Cloud Storage: GCS_BUCKET_NAME, GCS_ACCESS_KEY_ID, GCS_SECRET_ACCESS_KEY, GCS_ENDPOINT
|
|
||||||
// Force local storage: FORCE_LOCAL_STORAGE=true
|
|
||||||
// Initialize Express app
|
|
||||||
const app = express();
|
|
||||||
// Security middleware
|
|
||||||
app.use(helmet({
|
|
||||||
contentSecurityPolicy: {
|
|
||||||
directives: {
|
|
||||||
defaultSrc: ["'self'"],
|
|
||||||
styleSrc: ["'self'", "'unsafe-inline'"],
|
|
||||||
scriptSrc: ["'self'"],
|
|
||||||
imgSrc: ["'self'", "data:", "https:"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
app.use(cors({
|
|
||||||
origin: CORS_ORIGIN,
|
|
||||||
methods: ['GET', 'POST'],
|
|
||||||
allowedHeaders: ['Content-Type', 'Authorization'],
|
|
||||||
}));
|
|
||||||
app.use(compression());
|
|
||||||
app.use(express.json({ limit: '10mb' }));
|
|
||||||
// Rate limiting
|
|
||||||
const limiter = rateLimit({
|
|
||||||
windowMs: RATE_LIMIT_WINDOW,
|
|
||||||
max: RATE_LIMIT_MAX,
|
|
||||||
message: {
|
|
||||||
error: 'Too many requests from this IP, please try again later.',
|
|
||||||
retryAfter: Math.ceil(RATE_LIMIT_WINDOW / 1000)
|
|
||||||
},
|
|
||||||
standardHeaders: true,
|
|
||||||
legacyHeaders: false,
|
|
||||||
});
|
|
||||||
app.use('/api/', limiter);
|
|
||||||
// Global BrainyData instance
|
|
||||||
let brainyInstance = null;
|
|
||||||
// Initialize BrainyData instance
|
|
||||||
async function initializeBrainy() {
|
|
||||||
if (brainyInstance) {
|
|
||||||
return brainyInstance;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
console.log('Initializing Brainy database...');
|
|
||||||
// Create storage adapter with cloud support
|
|
||||||
const storageOptions = {
|
|
||||||
requestPersistentStorage: true
|
|
||||||
};
|
|
||||||
// Add AWS S3 configuration if environment variables are present
|
|
||||||
if (process.env.S3_BUCKET_NAME) {
|
|
||||||
storageOptions.s3Storage = {
|
|
||||||
bucketName: process.env.S3_BUCKET_NAME,
|
|
||||||
region: process.env.S3_REGION || process.env.AWS_REGION || 'us-east-1',
|
|
||||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY,
|
|
||||||
sessionToken: process.env.S3_SESSION_TOKEN || process.env.AWS_SESSION_TOKEN
|
|
||||||
};
|
|
||||||
}
|
|
||||||
// Add Cloudflare R2 configuration if environment variables are present
|
|
||||||
if (process.env.R2_BUCKET_NAME) {
|
|
||||||
storageOptions.r2Storage = {
|
|
||||||
bucketName: process.env.R2_BUCKET_NAME,
|
|
||||||
accountId: process.env.R2_ACCOUNT_ID,
|
|
||||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
|
||||||
};
|
|
||||||
}
|
|
||||||
// Add Google Cloud Storage configuration if environment variables are present
|
|
||||||
if (process.env.GCS_BUCKET_NAME) {
|
|
||||||
storageOptions.gcsStorage = {
|
|
||||||
bucketName: process.env.GCS_BUCKET_NAME,
|
|
||||||
region: process.env.GCS_REGION,
|
|
||||||
endpoint: process.env.GCS_ENDPOINT,
|
|
||||||
accessKeyId: process.env.GCS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY
|
|
||||||
};
|
|
||||||
}
|
|
||||||
// Check if local storage is forced
|
|
||||||
if (process.env.FORCE_LOCAL_STORAGE === 'true') {
|
|
||||||
console.log('Forcing local filesystem storage (FORCE_LOCAL_STORAGE=true)');
|
|
||||||
storageOptions.forceFileSystemStorage = true;
|
|
||||||
// Set the data path for local storage
|
|
||||||
if (DATA_PATH) {
|
|
||||||
// We'll need to import FileSystemStorage and path for forced local storage
|
|
||||||
const { FileSystemStorage } = await import('@soulcraft/brainy');
|
|
||||||
const path = await import('path');
|
|
||||||
// Create storage with explicit path handling to avoid race condition
|
|
||||||
const nounsDir = path.join(DATA_PATH, 'nouns');
|
|
||||||
const verbsDir = path.join(DATA_PATH, 'verbs');
|
|
||||||
const metadataDir = path.join(DATA_PATH, 'metadata');
|
|
||||||
const indexDir = path.join(DATA_PATH, 'index');
|
|
||||||
// Create a storage instance with pre-computed paths
|
|
||||||
const storage = new FileSystemStorage(DATA_PATH);
|
|
||||||
// Initialize the storage adapter before using it
|
|
||||||
await storage.init();
|
|
||||||
brainyInstance = new BrainyData({
|
|
||||||
storageAdapter: storage,
|
|
||||||
distanceFunction: cosineDistance
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// If not forcing local storage or no specific data path, use createStorage
|
|
||||||
if (!brainyInstance) {
|
|
||||||
const storage = await createStorage(storageOptions);
|
|
||||||
brainyInstance = new BrainyData({
|
|
||||||
storageAdapter: storage,
|
|
||||||
distanceFunction: cosineDistance
|
|
||||||
});
|
|
||||||
}
|
|
||||||
await brainyInstance.init();
|
|
||||||
// Force read-only mode for security
|
|
||||||
brainyInstance.setReadOnly(true);
|
|
||||||
// Log storage information
|
|
||||||
console.log(`Brainy database initialized in read-only mode`);
|
|
||||||
console.log(`Database size: ${brainyInstance.size()} items`);
|
|
||||||
return brainyInstance;
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Failed to initialize Brainy database:', error);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Error handler middleware
|
|
||||||
const handleValidationErrors = (req, res, next) => {
|
|
||||||
const errors = validationResult(req);
|
|
||||||
if (!errors.isEmpty()) {
|
|
||||||
return res.status(400).json({
|
|
||||||
error: 'Validation failed',
|
|
||||||
details: errors.array()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
next();
|
|
||||||
};
|
|
||||||
// Health check endpoint
|
|
||||||
app.get('/health', (req, res) => {
|
|
||||||
res.json({
|
|
||||||
status: 'healthy',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
service: 'brainy-web-service',
|
|
||||||
version: '0.12.0'
|
|
||||||
});
|
|
||||||
});
|
|
||||||
// API Documentation endpoint
|
|
||||||
app.get('/api', (req, res) => {
|
|
||||||
res.json({
|
|
||||||
service: 'Brainy Web Service',
|
|
||||||
version: '0.12.0',
|
|
||||||
description: 'Read-only search service for Brainy vector graph database',
|
|
||||||
endpoints: {
|
|
||||||
'GET /health': 'Health check',
|
|
||||||
'GET /api': 'API documentation',
|
|
||||||
'GET /api/status': 'Database status',
|
|
||||||
'POST /api/search': 'Search for similar vectors',
|
|
||||||
'POST /api/search/text': 'Search using text query',
|
|
||||||
'GET /api/item/:id': 'Get item by ID',
|
|
||||||
'GET /api/items': 'Get all items (paginated)',
|
|
||||||
'POST /api/similar/:id': 'Find similar items to given ID'
|
|
||||||
},
|
|
||||||
security: {
|
|
||||||
readOnly: true,
|
|
||||||
rateLimit: {
|
|
||||||
windowMs: RATE_LIMIT_WINDOW,
|
|
||||||
maxRequests: RATE_LIMIT_MAX
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
// Database status endpoint
|
|
||||||
app.get('/api/status', async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy();
|
|
||||||
const status = await brainy.status();
|
|
||||||
res.json({
|
|
||||||
...status,
|
|
||||||
readOnly: brainy.isReadOnly(),
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Status check failed:', error);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to get database status',
|
|
||||||
message: error.message
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
// Search endpoint - vector search
|
|
||||||
app.post('/api/search', [
|
|
||||||
body('vector').isArray().withMessage('Vector must be an array'),
|
|
||||||
body('vector.*').isNumeric().withMessage('Vector elements must be numbers'),
|
|
||||||
body('k').optional().isInt({ min: 1, max: 100 }).withMessage('k must be between 1 and 100'),
|
|
||||||
body('nounTypes').optional().isArray().withMessage('nounTypes must be an array'),
|
|
||||||
body('includeVerbs').optional().isBoolean().withMessage('includeVerbs must be boolean'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy();
|
|
||||||
const { vector, k = 10, nounTypes, includeVerbs = false } = req.body;
|
|
||||||
const results = await brainy.search(vector, k, {
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs,
|
|
||||||
searchMode: 'local' // Force local search for security
|
|
||||||
});
|
|
||||||
res.json({
|
|
||||||
results,
|
|
||||||
query: {
|
|
||||||
vectorLength: vector.length,
|
|
||||||
k,
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Search failed:', error);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Search failed',
|
|
||||||
message: error.message
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
// Text search endpoint
|
|
||||||
app.post('/api/search/text', [
|
|
||||||
body('query').isString().isLength({ min: 1, max: 1000 }).withMessage('Query must be a string between 1 and 1000 characters'),
|
|
||||||
body('k').optional().isInt({ min: 1, max: 100 }).withMessage('k must be between 1 and 100'),
|
|
||||||
body('nounTypes').optional().isArray().withMessage('nounTypes must be an array'),
|
|
||||||
body('includeVerbs').optional().isBoolean().withMessage('includeVerbs must be boolean'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy();
|
|
||||||
const { query, k = 10, nounTypes, includeVerbs = false } = req.body;
|
|
||||||
const results = await brainy.searchText(query, k, {
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs
|
|
||||||
});
|
|
||||||
res.json({
|
|
||||||
results,
|
|
||||||
query: {
|
|
||||||
text: query,
|
|
||||||
k,
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Text search failed:', error);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Text search failed',
|
|
||||||
message: error.message
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
// Get item by ID
|
|
||||||
app.get('/api/item/:id', [
|
|
||||||
param('id').isString().isLength({ min: 1, max: 100 }).withMessage('ID must be a string between 1 and 100 characters'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy();
|
|
||||||
const { id } = req.params;
|
|
||||||
const item = await brainy.get(id);
|
|
||||||
if (!item) {
|
|
||||||
return res.status(404).json({
|
|
||||||
error: 'Item not found',
|
|
||||||
id
|
|
||||||
});
|
|
||||||
}
|
|
||||||
res.json({
|
|
||||||
item,
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Get item failed:', error);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to get item',
|
|
||||||
message: error.message
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
// Get all items (paginated)
|
|
||||||
app.get('/api/items', [
|
|
||||||
query('page').optional().isInt({ min: 1 }).withMessage('Page must be a positive integer'),
|
|
||||||
query('limit').optional().isInt({ min: 1, max: 100 }).withMessage('Limit must be between 1 and 100'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy();
|
|
||||||
const page = parseInt(req.query.page) || 1;
|
|
||||||
const limit = parseInt(req.query.limit) || 20;
|
|
||||||
const allNouns = await brainy.getAllNouns();
|
|
||||||
const total = allNouns.length;
|
|
||||||
const startIndex = (page - 1) * limit;
|
|
||||||
const endIndex = startIndex + limit;
|
|
||||||
const items = allNouns.slice(startIndex, endIndex);
|
|
||||||
res.json({
|
|
||||||
items,
|
|
||||||
pagination: {
|
|
||||||
page,
|
|
||||||
limit,
|
|
||||||
total,
|
|
||||||
totalPages: Math.ceil(total / limit),
|
|
||||||
hasNext: endIndex < total,
|
|
||||||
hasPrev: page > 1
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Get items failed:', error);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to get items',
|
|
||||||
message: error.message
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
// Find similar items
|
|
||||||
app.post('/api/similar/:id', [
|
|
||||||
param('id').isString().isLength({ min: 1, max: 100 }).withMessage('ID must be a string between 1 and 100 characters'),
|
|
||||||
body('k').optional().isInt({ min: 1, max: 100 }).withMessage('k must be between 1 and 100'),
|
|
||||||
body('includeVerbs').optional().isBoolean().withMessage('includeVerbs must be boolean'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy();
|
|
||||||
const { id } = req.params;
|
|
||||||
const { k = 10, includeVerbs = false } = req.body;
|
|
||||||
const results = await brainy.findSimilar(id, {
|
|
||||||
limit: k,
|
|
||||||
includeVerbs
|
|
||||||
});
|
|
||||||
res.json({
|
|
||||||
results,
|
|
||||||
query: {
|
|
||||||
id,
|
|
||||||
k,
|
|
||||||
includeVerbs
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Find similar failed:', error);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to find similar items',
|
|
||||||
message: error.message
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
// 404 handler
|
|
||||||
app.use('*', (req, res) => {
|
|
||||||
res.status(404).json({
|
|
||||||
error: 'Endpoint not found',
|
|
||||||
path: req.originalUrl,
|
|
||||||
method: req.method
|
|
||||||
});
|
|
||||||
});
|
|
||||||
// Global error handler
|
|
||||||
app.use((err, req, res, next) => {
|
|
||||||
console.error('Unhandled error:', err);
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Internal server error',
|
|
||||||
message: process.env.NODE_ENV === 'development' ? err.message : 'Something went wrong'
|
|
||||||
});
|
|
||||||
});
|
|
||||||
// Graceful shutdown
|
|
||||||
process.on('SIGTERM', async () => {
|
|
||||||
console.log('SIGTERM received, shutting down gracefully...');
|
|
||||||
if (brainyInstance) {
|
|
||||||
await brainyInstance.shutDown();
|
|
||||||
}
|
|
||||||
process.exit(0);
|
|
||||||
});
|
|
||||||
process.on('SIGINT', async () => {
|
|
||||||
console.log('SIGINT received, shutting down gracefully...');
|
|
||||||
if (brainyInstance) {
|
|
||||||
await brainyInstance.shutDown();
|
|
||||||
}
|
|
||||||
process.exit(0);
|
|
||||||
});
|
|
||||||
// Start server
|
|
||||||
async function startServer() {
|
|
||||||
try {
|
|
||||||
// Initialize Brainy on startup
|
|
||||||
await initializeBrainy();
|
|
||||||
app.listen(Number(PORT), HOST, () => {
|
|
||||||
console.log(`🚀 Brainy Web Service started`);
|
|
||||||
console.log(`📍 Server running on http://${HOST}:${PORT}`);
|
|
||||||
console.log(`📊 API documentation available at http://${HOST}:${PORT}/api`);
|
|
||||||
console.log(`🔒 Read-only mode: ${brainyInstance?.isReadOnly()}`);
|
|
||||||
console.log(`📁 Data path: ${DATA_PATH}`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
console.error('Failed to start server:', error);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Start the server
|
|
||||||
startServer().catch(console.error);
|
|
||||||
//# sourceMappingURL=server.js.map
|
|
||||||
1
web-service-package/dist/server.js.map
vendored
1
web-service-package/dist/server.js.map
vendored
File diff suppressed because one or more lines are too long
5605
web-service-package/package-lock.json
generated
5605
web-service-package/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,84 +0,0 @@
|
||||||
{
|
|
||||||
"name": "@soulcraft/brainy-web-service",
|
|
||||||
"version": "0.16.0",
|
|
||||||
"description": "Web service wrapper for the Brainy vector graph database - read-only search service",
|
|
||||||
"type": "module",
|
|
||||||
"main": "dist/server.js",
|
|
||||||
"bin": {
|
|
||||||
"brainy-server": "server-wrapper.js"
|
|
||||||
},
|
|
||||||
"files": [
|
|
||||||
"server-wrapper.js",
|
|
||||||
"README.md",
|
|
||||||
"dist/server.js",
|
|
||||||
"dist/server.js.map"
|
|
||||||
],
|
|
||||||
"scripts": {
|
|
||||||
"build": "rollup -c rollup.config.js",
|
|
||||||
"start": "node dist/server.js",
|
|
||||||
"dev": "node --loader ts-node/esm src/server.ts",
|
|
||||||
"prepare": "npm run build",
|
|
||||||
"test": "vitest run",
|
|
||||||
"test:watch": "vitest",
|
|
||||||
"test:cloud": "vitest run tests/cloud-storage.test.ts",
|
|
||||||
"test:service": "vitest run tests/service.test.ts",
|
|
||||||
"_version": "echo 'Version updated in package.json'",
|
|
||||||
"_version:patch": "npm version patch",
|
|
||||||
"_version:minor": "npm version minor",
|
|
||||||
"_version:major": "npm version major",
|
|
||||||
"_deploy": "npm run build && npm publish",
|
|
||||||
"_dry-run": "npm pack --dry-run"
|
|
||||||
},
|
|
||||||
"keywords": [
|
|
||||||
"vector-database",
|
|
||||||
"hnsw",
|
|
||||||
"web-service",
|
|
||||||
"rest-api",
|
|
||||||
"search-service",
|
|
||||||
"browser",
|
|
||||||
"graph-database"
|
|
||||||
],
|
|
||||||
"author": "David Snelling (david@soulcraft.com)",
|
|
||||||
"license": "MIT",
|
|
||||||
"private": false,
|
|
||||||
"publishConfig": {
|
|
||||||
"access": "public"
|
|
||||||
},
|
|
||||||
"homepage": "https://github.com/soulcraft-research/brainy",
|
|
||||||
"bugs": {
|
|
||||||
"url": "https://github.com/soulcraft-research/brainy/issues"
|
|
||||||
},
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "git+https://github.com/soulcraft-research/brainy.git"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"@soulcraft/brainy": "^0.24.0",
|
|
||||||
"express": "^4.18.2",
|
|
||||||
"cors": "^2.8.5",
|
|
||||||
"helmet": "^7.1.0",
|
|
||||||
"express-rate-limit": "^7.1.5",
|
|
||||||
"express-validator": "^7.0.1",
|
|
||||||
"compression": "^1.7.4",
|
|
||||||
"@aws-sdk/client-s3": "^3.450.0"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@rollup/plugin-commonjs": "^25.0.7",
|
|
||||||
"@rollup/plugin-json": "^6.1.0",
|
|
||||||
"@rollup/plugin-node-resolve": "^15.2.3",
|
|
||||||
"@rollup/plugin-typescript": "^11.1.6",
|
|
||||||
"@types/node": "^20.11.30",
|
|
||||||
"@types/express": "^4.17.21",
|
|
||||||
"@types/cors": "^2.8.17",
|
|
||||||
"@types/compression": "^1.7.5",
|
|
||||||
"rollup": "^4.13.0",
|
|
||||||
"@rollup/plugin-terser": "^0.4.4",
|
|
||||||
"typescript": "^5.4.5",
|
|
||||||
"ts-node": "^10.9.2",
|
|
||||||
"vitest": "^3.2.4",
|
|
||||||
"axios": "^1.6.0"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">=24.4.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,57 +0,0 @@
|
||||||
import resolve from '@rollup/plugin-node-resolve'
|
|
||||||
import commonjs from '@rollup/plugin-commonjs'
|
|
||||||
import typescript from '@rollup/plugin-typescript'
|
|
||||||
import json from '@rollup/plugin-json'
|
|
||||||
import terser from '@rollup/plugin-terser'
|
|
||||||
|
|
||||||
const isProduction = process.env.NODE_ENV === 'production'
|
|
||||||
|
|
||||||
export default {
|
|
||||||
input: 'src/server.ts',
|
|
||||||
output: {
|
|
||||||
file: 'dist/server.js',
|
|
||||||
format: 'es',
|
|
||||||
sourcemap: true,
|
|
||||||
banner: '#!/usr/bin/env node'
|
|
||||||
},
|
|
||||||
external: [
|
|
||||||
// Node.js built-ins
|
|
||||||
'fs',
|
|
||||||
'path',
|
|
||||||
'url',
|
|
||||||
'crypto',
|
|
||||||
'os',
|
|
||||||
'util',
|
|
||||||
'events',
|
|
||||||
'stream',
|
|
||||||
'buffer',
|
|
||||||
'querystring',
|
|
||||||
'http',
|
|
||||||
'https',
|
|
||||||
'net',
|
|
||||||
'tls',
|
|
||||||
'zlib',
|
|
||||||
// External dependencies that should not be bundled
|
|
||||||
'express',
|
|
||||||
'cors',
|
|
||||||
'helmet',
|
|
||||||
'compression',
|
|
||||||
'express-rate-limit',
|
|
||||||
'express-validator',
|
|
||||||
'@soulcraft/brainy'
|
|
||||||
],
|
|
||||||
plugins: [
|
|
||||||
resolve({
|
|
||||||
preferBuiltins: true,
|
|
||||||
exportConditions: ['node']
|
|
||||||
}),
|
|
||||||
commonjs(),
|
|
||||||
json(),
|
|
||||||
typescript({
|
|
||||||
tsconfig: './tsconfig.json',
|
|
||||||
sourceMap: true,
|
|
||||||
inlineSources: !isProduction
|
|
||||||
}),
|
|
||||||
...(isProduction ? [terser()] : [])
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
// Entry point wrapper for the Brainy Web Service
|
|
||||||
// This file serves as the executable entry point when installed via npm
|
|
||||||
|
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
import { dirname, join } from 'path'
|
|
||||||
import { existsSync } from 'fs'
|
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
|
||||||
const __dirname = dirname(__filename)
|
|
||||||
|
|
||||||
// Try to load the built version first, fallback to source
|
|
||||||
const builtServerPath = join(__dirname, 'dist', 'server.js')
|
|
||||||
const sourceServerPath = join(__dirname, 'src', 'server.ts')
|
|
||||||
|
|
||||||
if (existsSync(builtServerPath)) {
|
|
||||||
// Load the built version
|
|
||||||
console.log('Starting Brainy Web Service (built version)...')
|
|
||||||
import(builtServerPath).catch(error => {
|
|
||||||
console.error('Failed to start server:', error)
|
|
||||||
process.exit(1)
|
|
||||||
})
|
|
||||||
} else if (existsSync(sourceServerPath)) {
|
|
||||||
// Development mode - load source with ts-node
|
|
||||||
console.log('Starting Brainy Web Service (development mode)...')
|
|
||||||
console.log('Note: For production, run "npm run build" first')
|
|
||||||
|
|
||||||
// Check if ts-node is available and load source
|
|
||||||
import('ts-node/esm').then(() => {
|
|
||||||
import(sourceServerPath).catch(error => {
|
|
||||||
console.error('Failed to start server:', error)
|
|
||||||
process.exit(1)
|
|
||||||
})
|
|
||||||
}).catch(error => {
|
|
||||||
console.error('ts-node not found. Please install it for development mode or run "npm run build" first.')
|
|
||||||
console.error('Install ts-node: npm install -g ts-node')
|
|
||||||
process.exit(1)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
console.error('Server files not found. Please ensure the package is properly installed.')
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
|
|
@ -1,474 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
import express from 'express'
|
|
||||||
import cors from 'cors'
|
|
||||||
import helmet from 'helmet'
|
|
||||||
import compression from 'compression'
|
|
||||||
import rateLimit from 'express-rate-limit'
|
|
||||||
import { body, query, param, validationResult } from 'express-validator'
|
|
||||||
import { BrainyData, createStorage, cosineDistance } from '@soulcraft/brainy'
|
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
import { dirname, join } from 'path'
|
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
|
||||||
const __dirname = dirname(__filename)
|
|
||||||
|
|
||||||
// Configuration
|
|
||||||
const PORT = process.env.PORT || 3000
|
|
||||||
const HOST = process.env.HOST || '0.0.0.0'
|
|
||||||
const DATA_PATH = process.env.BRAINY_DATA_PATH || join(__dirname, '..', 'data')
|
|
||||||
const CORS_ORIGIN = process.env.CORS_ORIGIN || '*'
|
|
||||||
const RATE_LIMIT_WINDOW = parseInt(process.env.RATE_LIMIT_WINDOW || '900000') // 15 minutes
|
|
||||||
const RATE_LIMIT_MAX = parseInt(process.env.RATE_LIMIT_MAX || '100') // 100 requests per window
|
|
||||||
|
|
||||||
// Cloud Storage Configuration
|
|
||||||
// The createStorage function will automatically detect and use these environment variables:
|
|
||||||
// AWS S3: S3_BUCKET_NAME, S3_ACCESS_KEY_ID (or AWS_ACCESS_KEY_ID), S3_SECRET_ACCESS_KEY (or AWS_SECRET_ACCESS_KEY), S3_REGION (or AWS_REGION)
|
|
||||||
// Cloudflare R2: R2_BUCKET_NAME, R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY
|
|
||||||
// Google Cloud Storage: GCS_BUCKET_NAME, GCS_ACCESS_KEY_ID, GCS_SECRET_ACCESS_KEY, GCS_ENDPOINT
|
|
||||||
// Force local storage: FORCE_LOCAL_STORAGE=true
|
|
||||||
|
|
||||||
// Initialize Express app
|
|
||||||
const app = express()
|
|
||||||
|
|
||||||
// Security middleware
|
|
||||||
app.use(helmet({
|
|
||||||
contentSecurityPolicy: {
|
|
||||||
directives: {
|
|
||||||
defaultSrc: ["'self'"],
|
|
||||||
styleSrc: ["'self'", "'unsafe-inline'"],
|
|
||||||
scriptSrc: ["'self'"],
|
|
||||||
imgSrc: ["'self'", "data:", "https:"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}))
|
|
||||||
|
|
||||||
app.use(cors({
|
|
||||||
origin: CORS_ORIGIN,
|
|
||||||
methods: ['GET', 'POST'],
|
|
||||||
allowedHeaders: ['Content-Type', 'Authorization'],
|
|
||||||
}))
|
|
||||||
|
|
||||||
app.use(compression())
|
|
||||||
app.use(express.json({ limit: '10mb' }))
|
|
||||||
|
|
||||||
// Rate limiting
|
|
||||||
const limiter = rateLimit({
|
|
||||||
windowMs: RATE_LIMIT_WINDOW,
|
|
||||||
max: RATE_LIMIT_MAX,
|
|
||||||
message: {
|
|
||||||
error: 'Too many requests from this IP, please try again later.',
|
|
||||||
retryAfter: Math.ceil(RATE_LIMIT_WINDOW / 1000)
|
|
||||||
},
|
|
||||||
standardHeaders: true,
|
|
||||||
legacyHeaders: false,
|
|
||||||
})
|
|
||||||
|
|
||||||
app.use('/api/', limiter)
|
|
||||||
|
|
||||||
// Global BrainyData instance
|
|
||||||
let brainyInstance: BrainyData | null = null
|
|
||||||
|
|
||||||
// Initialize BrainyData instance
|
|
||||||
async function initializeBrainy(): Promise<BrainyData> {
|
|
||||||
if (brainyInstance) {
|
|
||||||
return brainyInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
console.log('Initializing Brainy database...')
|
|
||||||
|
|
||||||
// Create storage adapter with cloud support
|
|
||||||
const storageOptions: any = {
|
|
||||||
requestPersistentStorage: true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add AWS S3 configuration if environment variables are present
|
|
||||||
if (process.env.S3_BUCKET_NAME) {
|
|
||||||
storageOptions.s3Storage = {
|
|
||||||
bucketName: process.env.S3_BUCKET_NAME,
|
|
||||||
region: process.env.S3_REGION || process.env.AWS_REGION || 'us-east-1',
|
|
||||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY,
|
|
||||||
sessionToken: process.env.S3_SESSION_TOKEN || process.env.AWS_SESSION_TOKEN
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add Cloudflare R2 configuration if environment variables are present
|
|
||||||
if (process.env.R2_BUCKET_NAME) {
|
|
||||||
storageOptions.r2Storage = {
|
|
||||||
bucketName: process.env.R2_BUCKET_NAME,
|
|
||||||
accountId: process.env.R2_ACCOUNT_ID,
|
|
||||||
accessKeyId: process.env.R2_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add Google Cloud Storage configuration if environment variables are present
|
|
||||||
if (process.env.GCS_BUCKET_NAME) {
|
|
||||||
storageOptions.gcsStorage = {
|
|
||||||
bucketName: process.env.GCS_BUCKET_NAME,
|
|
||||||
region: process.env.GCS_REGION,
|
|
||||||
endpoint: process.env.GCS_ENDPOINT,
|
|
||||||
accessKeyId: process.env.GCS_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if local storage is forced
|
|
||||||
if (process.env.FORCE_LOCAL_STORAGE === 'true') {
|
|
||||||
console.log('Forcing local filesystem storage (FORCE_LOCAL_STORAGE=true)')
|
|
||||||
storageOptions.forceFileSystemStorage = true
|
|
||||||
// Set the data path for local storage
|
|
||||||
if (DATA_PATH) {
|
|
||||||
// We'll need to import FileSystemStorage and path for forced local storage
|
|
||||||
const { FileSystemStorage } = await import('@soulcraft/brainy')
|
|
||||||
const path = await import('path')
|
|
||||||
|
|
||||||
// Create storage with explicit path handling to avoid race condition
|
|
||||||
const nounsDir = path.join(DATA_PATH, 'nouns')
|
|
||||||
const verbsDir = path.join(DATA_PATH, 'verbs')
|
|
||||||
const metadataDir = path.join(DATA_PATH, 'metadata')
|
|
||||||
const indexDir = path.join(DATA_PATH, 'index')
|
|
||||||
|
|
||||||
// Create a storage instance with pre-computed paths
|
|
||||||
const storage = new FileSystemStorage(DATA_PATH)
|
|
||||||
|
|
||||||
// Initialize the storage adapter before using it
|
|
||||||
await storage.init()
|
|
||||||
|
|
||||||
brainyInstance = new BrainyData({
|
|
||||||
storageAdapter: storage,
|
|
||||||
distanceFunction: cosineDistance
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If not forcing local storage or no specific data path, use createStorage
|
|
||||||
if (!brainyInstance) {
|
|
||||||
const storage = await createStorage(storageOptions)
|
|
||||||
|
|
||||||
brainyInstance = new BrainyData({
|
|
||||||
storageAdapter: storage,
|
|
||||||
distanceFunction: cosineDistance
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
await brainyInstance.init()
|
|
||||||
|
|
||||||
// Force read-only mode for security
|
|
||||||
brainyInstance.setReadOnly(true)
|
|
||||||
|
|
||||||
// Log storage information
|
|
||||||
console.log(`Brainy database initialized in read-only mode`)
|
|
||||||
console.log(`Database size: ${brainyInstance.size()} items`)
|
|
||||||
|
|
||||||
return brainyInstance
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Failed to initialize Brainy database:', error)
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Error handler middleware
|
|
||||||
const handleValidationErrors = (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
|
||||||
const errors = validationResult(req)
|
|
||||||
if (!errors.isEmpty()) {
|
|
||||||
return res.status(400).json({
|
|
||||||
error: 'Validation failed',
|
|
||||||
details: errors.array()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
next()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Health check endpoint
|
|
||||||
app.get('/health', (req, res) => {
|
|
||||||
res.json({
|
|
||||||
status: 'healthy',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
service: 'brainy-web-service',
|
|
||||||
version: '0.12.0'
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// API Documentation endpoint
|
|
||||||
app.get('/api', (req, res) => {
|
|
||||||
res.json({
|
|
||||||
service: 'Brainy Web Service',
|
|
||||||
version: '0.12.0',
|
|
||||||
description: 'Read-only search service for Brainy vector graph database',
|
|
||||||
endpoints: {
|
|
||||||
'GET /health': 'Health check',
|
|
||||||
'GET /api': 'API documentation',
|
|
||||||
'GET /api/status': 'Database status',
|
|
||||||
'POST /api/search': 'Search for similar vectors',
|
|
||||||
'POST /api/search/text': 'Search using text query',
|
|
||||||
'GET /api/item/:id': 'Get item by ID',
|
|
||||||
'GET /api/items': 'Get all items (paginated)',
|
|
||||||
'POST /api/similar/:id': 'Find similar items to given ID'
|
|
||||||
},
|
|
||||||
security: {
|
|
||||||
readOnly: true,
|
|
||||||
rateLimit: {
|
|
||||||
windowMs: RATE_LIMIT_WINDOW,
|
|
||||||
maxRequests: RATE_LIMIT_MAX
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Database status endpoint
|
|
||||||
app.get('/api/status', async (req, res) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy()
|
|
||||||
const status = await brainy.status()
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
...status,
|
|
||||||
readOnly: brainy.isReadOnly(),
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Status check failed:', error)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to get database status',
|
|
||||||
message: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Search endpoint - vector search
|
|
||||||
app.post('/api/search', [
|
|
||||||
body('vector').isArray().withMessage('Vector must be an array'),
|
|
||||||
body('vector.*').isNumeric().withMessage('Vector elements must be numbers'),
|
|
||||||
body('k').optional().isInt({ min: 1, max: 100 }).withMessage('k must be between 1 and 100'),
|
|
||||||
body('nounTypes').optional().isArray().withMessage('nounTypes must be an array'),
|
|
||||||
body('includeVerbs').optional().isBoolean().withMessage('includeVerbs must be boolean'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req: express.Request, res: express.Response) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy()
|
|
||||||
const { vector, k = 10, nounTypes, includeVerbs = false } = req.body
|
|
||||||
|
|
||||||
const results = await brainy.search(vector, k, {
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs,
|
|
||||||
searchMode: 'local' // Force local search for security
|
|
||||||
})
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
results,
|
|
||||||
query: {
|
|
||||||
vectorLength: vector.length,
|
|
||||||
k,
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Search failed:', error)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Search failed',
|
|
||||||
message: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Text search endpoint
|
|
||||||
app.post('/api/search/text', [
|
|
||||||
body('query').isString().isLength({ min: 1, max: 1000 }).withMessage('Query must be a string between 1 and 1000 characters'),
|
|
||||||
body('k').optional().isInt({ min: 1, max: 100 }).withMessage('k must be between 1 and 100'),
|
|
||||||
body('nounTypes').optional().isArray().withMessage('nounTypes must be an array'),
|
|
||||||
body('includeVerbs').optional().isBoolean().withMessage('includeVerbs must be boolean'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req: express.Request, res: express.Response) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy()
|
|
||||||
const { query, k = 10, nounTypes, includeVerbs = false } = req.body
|
|
||||||
|
|
||||||
const results = await brainy.searchText(query, k, {
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs
|
|
||||||
})
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
results,
|
|
||||||
query: {
|
|
||||||
text: query,
|
|
||||||
k,
|
|
||||||
nounTypes,
|
|
||||||
includeVerbs
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Text search failed:', error)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Text search failed',
|
|
||||||
message: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get item by ID
|
|
||||||
app.get('/api/item/:id', [
|
|
||||||
param('id').isString().isLength({ min: 1, max: 100 }).withMessage('ID must be a string between 1 and 100 characters'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req: express.Request, res: express.Response) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy()
|
|
||||||
const { id } = req.params
|
|
||||||
|
|
||||||
const item = await brainy.get(id)
|
|
||||||
|
|
||||||
if (!item) {
|
|
||||||
return res.status(404).json({
|
|
||||||
error: 'Item not found',
|
|
||||||
id
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
item,
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Get item failed:', error)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to get item',
|
|
||||||
message: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get all items (paginated)
|
|
||||||
app.get('/api/items', [
|
|
||||||
query('page').optional().isInt({ min: 1 }).withMessage('Page must be a positive integer'),
|
|
||||||
query('limit').optional().isInt({ min: 1, max: 100 }).withMessage('Limit must be between 1 and 100'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req: express.Request, res: express.Response) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy()
|
|
||||||
const page = parseInt(req.query.page as string) || 1
|
|
||||||
const limit = parseInt(req.query.limit as string) || 20
|
|
||||||
|
|
||||||
const allNouns = await brainy.getAllNouns()
|
|
||||||
const total = allNouns.length
|
|
||||||
const startIndex = (page - 1) * limit
|
|
||||||
const endIndex = startIndex + limit
|
|
||||||
const items = allNouns.slice(startIndex, endIndex)
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
items,
|
|
||||||
pagination: {
|
|
||||||
page,
|
|
||||||
limit,
|
|
||||||
total,
|
|
||||||
totalPages: Math.ceil(total / limit),
|
|
||||||
hasNext: endIndex < total,
|
|
||||||
hasPrev: page > 1
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Get items failed:', error)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to get items',
|
|
||||||
message: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Find similar items
|
|
||||||
app.post('/api/similar/:id', [
|
|
||||||
param('id').isString().isLength({ min: 1, max: 100 }).withMessage('ID must be a string between 1 and 100 characters'),
|
|
||||||
body('k').optional().isInt({ min: 1, max: 100 }).withMessage('k must be between 1 and 100'),
|
|
||||||
body('includeVerbs').optional().isBoolean().withMessage('includeVerbs must be boolean'),
|
|
||||||
handleValidationErrors
|
|
||||||
], async (req: express.Request, res: express.Response) => {
|
|
||||||
try {
|
|
||||||
const brainy = await initializeBrainy()
|
|
||||||
const { id } = req.params
|
|
||||||
const { k = 10, includeVerbs = false } = req.body
|
|
||||||
|
|
||||||
const results = await brainy.findSimilar(id, {
|
|
||||||
limit: k,
|
|
||||||
includeVerbs
|
|
||||||
})
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
results,
|
|
||||||
query: {
|
|
||||||
id,
|
|
||||||
k,
|
|
||||||
includeVerbs
|
|
||||||
},
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Find similar failed:', error)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Failed to find similar items',
|
|
||||||
message: (error as Error).message
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// 404 handler
|
|
||||||
app.use('*', (req, res) => {
|
|
||||||
res.status(404).json({
|
|
||||||
error: 'Endpoint not found',
|
|
||||||
path: req.originalUrl,
|
|
||||||
method: req.method
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Global error handler
|
|
||||||
app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {
|
|
||||||
console.error('Unhandled error:', err)
|
|
||||||
res.status(500).json({
|
|
||||||
error: 'Internal server error',
|
|
||||||
message: process.env.NODE_ENV === 'development' ? err.message : 'Something went wrong'
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Graceful shutdown
|
|
||||||
process.on('SIGTERM', async () => {
|
|
||||||
console.log('SIGTERM received, shutting down gracefully...')
|
|
||||||
if (brainyInstance) {
|
|
||||||
await brainyInstance.shutDown()
|
|
||||||
}
|
|
||||||
process.exit(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
process.on('SIGINT', async () => {
|
|
||||||
console.log('SIGINT received, shutting down gracefully...')
|
|
||||||
if (brainyInstance) {
|
|
||||||
await brainyInstance.shutDown()
|
|
||||||
}
|
|
||||||
process.exit(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Start server
|
|
||||||
async function startServer() {
|
|
||||||
try {
|
|
||||||
// Initialize Brainy on startup
|
|
||||||
await initializeBrainy()
|
|
||||||
|
|
||||||
app.listen(Number(PORT), HOST, () => {
|
|
||||||
console.log(`🚀 Brainy Web Service started`)
|
|
||||||
console.log(`📍 Server running on http://${HOST}:${PORT}`)
|
|
||||||
console.log(`📊 API documentation available at http://${HOST}:${PORT}/api`)
|
|
||||||
console.log(`🔒 Read-only mode: ${brainyInstance?.isReadOnly()}`)
|
|
||||||
console.log(`📁 Data path: ${DATA_PATH}`)
|
|
||||||
})
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Failed to start server:', error)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start the server
|
|
||||||
startServer().catch(console.error)
|
|
||||||
|
|
@ -1,282 +0,0 @@
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
||||||
import axios from 'axios'
|
|
||||||
import { spawn, ChildProcess } from 'child_process'
|
|
||||||
import { setTimeout as setTimeoutPromise } from 'timers/promises'
|
|
||||||
|
|
||||||
const BASE_URL = process.env.TEST_BASE_URL || 'http://localhost:3001' // Use different port to avoid conflicts
|
|
||||||
const API_URL = `${BASE_URL}/api`
|
|
||||||
|
|
||||||
describe('Brainy Web Service Cloud Storage Integration', () => {
|
|
||||||
let serverProcess: ChildProcess | null = null
|
|
||||||
|
|
||||||
const startServer = async (env: Record<string, string> = {}): Promise<void> => {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const serverEnv = {
|
|
||||||
...process.env,
|
|
||||||
PORT: '3001', // Use different port for testing
|
|
||||||
NODE_ENV: 'development',
|
|
||||||
...env
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(` Starting server with environment:`)
|
|
||||||
Object.keys(env).forEach(key => {
|
|
||||||
if (key.includes('SECRET') || key.includes('KEY')) {
|
|
||||||
console.log(` ${key}=***`)
|
|
||||||
} else {
|
|
||||||
console.log(` ${key}=${env[key]}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess = spawn('node', ['src/server.ts'], {
|
|
||||||
env: serverEnv,
|
|
||||||
stdio: ['pipe', 'pipe', 'pipe'],
|
|
||||||
cwd: process.cwd()
|
|
||||||
})
|
|
||||||
|
|
||||||
let output = ''
|
|
||||||
let errorOutput = ''
|
|
||||||
|
|
||||||
serverProcess.stdout?.on('data', (data) => {
|
|
||||||
const dataStr = data.toString()
|
|
||||||
output += dataStr
|
|
||||||
|
|
||||||
// Check for error messages in stdout that indicate initialization failure
|
|
||||||
if (dataStr.includes('Failed to initialize') || dataStr.includes('Error:')) {
|
|
||||||
console.log(` Server stdout error: ${dataStr.trim()}`)
|
|
||||||
reject(new Error(`Server initialization failed: ${dataStr.trim()}`))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (output.includes('Server running on')) {
|
|
||||||
resolve()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess.stderr?.on('data', (data) => {
|
|
||||||
const dataStr = data.toString()
|
|
||||||
errorOutput += dataStr
|
|
||||||
console.log(` Server stderr: ${dataStr.trim()}`)
|
|
||||||
|
|
||||||
// Check for error messages in stderr that indicate initialization failure
|
|
||||||
if (dataStr.includes('Failed to initialize') || dataStr.includes('Error:')) {
|
|
||||||
reject(new Error(`Server initialization failed: ${dataStr.trim()}`))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess.on('error', (error) => {
|
|
||||||
reject(new Error(`Failed to start server: ${error.message}`))
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess.on('exit', (code, signal) => {
|
|
||||||
// If the process exited with a non-zero code, it's an error
|
|
||||||
if (code !== 0 && code !== null) {
|
|
||||||
reject(new Error(`Server exited with code ${code}. Error: ${errorOutput}`))
|
|
||||||
}
|
|
||||||
// If the process was terminated by a signal and we have error output, it's an error
|
|
||||||
else if (signal && errorOutput) {
|
|
||||||
reject(new Error(`Server terminated by signal ${signal}. Error: ${errorOutput}`))
|
|
||||||
}
|
|
||||||
// If we have error output but no code or signal, it's still an error
|
|
||||||
else if (errorOutput && errorOutput.includes('Error:')) {
|
|
||||||
reject(new Error(`Server exited with error: ${errorOutput}`))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Timeout after 10 seconds
|
|
||||||
const timeoutId = setTimeout(() => {
|
|
||||||
if (serverProcess && !serverProcess.killed) {
|
|
||||||
reject(new Error('Server startup timeout'))
|
|
||||||
}
|
|
||||||
}, 10000)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
const stopServer = async (): Promise<void> => {
|
|
||||||
if (serverProcess && !serverProcess.killed) {
|
|
||||||
serverProcess.kill('SIGTERM')
|
|
||||||
await setTimeoutPromise(2000) // Wait for graceful shutdown
|
|
||||||
if (!serverProcess.killed) {
|
|
||||||
serverProcess.kill('SIGKILL')
|
|
||||||
}
|
|
||||||
serverProcess = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const waitForServer = async (maxAttempts: number = 10): Promise<boolean> => {
|
|
||||||
for (let i = 0; i < maxAttempts; i++) {
|
|
||||||
try {
|
|
||||||
await axios.get(`${BASE_URL}/health`, { timeout: 1000 })
|
|
||||||
return true
|
|
||||||
} catch (error) {
|
|
||||||
await setTimeoutPromise(1000)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error('Server did not become ready in time')
|
|
||||||
}
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
// Ensure server is stopped after each test
|
|
||||||
await stopServer()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Local Storage Fallback', () => {
|
|
||||||
it('should use local storage when forced', async () => {
|
|
||||||
console.log(' Testing local storage fallback...')
|
|
||||||
|
|
||||||
await startServer({
|
|
||||||
FORCE_LOCAL_STORAGE: 'true',
|
|
||||||
BRAINY_DATA_PATH: './test-data'
|
|
||||||
})
|
|
||||||
|
|
||||||
await waitForServer()
|
|
||||||
|
|
||||||
// Test health endpoint
|
|
||||||
const healthResponse = await axios.get(`${BASE_URL}/health`)
|
|
||||||
expect(healthResponse.status).toBe(200)
|
|
||||||
|
|
||||||
// Test status endpoint to verify storage type
|
|
||||||
const statusResponse = await axios.get(`${API_URL}/status`)
|
|
||||||
expect(statusResponse.status).toBe(200)
|
|
||||||
|
|
||||||
console.log(` Storage type detected: ${statusResponse.data.type || 'unknown'}`)
|
|
||||||
console.log(` Read-only mode: ${statusResponse.data.readOnly}`)
|
|
||||||
|
|
||||||
expect(statusResponse.data.readOnly).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Cloud Storage Configuration', () => {
|
|
||||||
it('should handle AWS S3 configuration (mock)', async () => {
|
|
||||||
console.log(' Testing cloud storage configuration (mock)...')
|
|
||||||
|
|
||||||
// Test with mock S3 configuration (will fail to connect but should show proper config)
|
|
||||||
await expect(async () => {
|
|
||||||
await startServer({
|
|
||||||
S3_BUCKET_NAME: 'test-bucket',
|
|
||||||
S3_ACCESS_KEY_ID: 'test-key',
|
|
||||||
S3_SECRET_ACCESS_KEY: 'test-secret',
|
|
||||||
S3_REGION: 'us-west-2'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Wait a bit for initialization attempt
|
|
||||||
await setTimeoutPromise(3000)
|
|
||||||
}).rejects.toThrow() // Expected to fail with mock credentials
|
|
||||||
|
|
||||||
console.log(' Cloud storage configuration test completed (connection failure expected with mock credentials)')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle Cloudflare R2 configuration (mock)', async () => {
|
|
||||||
console.log(' Testing Cloudflare R2 configuration (mock)...')
|
|
||||||
|
|
||||||
await expect(async () => {
|
|
||||||
await startServer({
|
|
||||||
R2_BUCKET_NAME: 'test-bucket',
|
|
||||||
R2_ACCOUNT_ID: 'test-account',
|
|
||||||
R2_ACCESS_KEY_ID: 'test-key',
|
|
||||||
R2_SECRET_ACCESS_KEY: 'test-secret'
|
|
||||||
})
|
|
||||||
|
|
||||||
await setTimeoutPromise(3000)
|
|
||||||
}).rejects.toThrow() // Expected to fail with mock credentials
|
|
||||||
|
|
||||||
console.log(' R2 configuration test completed (connection failure expected with mock credentials)')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should handle Google Cloud Storage configuration (mock)', async () => {
|
|
||||||
console.log(' Testing Google Cloud Storage configuration (mock)...')
|
|
||||||
|
|
||||||
await expect(async () => {
|
|
||||||
await startServer({
|
|
||||||
GCS_BUCKET_NAME: 'test-bucket',
|
|
||||||
GCS_ACCESS_KEY_ID: 'test-key',
|
|
||||||
GCS_SECRET_ACCESS_KEY: 'test-secret',
|
|
||||||
GCS_ENDPOINT: 'https://storage.googleapis.com'
|
|
||||||
})
|
|
||||||
|
|
||||||
await setTimeoutPromise(3000)
|
|
||||||
}).rejects.toThrow() // Expected to fail with mock credentials
|
|
||||||
|
|
||||||
console.log(' GCS configuration test completed (connection failure expected with mock credentials)')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Environment Variable Priority', () => {
|
|
||||||
it('should prioritize cloud storage over local storage', async () => {
|
|
||||||
console.log(' Testing environment variable priority...')
|
|
||||||
|
|
||||||
// Test that cloud storage config takes priority over local storage
|
|
||||||
await expect(async () => {
|
|
||||||
await startServer({
|
|
||||||
BRAINY_DATA_PATH: './test-data',
|
|
||||||
S3_BUCKET_NAME: 'priority-test-bucket',
|
|
||||||
S3_ACCESS_KEY_ID: 'test-key',
|
|
||||||
S3_SECRET_ACCESS_KEY: 'test-secret',
|
|
||||||
S3_REGION: 'us-west-2'
|
|
||||||
})
|
|
||||||
|
|
||||||
await setTimeoutPromise(3000)
|
|
||||||
}).rejects.toThrow() // Expected to fail as it tries cloud storage first
|
|
||||||
|
|
||||||
console.log(' Environment variable priority test completed')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Force Local Storage Override', () => {
|
|
||||||
it('should override cloud storage config when forced', async () => {
|
|
||||||
console.log(' Testing FORCE_LOCAL_STORAGE override...')
|
|
||||||
|
|
||||||
// Test that FORCE_LOCAL_STORAGE overrides cloud storage config
|
|
||||||
await startServer({
|
|
||||||
FORCE_LOCAL_STORAGE: 'true',
|
|
||||||
BRAINY_DATA_PATH: './test-data',
|
|
||||||
S3_BUCKET_NAME: 'should-be-ignored',
|
|
||||||
S3_ACCESS_KEY_ID: 'should-be-ignored',
|
|
||||||
S3_SECRET_ACCESS_KEY: 'should-be-ignored',
|
|
||||||
S3_REGION: 'us-west-2'
|
|
||||||
})
|
|
||||||
|
|
||||||
await waitForServer()
|
|
||||||
|
|
||||||
const statusResponse = await axios.get(`${API_URL}/status`)
|
|
||||||
expect(statusResponse.status).toBe(200)
|
|
||||||
|
|
||||||
console.log(` Forced local storage - Storage type: ${statusResponse.data.type || 'unknown'}`)
|
|
||||||
expect(statusResponse.data.readOnly).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Configuration Detection', () => {
|
|
||||||
it('should properly detect storage configurations', () => {
|
|
||||||
// Test configuration detection logic
|
|
||||||
const testConfigs = [
|
|
||||||
{
|
|
||||||
name: 'AWS S3',
|
|
||||||
env: { S3_BUCKET_NAME: 'test', S3_ACCESS_KEY_ID: 'test', S3_SECRET_ACCESS_KEY: 'test' },
|
|
||||||
expected: 'cloud'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'Cloudflare R2',
|
|
||||||
env: { R2_BUCKET_NAME: 'test', R2_ACCESS_KEY_ID: 'test', R2_SECRET_ACCESS_KEY: 'test' },
|
|
||||||
expected: 'cloud'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'Local fallback',
|
|
||||||
env: { BRAINY_DATA_PATH: './test-data' },
|
|
||||||
expected: 'local'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'Force local override',
|
|
||||||
env: { FORCE_LOCAL_STORAGE: 'true', S3_BUCKET_NAME: 'test' },
|
|
||||||
expected: 'local'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
testConfigs.forEach(config => {
|
|
||||||
console.log(` Configuration: ${config.name} -> Expected: ${config.expected}`)
|
|
||||||
expect(config.env).toBeDefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(' Configuration detection logic verified')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,278 +0,0 @@
|
||||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
|
||||||
import axios from 'axios'
|
|
||||||
import { setTimeout as setTimeoutPromise } from 'timers/promises'
|
|
||||||
import { spawn, ChildProcess } from 'child_process'
|
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
import { dirname, join } from 'path'
|
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url)
|
|
||||||
const __dirname = dirname(__filename)
|
|
||||||
|
|
||||||
const BASE_URL = process.env.TEST_BASE_URL || 'http://localhost:3000'
|
|
||||||
const API_URL = `${BASE_URL}/api`
|
|
||||||
|
|
||||||
let serverProcess: ChildProcess | null = null
|
|
||||||
|
|
||||||
const startServer = async (env: Record<string, string> = {}): Promise<void> => {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const serverPath = join(__dirname, '..', 'src', 'server.ts')
|
|
||||||
|
|
||||||
serverProcess = spawn('node', ['--loader', 'ts-node/esm', serverPath], {
|
|
||||||
env: { ...process.env, ...env, PORT: '3000' },
|
|
||||||
stdio: ['pipe', 'pipe', 'pipe']
|
|
||||||
})
|
|
||||||
|
|
||||||
let errorOutput = ''
|
|
||||||
|
|
||||||
serverProcess.stdout?.on('data', (data) => {
|
|
||||||
const output = data.toString()
|
|
||||||
console.log(`Server: ${output.trim()}`)
|
|
||||||
if (output.includes('Server running on')) {
|
|
||||||
resolve()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess.stderr?.on('data', (data) => {
|
|
||||||
errorOutput += data.toString()
|
|
||||||
console.error(`Server Error: ${data.toString().trim()}`)
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess.on('error', (error) => {
|
|
||||||
reject(new Error(`Failed to start server: ${error.message}`))
|
|
||||||
})
|
|
||||||
|
|
||||||
serverProcess.on('exit', (code) => {
|
|
||||||
if (code !== 0 && code !== null) {
|
|
||||||
reject(new Error(`Server exited with code ${code}. Error: ${errorOutput}`))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Timeout after 10 seconds
|
|
||||||
const timeoutId = setTimeout(() => {
|
|
||||||
if (serverProcess && !serverProcess.killed) {
|
|
||||||
reject(new Error('Server startup timeout'))
|
|
||||||
}
|
|
||||||
}, 10000)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
const stopServer = async (): Promise<void> => {
|
|
||||||
if (serverProcess && !serverProcess.killed) {
|
|
||||||
serverProcess.kill('SIGTERM')
|
|
||||||
await setTimeoutPromise(2000) // Wait for graceful shutdown
|
|
||||||
if (!serverProcess.killed) {
|
|
||||||
serverProcess.kill('SIGKILL')
|
|
||||||
}
|
|
||||||
serverProcess = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const waitForServer = async (maxAttempts: number = 10): Promise<boolean> => {
|
|
||||||
for (let i = 0; i < maxAttempts; i++) {
|
|
||||||
try {
|
|
||||||
await axios.get(`${BASE_URL}/health`, { timeout: 1000 })
|
|
||||||
return true
|
|
||||||
} catch (error) {
|
|
||||||
await setTimeoutPromise(1000)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error('Server did not become ready in time')
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Brainy Web Service', () => {
|
|
||||||
beforeAll(async () => {
|
|
||||||
console.log('Starting server for tests...')
|
|
||||||
await startServer({
|
|
||||||
FORCE_LOCAL_STORAGE: 'true',
|
|
||||||
BRAINY_DATA_PATH: './test-data'
|
|
||||||
})
|
|
||||||
await waitForServer()
|
|
||||||
console.log('Server ready for tests')
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
console.log('Stopping server after tests...')
|
|
||||||
await stopServer()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Health Check', () => {
|
|
||||||
it('should return healthy status', async () => {
|
|
||||||
const response = await axios.get(`${BASE_URL}/health`)
|
|
||||||
|
|
||||||
expect(response.status).toBe(200)
|
|
||||||
expect(response.data.status).toBe('healthy')
|
|
||||||
expect(response.data.service).toBeDefined()
|
|
||||||
expect(response.data.version).toBeDefined()
|
|
||||||
|
|
||||||
console.log(` Service: ${response.data.service} v${response.data.version}`)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('API Documentation', () => {
|
|
||||||
it('should provide complete API documentation', async () => {
|
|
||||||
const response = await axios.get(`${API_URL}`)
|
|
||||||
|
|
||||||
expect(response.status).toBe(200)
|
|
||||||
expect(response.data.endpoints).toBeDefined()
|
|
||||||
|
|
||||||
const expectedEndpoints = [
|
|
||||||
'GET /health',
|
|
||||||
'GET /api/status',
|
|
||||||
'POST /api/search',
|
|
||||||
'POST /api/search/text',
|
|
||||||
'GET /api/item/:id'
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const endpoint of expectedEndpoints) {
|
|
||||||
expect(response.data.endpoints[endpoint]).toBeDefined()
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(` Found ${Object.keys(response.data.endpoints).length} documented endpoints`)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Database Status', () => {
|
|
||||||
it('should return database status with read-only mode', async () => {
|
|
||||||
const response = await axios.get(`${API_URL}/status`)
|
|
||||||
|
|
||||||
expect(response.status).toBe(200)
|
|
||||||
expect(typeof response.data.readOnly).toBe('boolean')
|
|
||||||
expect(response.data.readOnly).toBe(true) // Should be in read-only mode for security
|
|
||||||
|
|
||||||
console.log(` Database size: ${response.data.size || 0} items`)
|
|
||||||
console.log(` Read-only mode: ${response.data.readOnly}`)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Input Validation', () => {
|
|
||||||
it('should reject invalid vector search requests', async () => {
|
|
||||||
await expect(async () => {
|
|
||||||
await axios.post(`${API_URL}/search`, {
|
|
||||||
vector: "not an array",
|
|
||||||
k: 5
|
|
||||||
})
|
|
||||||
}).rejects.toThrow()
|
|
||||||
|
|
||||||
try {
|
|
||||||
await axios.post(`${API_URL}/search`, {
|
|
||||||
vector: "not an array",
|
|
||||||
k: 5
|
|
||||||
})
|
|
||||||
} catch (error: any) {
|
|
||||||
expect(error.response?.status).toBe(400)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject empty text search queries', async () => {
|
|
||||||
try {
|
|
||||||
await axios.post(`${API_URL}/search/text`, {
|
|
||||||
query: "", // Empty query should be rejected
|
|
||||||
k: 5
|
|
||||||
})
|
|
||||||
throw new Error('Should have rejected empty query')
|
|
||||||
} catch (error: any) {
|
|
||||||
expect(error.response?.status).toBe(400)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should reject k parameter that is too large', async () => {
|
|
||||||
try {
|
|
||||||
await axios.post(`${API_URL}/search/text`, {
|
|
||||||
query: "test",
|
|
||||||
k: 1000 // Too large
|
|
||||||
})
|
|
||||||
throw new Error('Should have rejected k > 100')
|
|
||||||
} catch (error: any) {
|
|
||||||
expect(error.response?.status).toBe(400)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should validate input correctly', () => {
|
|
||||||
console.log(' Input validation working correctly')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Security Headers', () => {
|
|
||||||
it('should include security headers', async () => {
|
|
||||||
const response = await axios.get(`${BASE_URL}/health`)
|
|
||||||
|
|
||||||
const securityHeaders = [
|
|
||||||
'x-content-type-options',
|
|
||||||
'x-frame-options',
|
|
||||||
'x-xss-protection'
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const header of securityHeaders) {
|
|
||||||
if (!response.headers[header]) {
|
|
||||||
console.log(` Warning: Missing security header: ${header}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(' Security headers check completed')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('CORS Support', () => {
|
|
||||||
it('should include CORS headers', async () => {
|
|
||||||
const response = await axios.options(`${API_URL}/status`)
|
|
||||||
|
|
||||||
expect(response.headers['access-control-allow-origin']).toBeDefined()
|
|
||||||
console.log(' CORS headers present')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Rate Limiting', () => {
|
|
||||||
it('should implement rate limiting', async () => {
|
|
||||||
console.log(' Testing rate limiting (this may take a moment)...')
|
|
||||||
|
|
||||||
// Make multiple rapid requests to test rate limiting
|
|
||||||
const requests = []
|
|
||||||
for (let i = 0; i < 10; i++) {
|
|
||||||
requests.push(
|
|
||||||
axios.get(`${BASE_URL}/health`).catch(err => err.response)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const responses = await Promise.all(requests)
|
|
||||||
const successCount = responses.filter(r => r?.status === 200).length
|
|
||||||
|
|
||||||
expect(successCount).toBeGreaterThan(0) // At least some should succeed
|
|
||||||
console.log(` ${successCount}/10 requests succeeded (rate limiting active)`)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('404 Handling', () => {
|
|
||||||
it('should return 404 for non-existent endpoints', async () => {
|
|
||||||
try {
|
|
||||||
await axios.get(`${API_URL}/nonexistent`)
|
|
||||||
throw new Error('Should have returned 404 for non-existent endpoint')
|
|
||||||
} catch (error: any) {
|
|
||||||
expect(error.response?.status).toBe(404)
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(' 404 handling working correctly')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Read-Only Security', () => {
|
|
||||||
it('should not expose dangerous write endpoints', async () => {
|
|
||||||
const dangerousEndpoints = [
|
|
||||||
{ method: 'post', path: '/api/add' },
|
|
||||||
{ method: 'delete', path: '/api/item/test' },
|
|
||||||
{ method: 'put', path: '/api/item/test' },
|
|
||||||
{ method: 'post', path: '/api/clear' }
|
|
||||||
]
|
|
||||||
|
|
||||||
for (const endpoint of dangerousEndpoints) {
|
|
||||||
try {
|
|
||||||
await (axios as any)[endpoint.method](`${BASE_URL}${endpoint.path}`)
|
|
||||||
throw new Error(`Dangerous endpoint ${endpoint.method.toUpperCase()} ${endpoint.path} should not exist`)
|
|
||||||
} catch (error: any) {
|
|
||||||
expect(error.response?.status).toBe(404)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(' No dangerous write endpoints exposed')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
"target": "ES2022",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleResolution": "node",
|
|
||||||
"allowSyntheticDefaultImports": true,
|
|
||||||
"esModuleInterop": true,
|
|
||||||
"allowJs": true,
|
|
||||||
"strict": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"forceConsistentCasingInFileNames": true,
|
|
||||||
"declaration": true,
|
|
||||||
"declarationMap": true,
|
|
||||||
"sourceMap": true,
|
|
||||||
"outDir": "./dist",
|
|
||||||
"rootDir": "./src",
|
|
||||||
"resolveJsonModule": true,
|
|
||||||
"types": ["node"],
|
|
||||||
"lib": ["ES2022"]
|
|
||||||
},
|
|
||||||
"include": [
|
|
||||||
"src/**/*"
|
|
||||||
],
|
|
||||||
"exclude": [
|
|
||||||
"node_modules",
|
|
||||||
"dist",
|
|
||||||
"**/*.test.ts",
|
|
||||||
"**/*.spec.ts"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
import { defineConfig } from 'vitest/config'
|
|
||||||
|
|
||||||
export default defineConfig({
|
|
||||||
test: {
|
|
||||||
// Global test utilities
|
|
||||||
globals: true,
|
|
||||||
// Longer timeout for server startup/shutdown and HTTP requests
|
|
||||||
testTimeout: 30000, // 30 seconds for web service operations
|
|
||||||
hookTimeout: 30000,
|
|
||||||
// Include test files in tests directory
|
|
||||||
include: ['tests/**/*.{test,spec}.{js,ts}'],
|
|
||||||
// Node environment for server testing
|
|
||||||
environment: 'node',
|
|
||||||
// Exclude unnecessary files
|
|
||||||
exclude: [
|
|
||||||
'node_modules/**',
|
|
||||||
'dist/**',
|
|
||||||
'src/**',
|
|
||||||
'*.js' // Exclude old JS test files
|
|
||||||
],
|
|
||||||
// Use default reporter with summary
|
|
||||||
reporters: ['default'],
|
|
||||||
// Enable console output for debugging
|
|
||||||
silent: false,
|
|
||||||
// Don't bail on first failure to see all test results
|
|
||||||
bail: 0,
|
|
||||||
// Disable coverage by default
|
|
||||||
coverage: {
|
|
||||||
enabled: false
|
|
||||||
},
|
|
||||||
// Show detailed output for web service testing
|
|
||||||
logHeapUsage: false,
|
|
||||||
hideSkippedTests: false,
|
|
||||||
printConsoleTrace: true,
|
|
||||||
// Filter out server startup noise but keep important messages
|
|
||||||
onConsoleLog: (log: string, type: 'stdout' | 'stderr'): false | void => {
|
|
||||||
const noisePatterns: string[] = [
|
|
||||||
'Brainy running in Node.js environment',
|
|
||||||
'Using file system storage for Node.js environment',
|
|
||||||
'Platform node has already been set',
|
|
||||||
'Hi there 👋. Looks like you are running TensorFlow.js'
|
|
||||||
]
|
|
||||||
|
|
||||||
// Return false (don't show) if log contains any noise pattern
|
|
||||||
if (noisePatterns.some((pattern) => log.includes(pattern))) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
// Resolve configuration for proper module handling
|
|
||||||
resolve: {
|
|
||||||
alias: {
|
|
||||||
'@': './src',
|
|
||||||
'@tests': './tests'
|
|
||||||
}
|
|
||||||
},
|
|
||||||
// Define test environment
|
|
||||||
define: {
|
|
||||||
'process.env.NODE_ENV': '"test"'
|
|
||||||
}
|
|
||||||
})
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue