David Snelling
2f6ab9559a
Replace JavaScript Sets with hardware-accelerated RoaringBitmap32 for metadata indexes. Key improvements: - 1.4x average speedup, up to 3.3x on 10K entities - 90% memory reduction (40 bytes/UUID → 4 bytes/int) - Hardware-accelerated multi-field intersection via SIMD (AVX2/SSE4.2) - EntityIdMapper for bidirectional UUID ↔ integer mapping - Portable serialization format Benchmark results (1,000 queries): - 10K entities: 3.74ms → 1.14ms (3.3x faster, 90% memory savings) - 100K entities: 2.60ms → 1.78ms (1.5x faster, 88% memory savings) Implementation: - Add EntityIdMapper class for UUID/int mapping with persistence - Modify ChunkData to use Map<string, RoaringBitmap32> - Add getIdsForMultipleFields() for fast bitmap intersection - Include comprehensive tests (25 tests passing) - Add performance benchmark comparing Set vs Roaring Technical details: - roaring@2.4.0 dependency - Maintains backward compatibility - All queries still return UUID strings - Automatic persistence via storage adapter
221 lines
7.2 KiB
JSON
221 lines
7.2 KiB
JSON
{
|
||
"name": "@soulcraft/brainy",
|
||
"version": "3.42.0",
|
||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
||
"main": "dist/index.js",
|
||
"module": "dist/index.js",
|
||
"types": "dist/index.d.ts",
|
||
"type": "module",
|
||
"bin": {
|
||
"brainy": "bin/brainy.js"
|
||
},
|
||
"sideEffects": [
|
||
"./dist/setup.js",
|
||
"./dist/utils/textEncoding.js",
|
||
"./src/setup.ts",
|
||
"./src/utils/textEncoding.ts"
|
||
],
|
||
"exports": {
|
||
".": {
|
||
"import": "./dist/index.js",
|
||
"types": "./dist/index.d.ts"
|
||
},
|
||
"./setup": {
|
||
"import": "./dist/setup.js",
|
||
"types": "./dist/setup.d.ts"
|
||
},
|
||
"./types/graphTypes": {
|
||
"import": "./dist/types/graphTypes.js",
|
||
"types": "./dist/types/graphTypes.d.ts"
|
||
},
|
||
"./types/augmentations": {
|
||
"import": "./dist/types/augmentations.js",
|
||
"types": "./dist/types/augmentations.d.ts"
|
||
},
|
||
"./utils/textEncoding": {
|
||
"import": "./dist/utils/textEncoding.js",
|
||
"types": "./dist/utils/textEncoding.d.ts"
|
||
},
|
||
"./universal": {
|
||
"import": "./dist/universal/index.js",
|
||
"types": "./dist/universal/index.d.ts"
|
||
}
|
||
},
|
||
"browser": {
|
||
"./dist/distributed": false,
|
||
"./dist/cli": false,
|
||
"./dist/scripts": false,
|
||
"./dist/cortex": false,
|
||
"./dist/cortex/neuralImport.js": false,
|
||
"./dist/cortex/performanceMonitor.js": false,
|
||
"./dist/cortex/healthCheck.js": false,
|
||
"./dist/cortex/backupRestore.js": false
|
||
},
|
||
"engines": {
|
||
"node": "22.x"
|
||
},
|
||
"scripts": {
|
||
"build": "npm run build:types:if-needed && npm run build:patterns:if-needed && tsc && tsc -p tsconfig.cli.json",
|
||
"build:types": "tsx scripts/buildTypeEmbeddings.ts",
|
||
"build:types:if-needed": "node scripts/check-type-embeddings.cjs || npm run build:types",
|
||
"build:types:force": "npm run build:types",
|
||
"build:patterns": "tsx scripts/buildEmbeddedPatterns.ts",
|
||
"build:patterns:if-needed": "node scripts/check-patterns.cjs || npm run build:patterns",
|
||
"build:patterns:force": "npm run build:patterns",
|
||
"prepare": "npm run build",
|
||
"test": "npm run test:unit",
|
||
"test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts",
|
||
"test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage",
|
||
"test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts",
|
||
"test:integration": "NODE_OPTIONS='--max-old-space-size=32768' vitest run --config tests/configs/vitest.integration.config.ts",
|
||
"test:s3": "vitest run tests/integration/s3-storage.test.ts",
|
||
"test:distributed": "vitest run tests/integration/distributed.test.ts",
|
||
"test:cloud": "npm run test:s3 && npm run test:distributed",
|
||
"test:all": "npm run test:unit && npm run test:integration",
|
||
"test:ci-unit": "CI=true vitest run --config tests/configs/vitest.unit.config.ts",
|
||
"test:ci-integration": "NODE_OPTIONS='--max-old-space-size=16384' CI=true vitest run --config tests/configs/vitest.integration.config.ts",
|
||
"test:ci": "npm run test:ci-unit",
|
||
"download-models": "node scripts/download-models.cjs",
|
||
"download-models:q8": "node scripts/download-models.cjs",
|
||
"models:verify": "node scripts/ensure-models.js",
|
||
"lint": "eslint --ext .ts,.js src/",
|
||
"lint:fix": "eslint --ext .ts,.js src/ --fix",
|
||
"format": "prettier --write \"src/**/*.{ts,js}\"",
|
||
"format:check": "prettier --check \"src/**/*.{ts,js}\"",
|
||
"migrate:logger": "tsx scripts/migrate-to-structured-logger.ts",
|
||
"migrate:logger:dry": "tsx scripts/migrate-to-structured-logger.ts --dry-run",
|
||
"release": "./scripts/release.sh patch",
|
||
"release:patch": "./scripts/release.sh patch",
|
||
"release:minor": "./scripts/release.sh minor",
|
||
"release:major": "./scripts/release.sh major",
|
||
"release:dry": "./scripts/release.sh patch --dry-run",
|
||
"release:standard-version": "standard-version",
|
||
"release:standard-version:patch": "standard-version --release-as patch",
|
||
"release:standard-version:minor": "standard-version --release-as minor",
|
||
"release:standard-version:major": "standard-version --release-as major"
|
||
},
|
||
"keywords": [
|
||
"ai-database",
|
||
"vector-database",
|
||
"graph-database",
|
||
"field-filtering",
|
||
"triple-intelligence",
|
||
"hnsw",
|
||
"embeddings",
|
||
"semantic-search",
|
||
"machine-learning",
|
||
"artificial-intelligence",
|
||
"data-storage",
|
||
"indexing",
|
||
"typescript"
|
||
],
|
||
"author": "Brainy Contributors",
|
||
"license": "MIT",
|
||
"private": false,
|
||
"publishConfig": {
|
||
"access": "public"
|
||
},
|
||
"homepage": "https://github.com/soulcraftlabs/brainy",
|
||
"bugs": {
|
||
"url": "https://github.com/soulcraftlabs/brainy/issues"
|
||
},
|
||
"repository": {
|
||
"type": "git",
|
||
"url": "git+https://github.com/soulcraftlabs/brainy.git"
|
||
},
|
||
"files": [
|
||
"dist/**/*.js",
|
||
"dist/**/*.d.ts",
|
||
"bin/",
|
||
"scripts/download-models.cjs",
|
||
"scripts/ensure-models.js",
|
||
"scripts/prepare-models.js",
|
||
"brainy.png",
|
||
"LICENSE",
|
||
"README.md",
|
||
"CHANGELOG.md"
|
||
],
|
||
"overrides": {
|
||
"boolean": "3.2.0"
|
||
},
|
||
"devDependencies": {
|
||
"@rollup/plugin-commonjs": "^28.0.6",
|
||
"@rollup/plugin-node-resolve": "^16.0.1",
|
||
"@rollup/plugin-replace": "^6.0.2",
|
||
"@rollup/plugin-terser": "^0.4.4",
|
||
"@testcontainers/redis": "^11.5.1",
|
||
"@types/node": "^20.11.30",
|
||
"@types/uuid": "^10.0.0",
|
||
"@types/ws": "^8.18.1",
|
||
"@typescript-eslint/eslint-plugin": "^8.0.0",
|
||
"@typescript-eslint/parser": "^8.0.0",
|
||
"@vitest/coverage-v8": "^3.2.4",
|
||
"jspdf": "^3.0.3",
|
||
"minio": "^8.0.5",
|
||
"standard-version": "^9.5.0",
|
||
"testcontainers": "^11.5.1",
|
||
"tsx": "^4.19.2",
|
||
"typescript": "^5.4.5",
|
||
"vitest": "^3.2.4"
|
||
},
|
||
"dependencies": {
|
||
"@aws-sdk/client-s3": "^3.540.0",
|
||
"@google-cloud/storage": "^7.14.0",
|
||
"@huggingface/transformers": "^3.7.2",
|
||
"boxen": "^8.0.1",
|
||
"chalk": "^5.3.0",
|
||
"chardet": "^2.0.0",
|
||
"cli-table3": "^0.6.5",
|
||
"commander": "^11.1.0",
|
||
"csv-parse": "^6.1.0",
|
||
"inquirer": "^12.9.3",
|
||
"ora": "^8.2.0",
|
||
"pdfjs-dist": "^4.0.379",
|
||
"prompts": "^2.4.2",
|
||
"roaring": "^2.4.0",
|
||
"uuid": "^9.0.1",
|
||
"ws": "^8.18.3",
|
||
"xlsx": "^0.18.5"
|
||
},
|
||
"prettier": {
|
||
"arrowParens": "always",
|
||
"bracketSameLine": true,
|
||
"bracketSpacing": true,
|
||
"htmlWhitespaceSensitivity": "css",
|
||
"printWidth": 80,
|
||
"proseWrap": "preserve",
|
||
"semi": false,
|
||
"singleQuote": true,
|
||
"tabWidth": 2,
|
||
"trailingComma": "none",
|
||
"useTabs": false
|
||
},
|
||
"eslintConfig": {
|
||
"root": true,
|
||
"extends": [
|
||
"eslint:recommended",
|
||
"plugin:@typescript-eslint/recommended"
|
||
],
|
||
"parser": "@typescript-eslint/parser",
|
||
"plugins": [
|
||
"@typescript-eslint"
|
||
],
|
||
"rules": {
|
||
"@typescript-eslint/no-explicit-any": "off",
|
||
"no-unused-vars": "off",
|
||
"@typescript-eslint/no-unused-vars": [
|
||
"warn",
|
||
{
|
||
"args": "after-used",
|
||
"argsIgnorePattern": "^_"
|
||
}
|
||
],
|
||
"semi": "off",
|
||
"@typescript-eslint/semi": [
|
||
"error",
|
||
"never"
|
||
],
|
||
"no-extra-semi": "off"
|
||
}
|
||
}
|
||
}
|