feat: migrate embeddings to Candle WASM + remove semantic type inference
Major architectural changes: 1. EMBEDDINGS ENGINE (ONNX → Candle WASM): - Replace ONNX Runtime with Rust Candle compiled to WASM - Embedded model in WASM binary (no external downloads) - Quantized Q8 precision with <50MB memory footprint - Zero-download, offline-first operation - Same embedding quality (all-MiniLM-L6-v2) 2. REMOVE SEMANTIC TYPE INFERENCE: - Delete embeddedKeywordEmbeddings.ts (14MB of pre-computed embeddings) - Remove typeAwareQueryPlanner.ts and semanticTypeInference.ts - Remove VerbExactMatchSignal (uses keyword embeddings) - Update SmartRelationshipExtractor to 3 signals (55%/30%/15% weights) API CHANGES (requires v7.0.0): - Removed: inferTypes(), inferNouns(), inferVerbs(), inferIntent() - Removed: getSemanticTypeInference(), SemanticTypeInference class - Removed: TypeInference, SemanticTypeInferenceOptions types Users can still use natural language queries in find() - they just need to specify type explicitly for type-optimized searches. PACKAGE SIZE IMPACT: - Compressed: 90.1 MB → 86.2 MB (-4.3%) - Uncompressed: 114.4 MB → 100.3 MB (-12%) - ~448K lines of code removed 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
81cd16e41b
commit
da7d2ed29d
60 changed files with 3887 additions and 448557 deletions
206
.github/workflows/ci.yml
vendored
Normal file
206
.github/workflows/ci.yml
vendored
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
NODE_VERSION: '20'
|
||||
RUST_VERSION: 'stable'
|
||||
|
||||
jobs:
|
||||
# Build and test TypeScript/JavaScript
|
||||
test-node:
|
||||
name: Test (Node.js)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build TypeScript
|
||||
run: npm run build
|
||||
|
||||
- name: Run unit tests
|
||||
run: npm run test:unit
|
||||
|
||||
- name: Run integration tests
|
||||
run: npm run test:integration
|
||||
|
||||
# Build and test with Bun
|
||||
test-bun:
|
||||
name: Test (Bun)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: latest
|
||||
|
||||
- name: Setup Node.js (for npm compatibility)
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build TypeScript
|
||||
run: npm run build
|
||||
|
||||
- name: Run Bun tests
|
||||
run: npm run test:bun
|
||||
|
||||
# Build Candle WASM (if Rust source changed)
|
||||
build-candle-wasm:
|
||||
name: Build Candle WASM
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-action@stable
|
||||
with:
|
||||
toolchain: ${{ env.RUST_VERSION }}
|
||||
targets: wasm32-unknown-unknown
|
||||
|
||||
- name: Install wasm-pack
|
||||
run: cargo install wasm-pack
|
||||
|
||||
- name: Cache Cargo registry
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
src/embeddings/candle-wasm/target
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
- name: Build WASM
|
||||
run: |
|
||||
cd src/embeddings/candle-wasm
|
||||
wasm-pack build --target web --release
|
||||
|
||||
- name: Upload WASM artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: candle-wasm
|
||||
path: src/embeddings/wasm/pkg/
|
||||
retention-days: 7
|
||||
|
||||
# Test Bun compile (standalone binary)
|
||||
test-bun-compile:
|
||||
name: Test Bun Compile
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build-candle-wasm]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: latest
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Download WASM artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: candle-wasm
|
||||
path: src/embeddings/wasm/pkg/
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build TypeScript
|
||||
run: npm run build
|
||||
|
||||
- name: Test Bun compile
|
||||
run: |
|
||||
# Create a test script
|
||||
cat > /tmp/test-compile.ts << 'EOF'
|
||||
import { Brainy } from './dist/index.js'
|
||||
|
||||
const brainy = new Brainy()
|
||||
await brainy.init()
|
||||
console.log('Brainy initialized!')
|
||||
|
||||
const embedding = await brainy.embed('Hello world')
|
||||
console.log(`Embedding dimension: ${embedding.length}`)
|
||||
|
||||
if (embedding.length !== 384) {
|
||||
throw new Error('Expected 384-dimensional embedding')
|
||||
}
|
||||
|
||||
console.log('Test passed!')
|
||||
EOF
|
||||
|
||||
# Compile to standalone binary
|
||||
bun build --compile /tmp/test-compile.ts --outfile /tmp/brainy-test
|
||||
|
||||
# Run the compiled binary
|
||||
/tmp/brainy-test
|
||||
|
||||
# Lint and type check
|
||||
lint:
|
||||
name: Lint & Type Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Lint
|
||||
run: npm run lint
|
||||
|
||||
# License check
|
||||
license-check:
|
||||
name: License Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Check licenses
|
||||
run: |
|
||||
# Install license checker
|
||||
npm install -g license-checker
|
||||
|
||||
# Check for problematic licenses
|
||||
license-checker --production --excludePrivatePackages \
|
||||
--failOn 'GPL;LGPL;AGPL;SSPL' \
|
||||
--summary
|
||||
Loading…
Add table
Add a link
Reference in a new issue