feat: enforce data/metadata separation, numeric range queries, improved docs
- Store data opaquely in add() and update() instead of spreading object properties into top-level metadata. data is for semantic search (HNSW), metadata is for structured where-filter queries (MetadataIndex). - Fix numeric range queries in MetadataIndex — use numeric-aware comparison instead of lexicographic string comparison for normalized values. - Add data field to RelateParams and Relation types for relationship content. - Add where.type → where.noun alias in metadata-only find() path. - Rewrite README: focused ~350 lines from 791, quick start first, feature showcase with mini-snippets, organized doc links, no version callouts. - Add DATA_MODEL.md and QUERY_OPERATORS.md reference docs. - Remove 10 outdated/redundant doc files consolidated into API reference. - Improve JSDoc on Entity, Relation, AddParams, FindParams, and core methods. - Fix tests asserting data properties appear in metadata (data model violation). - Deprecate verb.source/target in favor of from/to (public) and sourceId/targetId (storage).
This commit is contained in:
parent
edb5ec4696
commit
0ddc05a5bb
30 changed files with 1356 additions and 7001 deletions
249
src/brainy.ts
249
src/brainy.ts
|
|
@ -672,12 +672,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
/**
|
||||
* Add an entity to the database
|
||||
*
|
||||
* **Data vs Metadata:**
|
||||
* - `data`: Content used for vector embeddings. Searchable via **semantic similarity**
|
||||
* (HNSW vector index). NOT queryable via `where` filters. Pass a string for text
|
||||
* embedding, or any value for opaque storage.
|
||||
* - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where`
|
||||
* filters in `find()`. Put anything you want to filter/query on here (tags,
|
||||
* categories, dates, flags, etc.).
|
||||
*
|
||||
* @param params - Parameters for adding the entity
|
||||
* @param params.data - Content to embed and store (required)
|
||||
* @param params.data - Content to embed and store (required). Strings are auto-embedded.
|
||||
* @param params.type - NounType classification (required)
|
||||
* @param params.metadata - Custom metadata object
|
||||
* @param params.id - Custom ID (auto-generated if not provided)
|
||||
* @param params.vector - Pre-computed embedding vector
|
||||
* @param params.metadata - Custom queryable metadata (indexed, used in where filters)
|
||||
* @param params.id - Custom ID (auto-generated UUID if not provided)
|
||||
* @param params.vector - Pre-computed embedding vector (skips auto-embedding)
|
||||
* @param params.service - Service name for multi-tenancy
|
||||
* @param params.confidence - Type classification confidence (0-1)
|
||||
* @param params.weight - Entity importance/salience (0-1)
|
||||
|
|
@ -756,16 +764,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
}
|
||||
|
||||
// Prepare metadata for storage (backward compat format - unchanged)
|
||||
// Prepare metadata for storage
|
||||
// data is stored opaquely in the 'data' field - NOT spread into top-level metadata.
|
||||
// Only metadata fields are queryable via find({ where }).
|
||||
const storageMetadata = {
|
||||
...(typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data) ? params.data : {}),
|
||||
...params.metadata,
|
||||
data: params.data, // Store the raw data in metadata
|
||||
data: params.data,
|
||||
noun: params.type,
|
||||
service: params.service,
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now(),
|
||||
// Preserve confidence and weight if provided
|
||||
...(params.confidence !== undefined && { confidence: params.confidence }),
|
||||
...(params.weight !== undefined && { weight: params.weight }),
|
||||
...(params.createdBy && { createdBy: params.createdBy })
|
||||
|
|
@ -1138,7 +1146,54 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Update an entity
|
||||
* Update an existing entity
|
||||
*
|
||||
* Merges metadata by default — new fields are added, existing fields are overwritten,
|
||||
* and omitted fields are preserved. Set `merge: false` to replace metadata entirely.
|
||||
* If `data` is provided, the entity is re-embedded and re-indexed in HNSW.
|
||||
*
|
||||
* **Data vs Metadata:**
|
||||
* - `data`: Content used for vector embeddings (searchable via semantic similarity / HNSW).
|
||||
* NOT queryable via `where` filters. Pass a string for text search, or any value for storage.
|
||||
* - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where` filters
|
||||
* in `find()`. Put anything you want to filter/query on here.
|
||||
*
|
||||
* @param params - Update parameters
|
||||
* @param params.id - UUID of the entity to update (required)
|
||||
* @param params.data - New content to re-embed (triggers HNSW re-indexing)
|
||||
* @param params.type - Change entity type classification
|
||||
* @param params.metadata - Metadata fields to merge (or replace if merge=false)
|
||||
* @param params.merge - If true (default), merges metadata; if false, replaces it entirely
|
||||
* @param params.vector - Pre-computed vector (skips embedding)
|
||||
* @param params.confidence - Update type classification confidence (0-1)
|
||||
* @param params.weight - Update entity importance/salience (0-1)
|
||||
*
|
||||
* @example Update metadata (merge by default)
|
||||
* ```typescript
|
||||
* await brain.update({
|
||||
* id: entityId,
|
||||
* metadata: { status: 'reviewed', rating: 4.5 }
|
||||
* // Existing metadata fields preserved, only status and rating changed
|
||||
* })
|
||||
* ```
|
||||
*
|
||||
* @example Update data (re-embeds and re-indexes)
|
||||
* ```typescript
|
||||
* await brain.update({
|
||||
* id: entityId,
|
||||
* data: 'Updated description of the concept'
|
||||
* // Vector is recomputed, HNSW index updated
|
||||
* })
|
||||
* ```
|
||||
*
|
||||
* @example Replace metadata entirely
|
||||
* ```typescript
|
||||
* await brain.update({
|
||||
* id: entityId,
|
||||
* metadata: { onlyThisField: true },
|
||||
* merge: false // All previous metadata removed
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
async update(params: UpdateParams<T>): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -1168,16 +1223,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
? { ...existing.metadata, ...params.metadata }
|
||||
: params.metadata || existing.metadata
|
||||
|
||||
// Merge data objects if both old and new are objects
|
||||
const dataFields = typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data)
|
||||
? params.data
|
||||
: {}
|
||||
|
||||
// Prepare updated metadata object with data field
|
||||
// Prepare updated metadata object
|
||||
// data is stored opaquely in the 'data' field - NOT spread into top-level metadata.
|
||||
const updatedMetadata = {
|
||||
...newMetadata,
|
||||
...dataFields,
|
||||
data: params.data !== undefined ? params.data : existing.data, // Store data field
|
||||
data: params.data !== undefined ? params.data : existing.data,
|
||||
noun: params.type || existing.type,
|
||||
service: existing.service,
|
||||
createdAt: existing.createdAt,
|
||||
|
|
@ -1269,7 +1319,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Delete an entity
|
||||
* Delete an entity and all its relationships
|
||||
*
|
||||
* Removes the entity from all indexes (HNSW vector index, MetadataIndex,
|
||||
* GraphAdjacencyIndex) and deletes all relationships where this entity
|
||||
* is the source or target. All operations are executed atomically.
|
||||
*
|
||||
* @param id - UUID of the entity to delete. Silently returns for invalid/null IDs.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* await brain.delete(entityId)
|
||||
* const entity = await brain.get(entityId) // null
|
||||
* ```
|
||||
*/
|
||||
async delete(id: string): Promise<void> {
|
||||
// Handle invalid IDs gracefully
|
||||
|
|
@ -1324,9 +1386,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// ============= RELATIONSHIP OPERATIONS =============
|
||||
|
||||
/**
|
||||
* Create a relationship between entities
|
||||
* Create a relationship (verb) between two entities
|
||||
*
|
||||
* Relationships connect entities with typed edges. Duplicate relationships
|
||||
* (same from, to, and type) are detected and return the existing ID.
|
||||
*
|
||||
* **Data vs Metadata (on relationships):**
|
||||
* - `data`: Opaque content stored on the relationship (e.g., a description or
|
||||
* context for the edge). Overrides the auto-computed vector for this verb.
|
||||
* - `metadata`: Structured queryable fields on the edge (e.g., role, startDate).
|
||||
*
|
||||
* @param params - Parameters for creating the relationship
|
||||
* @param params.from - Source entity ID (required)
|
||||
* @param params.to - Target entity ID (required)
|
||||
* @param params.type - VerbType classification (required)
|
||||
* @param params.weight - Connection strength 0-1 (default: 1.0)
|
||||
* @param params.data - Content for the relationship (optional, overrides auto-computed vector)
|
||||
* @param params.metadata - Structured queryable fields on the edge
|
||||
* @param params.bidirectional - Create reverse edge too (default: false)
|
||||
* @param params.service - Multi-tenancy service name
|
||||
* @param params.confidence - Relationship certainty 0-1
|
||||
* @param params.evidence - Why this relationship exists
|
||||
* @returns Promise that resolves to the relationship ID
|
||||
*
|
||||
* @example
|
||||
|
|
@ -1474,12 +1554,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
|
||||
// Prepare verb metadata
|
||||
// CRITICAL: Include verb type in metadata for count tracking
|
||||
// User metadata spread FIRST, then system fields ALWAYS win (prevents collision)
|
||||
const verbMetadata = {
|
||||
verb: params.type, // Store verb type for count synchronization
|
||||
weight: params.weight ?? 1.0,
|
||||
...(params.metadata || {}),
|
||||
createdAt: Date.now()
|
||||
verb: params.type,
|
||||
weight: params.weight ?? 1.0,
|
||||
createdAt: Date.now(),
|
||||
...((params as any).data !== undefined && { data: (params as any).data })
|
||||
}
|
||||
|
||||
// Save to storage (vector and metadata separately)
|
||||
|
|
@ -1494,6 +1575,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
type: params.type,
|
||||
weight: params.weight ?? 1.0,
|
||||
metadata: params.metadata,
|
||||
data: (params as any).data,
|
||||
createdAt: Date.now()
|
||||
}
|
||||
|
||||
|
|
@ -1529,9 +1611,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
id: reverseId,
|
||||
sourceId: params.to,
|
||||
targetId: params.from,
|
||||
source: toEntity.type,
|
||||
target: fromEntity.type
|
||||
} as any
|
||||
source: params.to,
|
||||
target: params.from
|
||||
}
|
||||
|
||||
// Operation 4: Save reverse verb vector data
|
||||
tx.addOperation(
|
||||
|
|
@ -1561,7 +1643,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Delete a relationship
|
||||
* Delete a relationship (verb) by its ID
|
||||
*
|
||||
* Removes the relationship from the GraphAdjacencyIndex and deletes
|
||||
* the verb metadata from storage. Executed atomically.
|
||||
*
|
||||
* @param id - UUID of the relationship to delete
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const relId = await brain.relate({
|
||||
* from: personId, to: projectId, type: VerbType.WorksOn
|
||||
* })
|
||||
* await brain.unrelate(relId) // Relationship removed
|
||||
* ```
|
||||
*/
|
||||
async unrelate(id: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -1678,6 +1773,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* Unified find method - supports natural language and structured queries
|
||||
* Implements Triple Intelligence with parallel search optimization
|
||||
*
|
||||
* Combines three search dimensions in one query:
|
||||
* - **Vector (semantic):** `query` string is embedded and matched via HNSW similarity
|
||||
* - **Metadata:** `where` filters query the MetadataIndex (exact/range/set operators)
|
||||
* - **Graph:** `connected` traverses relationships via GraphAdjacencyIndex
|
||||
*
|
||||
* `data` is searchable via semantic/hybrid vector search (the `query` parameter).
|
||||
* `metadata` is searchable via structured `where` filters.
|
||||
* `type` in FindParams is an alias for filtering by `where.noun` (entity type).
|
||||
*
|
||||
* @param query - Natural language string or structured FindParams object
|
||||
* @returns Promise that resolves to array of search results with scores
|
||||
*
|
||||
|
|
@ -1882,7 +1986,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) {
|
||||
// Build filter for metadata index
|
||||
let filter: any = {}
|
||||
if (params.where) Object.assign(filter, params.where)
|
||||
if (params.where) {
|
||||
Object.assign(filter, params.where)
|
||||
// Alias: where.type → where.noun (storage field name for entity type)
|
||||
if ('type' in filter && !('noun' in filter)) {
|
||||
filter.noun = filter.type
|
||||
delete filter.type
|
||||
}
|
||||
}
|
||||
if (params.service) filter.service = params.service
|
||||
|
||||
if (params.type) {
|
||||
|
|
@ -1998,7 +2109,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
if (params.where || params.type || params.service || params.excludeVFS) {
|
||||
preResolvedFilter = {}
|
||||
if (params.where) Object.assign(preResolvedFilter, params.where)
|
||||
if (params.where) {
|
||||
Object.assign(preResolvedFilter, params.where)
|
||||
// Alias: where.type → where.noun (storage field name for entity type)
|
||||
if ('type' in preResolvedFilter && !('noun' in preResolvedFilter)) {
|
||||
preResolvedFilter.noun = preResolvedFilter.type
|
||||
delete preResolvedFilter.type
|
||||
}
|
||||
}
|
||||
if (params.service) preResolvedFilter.service = params.service
|
||||
if (params.excludeVFS === true) {
|
||||
preResolvedFilter.vfsType = { exists: false }
|
||||
|
|
@ -2386,11 +2504,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// ============= BATCH OPERATIONS =============
|
||||
|
||||
/**
|
||||
* Add multiple entities
|
||||
* Add multiple entities in a single batch operation
|
||||
*
|
||||
* Performance optimization: Uses batch embedding (embedBatch) to pre-compute
|
||||
* all vectors in a single WASM forward pass instead of N individual embed() calls.
|
||||
* This provides 5-10x speedup on bulk inserts.
|
||||
* Uses batch embedding (embedBatch) to pre-compute all vectors in a single
|
||||
* WASM forward pass instead of N individual embed() calls, providing 5-10x
|
||||
* speedup on bulk inserts. Automatically adapts batch size and parallelism
|
||||
* to the storage adapter (e.g., smaller batches for cloud storage).
|
||||
*
|
||||
* @param params - Batch add parameters
|
||||
* @param params.items - Array of AddParams (same shape as brain.add())
|
||||
* @param params.parallel - Process in parallel (default: true, auto-adapts per storage)
|
||||
* @param params.chunkSize - Batch size per storage round-trip (default: auto from storage)
|
||||
* @param params.onProgress - Callback: (completed, total) => void
|
||||
* @param params.continueOnError - If true, skip failed items instead of aborting
|
||||
* @returns BatchResult with successful (string[] of IDs), failed, total, duration
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const result = await brain.addMany({
|
||||
* items: [
|
||||
* { data: 'First entity', type: NounType.Document, metadata: { priority: 1 } },
|
||||
* { data: 'Second entity', type: NounType.Concept }
|
||||
* ],
|
||||
* onProgress: (done, total) => console.log(`${done}/${total}`)
|
||||
* })
|
||||
* console.log(`Added ${result.successful.length}, failed ${result.failed.length}`)
|
||||
* ```
|
||||
*/
|
||||
async addMany(params: AddManyParams<T>): Promise<BatchResult<string>> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -2682,7 +2821,28 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Create multiple relationships with batch processing
|
||||
* Create multiple relationships in a single batch operation
|
||||
*
|
||||
* Automatically adapts batch size and parallelism to the storage adapter.
|
||||
* Duplicate relationships are detected and deduplicated per relate().
|
||||
*
|
||||
* @param params - Batch relate parameters
|
||||
* @param params.items - Array of RelateParams (same shape as brain.relate())
|
||||
* @param params.parallel - Process in parallel (default: true, auto-adapts per storage)
|
||||
* @param params.chunkSize - Batch size per storage round-trip (default: auto from storage)
|
||||
* @param params.onProgress - Callback: (completed, total) => void
|
||||
* @param params.continueOnError - If true, skip failed items instead of aborting
|
||||
* @returns Array of relationship IDs (string[])
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const ids = await brain.relateMany({
|
||||
* items: [
|
||||
* { from: id1, to: id2, type: VerbType.RelatedTo },
|
||||
* { from: id1, to: id3, type: VerbType.Contains, data: 'section content' }
|
||||
* ]
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
async relateMany(params: RelateManyParams<T>): Promise<string[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -4436,7 +4596,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// For complex queries, use metadata index for efficient counting
|
||||
if (params.where || params.service) {
|
||||
let filter: any = {}
|
||||
if (params.where) Object.assign(filter, params.where)
|
||||
if (params.where) {
|
||||
Object.assign(filter, params.where)
|
||||
// Alias: where.type → where.noun (storage field name for entity type)
|
||||
if ('type' in filter && !('noun' in filter)) {
|
||||
filter.noun = filter.type
|
||||
delete filter.type
|
||||
}
|
||||
}
|
||||
if (params.service) filter.service = params.service
|
||||
if (params.type) {
|
||||
const types = Array.isArray(params.type) ? params.type : [params.type]
|
||||
|
|
@ -4494,7 +4661,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
if (filter?.type || filter?.where || filter?.service) {
|
||||
// Use MetadataIndexManager for efficient filtered streaming
|
||||
let filterObj: any = {}
|
||||
if (filter.where) Object.assign(filterObj, filter.where)
|
||||
if (filter.where) {
|
||||
Object.assign(filterObj, filter.where)
|
||||
// Alias: where.type → where.noun (storage field name for entity type)
|
||||
if ('type' in filterObj && !('noun' in filterObj)) {
|
||||
filterObj.noun = filterObj.type
|
||||
delete filterObj.type
|
||||
}
|
||||
}
|
||||
if (filter.service) filterObj.service = filter.service
|
||||
if (filter.type) {
|
||||
const types = Array.isArray(filter.type) ? filter.type : [filter.type]
|
||||
|
|
@ -6011,7 +6185,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
from: v.sourceId,
|
||||
to: v.targetId,
|
||||
type: (v.verb || v.type) as VerbType,
|
||||
weight: v.weight ?? 1.0, // Weight is at top-level
|
||||
weight: v.weight ?? 1.0,
|
||||
data: v.data,
|
||||
metadata: v.metadata,
|
||||
service: v.service as string,
|
||||
createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now()
|
||||
|
|
|
|||
|
|
@ -290,9 +290,11 @@ export interface GraphVerb {
|
|||
metadata?: any // Optional metadata for the verb
|
||||
service?: string // Multi-tenancy support - which service created this verb
|
||||
|
||||
// Additional properties used in the codebase
|
||||
source?: string // Entity UUID (same as sourceId, for graphTypes compatibility)
|
||||
target?: string // Entity UUID (same as targetId, for graphTypes compatibility)
|
||||
// Legacy field names (use from/to in public API, sourceId/targetId in storage)
|
||||
/** @deprecated Use `from` (public API) or `sourceId` (storage). Will be removed in next major. */
|
||||
source?: string // Entity UUID (same as sourceId)
|
||||
/** @deprecated Use `to` (public API) or `targetId` (storage). Will be removed in next major. */
|
||||
target?: string // Entity UUID (same as targetId)
|
||||
verb?: string // Alias for type
|
||||
data?: Record<string, any> // Additional flexible data storage
|
||||
embedding?: Vector // Alias for vector
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: 2026-01-27T21:46:03.408Z
|
||||
* Generated: 2026-02-09T16:59:48.867Z
|
||||
* Noun Types: 42
|
||||
* Verb Types: 127
|
||||
*
|
||||
|
|
@ -19,7 +19,7 @@ export const TYPE_METADATA = {
|
|||
verbTypes: 127,
|
||||
totalTypes: 169,
|
||||
embeddingDimensions: 384,
|
||||
generatedAt: "2026-01-27T21:46:03.408Z",
|
||||
generatedAt: "2026-02-09T16:59:48.867Z",
|
||||
sizeBytes: {
|
||||
embeddings: 259584,
|
||||
base64: 346112
|
||||
|
|
|
|||
|
|
@ -10,39 +10,72 @@ import { NounType, VerbType } from './graphTypes.js'
|
|||
// ============= Core Types =============
|
||||
|
||||
/**
|
||||
* Entity representation (replaces GraphNoun)
|
||||
* Entity (Noun) — the fundamental data unit in Brainy
|
||||
*
|
||||
* **Data vs Metadata:**
|
||||
* - `data`: Content used for vector embeddings. Searchable via **semantic similarity**
|
||||
* (HNSW index) and hybrid text+semantic search. NOT queryable via `where` filters.
|
||||
* - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where`
|
||||
* filters in `find()`. Standard system fields (noun, createdAt, etc.) are stored
|
||||
* alongside user metadata but extracted to top-level Entity fields on read.
|
||||
*/
|
||||
export interface Entity<T = any> {
|
||||
/** Unique identifier (UUID v4) */
|
||||
id: string
|
||||
/** Embedding vector (384-dim by default). Empty array when loaded without includeVectors. */
|
||||
vector: Vector
|
||||
/** Entity type classification (NounType enum) */
|
||||
type: NounType
|
||||
/** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */
|
||||
data?: any
|
||||
/** User-defined structured fields — indexed and queryable via `where` filters. */
|
||||
metadata?: T
|
||||
/** Multi-tenancy service identifier */
|
||||
service?: string
|
||||
/** Creation timestamp (ms since epoch) */
|
||||
createdAt: number
|
||||
/** Last update timestamp (ms since epoch) */
|
||||
updatedAt?: number
|
||||
/** Source that created this entity (e.g., augmentation info) */
|
||||
createdBy?: string
|
||||
confidence?: number // Type classification confidence (0-1)
|
||||
weight?: number // Entity importance/salience (0-1)
|
||||
/** Type classification confidence (0-1) */
|
||||
confidence?: number
|
||||
/** Entity importance/salience (0-1) */
|
||||
weight?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Relation representation (replaces GraphVerb)
|
||||
* Enhanced with confidence scoring and evidence tracking
|
||||
* Relation (Verb) — a typed edge connecting two entities
|
||||
*
|
||||
* **Data vs Metadata (on relationships):**
|
||||
* - `data`: Opaque content stored on the relationship. If provided during relate(),
|
||||
* overrides the auto-computed vector (default: average of source+target vectors).
|
||||
* - `metadata`: Structured queryable fields on the edge (e.g., role, startDate).
|
||||
*/
|
||||
export interface Relation<T = any> {
|
||||
/** Unique identifier (UUID v4) */
|
||||
id: string
|
||||
/** Source entity ID */
|
||||
from: string
|
||||
/** Target entity ID */
|
||||
to: string
|
||||
/** Relationship type classification (VerbType enum) */
|
||||
type: VerbType
|
||||
/** Connection strength (0-1, default: 1.0) */
|
||||
weight?: number
|
||||
/** Opaque content for the relationship (overrides auto-computed vector if provided) */
|
||||
data?: any
|
||||
/** User-defined structured fields on the edge */
|
||||
metadata?: T
|
||||
/** Multi-tenancy service identifier */
|
||||
service?: string
|
||||
/** Creation timestamp (ms since epoch) */
|
||||
createdAt: number
|
||||
/** Last update timestamp (ms since epoch) */
|
||||
updatedAt?: number
|
||||
|
||||
// NEW: Confidence and evidence (optional for backward compatibility)
|
||||
confidence?: number // 0-1 score indicating relationship certainty
|
||||
/** Relationship certainty (0-1) */
|
||||
confidence?: number
|
||||
/** Evidence for why this relationship was detected */
|
||||
evidence?: RelationEvidence
|
||||
}
|
||||
|
||||
|
|
@ -106,17 +139,31 @@ export interface ScoreExplanation {
|
|||
|
||||
/**
|
||||
* Parameters for adding entities
|
||||
*
|
||||
* **Data vs Metadata:**
|
||||
* - `data` is embedded into a vector and searchable via semantic similarity (HNSW).
|
||||
* It is NOT indexed by MetadataIndex and NOT queryable via `where` filters.
|
||||
* - `metadata` is indexed by MetadataIndex and queryable via `where` filters in `find()`.
|
||||
*/
|
||||
export interface AddParams<T = any> {
|
||||
data: any | Vector // Content to embed or pre-computed vector
|
||||
type: NounType // Entity type from enum
|
||||
metadata?: T // Optional metadata
|
||||
id?: string // Optional custom ID
|
||||
vector?: Vector // Pre-computed vector (skip embedding)
|
||||
service?: string // Multi-tenancy support
|
||||
confidence?: number // Type classification confidence (0-1)
|
||||
weight?: number // Entity importance/salience (0-1)
|
||||
createdBy?: { augmentation: string; version: string } // Track entity source
|
||||
/** Content to embed and store. Strings are auto-embedded; objects are JSON-stringified for embedding. */
|
||||
data: any | Vector
|
||||
/** Entity type classification (required) */
|
||||
type: NounType
|
||||
/** Structured queryable fields — indexed by MetadataIndex, used in `where` filters */
|
||||
metadata?: T
|
||||
/** Custom entity ID (auto-generated UUID v4 if not provided) */
|
||||
id?: string
|
||||
/** Pre-computed embedding vector (skips auto-embedding when provided) */
|
||||
vector?: Vector
|
||||
/** Multi-tenancy service identifier */
|
||||
service?: string
|
||||
/** Type classification confidence (0-1) */
|
||||
confidence?: number
|
||||
/** Entity importance/salience (0-1) */
|
||||
weight?: number
|
||||
/** Track which augmentation created this entity */
|
||||
createdBy?: { augmentation: string; version: string }
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -135,20 +182,33 @@ export interface UpdateParams<T = any> {
|
|||
|
||||
/**
|
||||
* Parameters for creating relationships
|
||||
* Enhanced with confidence scoring and evidence tracking
|
||||
*
|
||||
* **Data vs Metadata (on relationships):**
|
||||
* - `data`: Opaque content for the edge. If provided, overrides the auto-computed
|
||||
* vector (default: average of source+target entity vectors).
|
||||
* - `metadata`: Structured queryable fields on the edge.
|
||||
*/
|
||||
export interface RelateParams<T = any> {
|
||||
from: string // Source entity ID
|
||||
to: string // Target entity ID
|
||||
type: VerbType // Relationship type from enum
|
||||
weight?: number // Connection strength (0-1, default: 1)
|
||||
metadata?: T // Edge metadata
|
||||
bidirectional?: boolean // Create reverse edge too
|
||||
service?: string // Multi-tenancy
|
||||
|
||||
// NEW: Confidence and evidence (optional)
|
||||
confidence?: number // Relationship certainty (0-1)
|
||||
evidence?: RelationEvidence // Why this relationship exists
|
||||
/** Source entity ID (required — must exist) */
|
||||
from: string
|
||||
/** Target entity ID (required — must exist) */
|
||||
to: string
|
||||
/** Relationship type classification (required) */
|
||||
type: VerbType
|
||||
/** Connection strength (0-1, default: 1.0) */
|
||||
weight?: number
|
||||
/** Content for the relationship (optional — overrides auto-computed vector) */
|
||||
data?: any
|
||||
/** Structured queryable fields on the edge */
|
||||
metadata?: T
|
||||
/** Create reverse edge too (default: false) */
|
||||
bidirectional?: boolean
|
||||
/** Multi-tenancy service identifier */
|
||||
service?: string
|
||||
/** Relationship certainty (0-1) */
|
||||
confidence?: number
|
||||
/** Evidence for why this relationship exists */
|
||||
evidence?: RelationEvidence
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -157,6 +217,7 @@ export interface RelateParams<T = any> {
|
|||
export interface UpdateRelationParams<T = any> {
|
||||
id: string // Relation to update
|
||||
weight?: number // New weight
|
||||
data?: any // New content
|
||||
metadata?: Partial<T> // Metadata to update
|
||||
merge?: boolean // Merge or replace metadata
|
||||
}
|
||||
|
|
@ -164,16 +225,27 @@ export interface UpdateRelationParams<T = any> {
|
|||
// ============= Query Parameters =============
|
||||
|
||||
/**
|
||||
* Unified find parameters - Triple Intelligence
|
||||
* Unified find parameters — Triple Intelligence search
|
||||
*
|
||||
* Combines three search dimensions in one query:
|
||||
* - **Vector:** `query` or `vector` for semantic/hybrid similarity search (searches `data`)
|
||||
* - **Metadata:** `where` for structured field filters (queries `metadata` via MetadataIndex)
|
||||
* - **Graph:** `connected` for relationship traversal (via GraphAdjacencyIndex)
|
||||
*
|
||||
* See also: [Query Operators](../../docs/QUERY_OPERATORS.md) for all `where` operators.
|
||||
*/
|
||||
export interface FindParams<T = any> {
|
||||
// Vector Intelligence
|
||||
query?: string // Natural language or semantic search
|
||||
vector?: Vector // Direct vector search
|
||||
|
||||
/** Natural language or semantic search query (embedded and matched via HNSW + text index) */
|
||||
query?: string
|
||||
/** Direct vector search (pre-computed embedding) */
|
||||
vector?: Vector
|
||||
|
||||
// Metadata Intelligence
|
||||
type?: NounType | NounType[] // Filter by entity type(s)
|
||||
where?: Partial<T> // Metadata filters
|
||||
/** Filter by entity type(s). Alias for `where.noun`. */
|
||||
type?: NounType | NounType[]
|
||||
/** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */
|
||||
where?: Partial<T>
|
||||
|
||||
// Graph Intelligence
|
||||
connected?: GraphConstraints
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
* This module defines a comprehensive, standardized set of noun and verb types
|
||||
* that can be used to model any kind of graph, semantic network, or data model.
|
||||
*
|
||||
* **Stage 3 Coverage**: 95% domain coverage with 40 noun types and 88 verb types
|
||||
* **Stage 3 Coverage**: 95% domain coverage with 42 noun types and 127 verb types
|
||||
*
|
||||
* ## Purpose and Design Philosophy
|
||||
*
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* - **Semantic**: Types carry meaning that can be used for reasoning and inference
|
||||
* - **Complete**: Covers foundational ontological primitives to advanced relationships
|
||||
*
|
||||
* ## Noun Types (40 Entities)
|
||||
* ## Noun Types (42 Entities)
|
||||
*
|
||||
* Noun types represent entities in the graph and are organized into categories:
|
||||
*
|
||||
|
|
@ -91,7 +91,7 @@
|
|||
* ### Meta-Level (1)
|
||||
* - **Relationship**: Relationships as first-class entities for meta-level reasoning
|
||||
*
|
||||
* ## Verb Types (88 Relationships)
|
||||
* ## Verb Types (127 Relationships)
|
||||
*
|
||||
* Verb types represent relationships between entities and are organized into categories:
|
||||
*
|
||||
|
|
@ -322,7 +322,7 @@
|
|||
* succeeds (use inverse of precedes), belongsTo (use inverse of owns),
|
||||
* createdBy (use inverse of creates), supervises (use inverse of reportsTo)
|
||||
*
|
||||
* **Net Change**: +9 nouns (31 → 40), +48 verbs (40 → 88) = +57 types total
|
||||
* **Net Change**: +11 nouns (31 → 42), +87 verbs (40 → 127) = +98 types total
|
||||
* **Coverage**: 60% → 95% (Stage 3)
|
||||
*/
|
||||
|
||||
|
|
@ -369,10 +369,12 @@ export interface GraphNoun {
|
|||
*/
|
||||
export interface GraphVerb {
|
||||
id: string // Unique identifier for the verb
|
||||
/** @deprecated Use `from` (public API) or `sourceId` (storage). Will be removed in next major. */
|
||||
source: string // Entity UUID of the source noun
|
||||
/** @deprecated Use `to` (public API) or `targetId` (storage). Will be removed in next major. */
|
||||
target: string // Entity UUID of the target noun
|
||||
sourceId?: string // Alias for source (coreTypes compatibility)
|
||||
targetId?: string // Alias for target (coreTypes compatibility)
|
||||
sourceId?: string // Entity UUID of the source noun (storage convention)
|
||||
targetId?: string // Entity UUID of the target noun (storage convention)
|
||||
label?: string // Optional descriptive label
|
||||
verb: VerbType // Type of relationship
|
||||
createdAt: Timestamp | number // When the verb was created
|
||||
|
|
@ -562,7 +564,7 @@ export interface Relationship extends GraphNoun {
|
|||
}
|
||||
|
||||
/**
|
||||
* Defines valid noun types for graph entities (Stage 3: 40 types)
|
||||
* Defines valid noun types for graph entities (Stage 3: 42 types)
|
||||
* Used for categorizing different types of nodes
|
||||
*/
|
||||
export const NounType = {
|
||||
|
|
@ -647,7 +649,7 @@ export const NounType = {
|
|||
export type NounType = (typeof NounType)[keyof typeof NounType]
|
||||
|
||||
/**
|
||||
* Defines valid verb types for relationships (Stage 3: 88 types)
|
||||
* Defines valid verb types for relationships (Stage 3: 127 types)
|
||||
* Used for categorizing different types of connections
|
||||
*/
|
||||
export const VerbType = {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ import {
|
|||
AdaptiveChunkingStrategy,
|
||||
ChunkData,
|
||||
ChunkDescriptor,
|
||||
ZoneMap
|
||||
ZoneMap,
|
||||
compareNormalizedValues
|
||||
} from './metadataIndexChunking.js'
|
||||
import { EntityIdMapper } from './entityIdMapper.js'
|
||||
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
|
||||
|
|
@ -852,15 +853,18 @@ export class MetadataIndexManager {
|
|||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||
if (chunk) {
|
||||
for (const [value, bitmap] of chunk.entries) {
|
||||
// Check if value is in range (both value and normalized bounds are now bucketed)
|
||||
// Check if value is in range using numeric-aware comparison
|
||||
// (normalizeValue converts numbers to strings, so we must compare numerically)
|
||||
let inRange = true
|
||||
|
||||
if (normalizedMin !== undefined) {
|
||||
inRange = inRange && (includeMin ? value >= normalizedMin : value > normalizedMin)
|
||||
const cmp = compareNormalizedValues(value, normalizedMin)
|
||||
inRange = inRange && (includeMin ? cmp >= 0 : cmp > 0)
|
||||
}
|
||||
|
||||
if (normalizedMax !== undefined) {
|
||||
inRange = inRange && (includeMax ? value <= normalizedMax : value < normalizedMax)
|
||||
const cmp = compareNormalizedValues(value, normalizedMax)
|
||||
inRange = inRange && (includeMax ? cmp <= 0 : cmp < 0)
|
||||
}
|
||||
|
||||
if (inRange) {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,31 @@ import { prodLog } from './logger.js'
|
|||
import { RoaringBitmap32 } from './roaring/index.js'
|
||||
import type { EntityIdMapper } from './entityIdMapper.js'
|
||||
|
||||
// ============================================================================
|
||||
// Numeric-Aware Comparison
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Compare two normalized string values with numeric awareness.
|
||||
* Since normalizeValue() converts numbers to strings (e.g., 50 → "50"),
|
||||
* plain string comparison breaks numeric ordering ("50" > "100" is true
|
||||
* lexicographically but wrong numerically). This function detects when
|
||||
* both values are numeric strings and compares them as numbers.
|
||||
*
|
||||
* @returns negative if a < b, 0 if equal, positive if a > b
|
||||
*/
|
||||
export function compareNormalizedValues(a: string, b: string): number {
|
||||
const numA = Number(a)
|
||||
const numB = Number(b)
|
||||
if (!isNaN(numA) && !isNaN(numB) && a !== '' && b !== '') {
|
||||
return numA - numB
|
||||
}
|
||||
// Fall back to string comparison for non-numeric values
|
||||
if (a < b) return -1
|
||||
if (a > b) return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Core Data Structures
|
||||
// ============================================================================
|
||||
|
|
@ -350,15 +375,10 @@ export class SparseIndex {
|
|||
return zoneMap.hasNulls
|
||||
}
|
||||
|
||||
// Handle different types
|
||||
if (typeof value === 'number') {
|
||||
return value >= zoneMap.min && value <= zoneMap.max
|
||||
} else if (typeof value === 'string') {
|
||||
return value >= zoneMap.min && value <= zoneMap.max
|
||||
} else {
|
||||
// For other types, conservatively check
|
||||
return true
|
||||
}
|
||||
const strValue = String(value)
|
||||
const strMin = String(zoneMap.min)
|
||||
const strMax = String(zoneMap.max)
|
||||
return compareNormalizedValues(strValue, strMin) >= 0 && compareNormalizedValues(strValue, strMax) <= 0
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -375,16 +395,19 @@ export class SparseIndex {
|
|||
return true
|
||||
}
|
||||
|
||||
// Check overlap
|
||||
// Check overlap using numeric-aware comparison
|
||||
const strZoneMin = String(zoneMap.min)
|
||||
const strZoneMax = String(zoneMap.max)
|
||||
|
||||
if (min !== undefined && max !== undefined) {
|
||||
// Range: [min, max] overlaps with [zoneMin, zoneMax]
|
||||
return !(max < zoneMap.min || min > zoneMap.max)
|
||||
return !(compareNormalizedValues(String(max), strZoneMin) < 0 || compareNormalizedValues(String(min), strZoneMax) > 0)
|
||||
} else if (min !== undefined) {
|
||||
// >= min
|
||||
return zoneMap.max >= min
|
||||
return compareNormalizedValues(strZoneMax, String(min)) >= 0
|
||||
} else if (max !== undefined) {
|
||||
// <= max
|
||||
return zoneMap.min <= max
|
||||
return compareNormalizedValues(strZoneMin, String(max)) <= 0
|
||||
}
|
||||
|
||||
return true
|
||||
|
|
@ -454,13 +477,7 @@ export class SparseIndex {
|
|||
*/
|
||||
private sortChunks(): void {
|
||||
this.data.chunks.sort((a, b) => {
|
||||
// Handle different types
|
||||
if (typeof a.zoneMap.min === 'number' && typeof b.zoneMap.min === 'number') {
|
||||
return a.zoneMap.min - b.zoneMap.min
|
||||
} else if (typeof a.zoneMap.min === 'string' && typeof b.zoneMap.min === 'string') {
|
||||
return a.zoneMap.min.localeCompare(b.zoneMap.min)
|
||||
}
|
||||
return 0
|
||||
return compareNormalizedValues(String(a.zoneMap.min), String(b.zoneMap.min))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -716,8 +733,8 @@ export class ChunkManager {
|
|||
if (value === '__NULL__' || value === null || value === undefined) {
|
||||
hasNulls = true
|
||||
} else {
|
||||
if (value < min) min = value
|
||||
if (value > max) max = value
|
||||
if (compareNormalizedValues(value, min) < 0) min = value
|
||||
if (compareNormalizedValues(value, max) > 0) max = value
|
||||
}
|
||||
|
||||
// Get count from roaring bitmap
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ export function validateAddParams(params: AddParams): void {
|
|||
throw new Error(
|
||||
`Invalid NounType: '${params.type}'\n` +
|
||||
`\nValid types: ${Object.values(NounType).join(', ')}\n` +
|
||||
`\nExample: await brain.add({ data: 'text', type: NounType.Note })`
|
||||
`\nExample: await brain.add({ data: 'text', type: NounType.Document })`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue