feat: add neural extraction APIs with NounType taxonomy
Add brain.extract() and brain.extractConcepts() methods that use NeuralEntityExtractor with embeddings and sophisticated NounType taxonomy (30+ entity types) for semantic entity and concept extraction. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
27cc699555
commit
dd50d89ad6
41 changed files with 3807 additions and 7391 deletions
83
src/vfs/semantic/projections/AuthorProjection.ts
Normal file
83
src/vfs/semantic/projections/AuthorProjection.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Author Projection Strategy
|
||||
*
|
||||
* Maps author-based paths to files owned by that author
|
||||
* Uses EXISTING MetadataIndexManager for O(log n) queries
|
||||
*/
|
||||
|
||||
import { Brainy } from '../../../brainy.js'
|
||||
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
|
||||
import { FindParams } from '../../../types/brainy.types.js'
|
||||
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
|
||||
import { VFSEntity } from '../../types.js'
|
||||
|
||||
/**
|
||||
* Author Projection: /by-author/<authorName>/<subpath>
|
||||
*
|
||||
* Uses EXISTING infrastructure:
|
||||
* - Brainy.find() with metadata filters (REAL)
|
||||
* - MetadataIndexManager for O(log n) owner queries (REAL)
|
||||
* - VFSMetadata.owner field (REAL - types.ts line 44)
|
||||
*/
|
||||
export class AuthorProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'author'
|
||||
|
||||
/**
|
||||
* Convert author name to Brainy FindParams
|
||||
*/
|
||||
toQuery(authorName: string, subpath?: string): FindParams {
|
||||
const query: FindParams = {
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
owner: authorName
|
||||
},
|
||||
limit: 1000
|
||||
}
|
||||
|
||||
// Filter by filename if subpath specified
|
||||
if (subpath) {
|
||||
query.where = {
|
||||
...query.where,
|
||||
anyOf: [ // BFO logical operator (not $or)
|
||||
{ name: subpath },
|
||||
{ path: { endsWith: subpath } } // BFO operator (not $regex)
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve author to entity IDs using REAL Brainy.find()
|
||||
*/
|
||||
async resolve(brain: Brainy, vfs: VirtualFileSystem, authorName: string): Promise<string[]> {
|
||||
// Use REAL Brainy metadata filtering
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
owner: authorName
|
||||
},
|
||||
limit: 1000
|
||||
})
|
||||
|
||||
return this.extractIds(results)
|
||||
}
|
||||
|
||||
/**
|
||||
* List all unique authors
|
||||
* Uses aggregation over metadata
|
||||
*/
|
||||
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
|
||||
// Get all files with owner metadata
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
owner: { $exists: true }
|
||||
},
|
||||
limit
|
||||
})
|
||||
|
||||
return results.map(r => r.entity as VFSEntity)
|
||||
}
|
||||
}
|
||||
97
src/vfs/semantic/projections/ConceptProjection.ts
Normal file
97
src/vfs/semantic/projections/ConceptProjection.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
/**
|
||||
* Concept Projection Strategy
|
||||
*
|
||||
* Maps concept-based paths to files containing those concepts
|
||||
* Uses EXISTING ConceptSystem and MetadataIndexManager
|
||||
*/
|
||||
|
||||
import { Brainy } from '../../../brainy.js'
|
||||
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
|
||||
import { FindParams } from '../../../types/brainy.types.js'
|
||||
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
|
||||
import { VFSEntity } from '../../types.js'
|
||||
|
||||
/**
|
||||
* Concept Projection: /by-concept/<conceptName>/<subpath>
|
||||
*
|
||||
* Uses EXISTING infrastructure:
|
||||
* - Brainy.find() with metadata filters (REAL - line 580 in brainy.ts)
|
||||
* - MetadataIndexManager for O(log n) concept queries (REAL)
|
||||
* - ConceptSystem for concept extraction (REAL - ConceptSystem.ts)
|
||||
*/
|
||||
export class ConceptProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'concept'
|
||||
|
||||
/**
|
||||
* Convert concept name to Brainy FindParams
|
||||
* Uses EXISTING FindParams.where for metadata filtering
|
||||
*
|
||||
* Now uses flattened conceptNames array for O(log n) performance!
|
||||
*/
|
||||
toQuery(conceptName: string, subpath?: string): FindParams {
|
||||
const query: FindParams = {
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
conceptNames: { contains: conceptName } // O(log n) indexed query
|
||||
},
|
||||
limit: 1000
|
||||
}
|
||||
|
||||
// If subpath specified, also filter by filename
|
||||
if (subpath) {
|
||||
query.where = {
|
||||
...query.where,
|
||||
anyOf: [ // BFO logical operator
|
||||
{ name: subpath },
|
||||
{ path: { endsWith: subpath } } // BFO operator
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve concept to entity IDs using REAL Brainy.find()
|
||||
* VERIFIED: brain.find() exists at line 580 in brainy.ts
|
||||
*
|
||||
* NOW OPTIMIZED: Uses flattened conceptNames for O(log n) indexed queries!
|
||||
* No more post-filtering - direct index lookup
|
||||
*/
|
||||
async resolve(brain: Brainy, vfs: VirtualFileSystem, conceptName: string): Promise<string[]> {
|
||||
// Verify brain.find is a function (safety check)
|
||||
if (typeof brain.find !== 'function') {
|
||||
throw new Error('VERIFICATION FAILED: brain.find is not a function')
|
||||
}
|
||||
|
||||
// Direct O(log n) query using flattened conceptNames array
|
||||
// VFS automatically flattens concepts to conceptNames on write
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
conceptNames: { contains: conceptName } // Indexed array query
|
||||
},
|
||||
limit: 1000
|
||||
})
|
||||
|
||||
return this.extractIds(results)
|
||||
}
|
||||
|
||||
/**
|
||||
* List all files with concept metadata
|
||||
* Uses REAL Brainy.find() with metadata filter
|
||||
*/
|
||||
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
conceptNames: { exists: true } // Use flattened field
|
||||
},
|
||||
limit
|
||||
})
|
||||
|
||||
// Convert to VFSEntity array
|
||||
// VERIFIED: Result.entity exists in brainy.types.ts
|
||||
return results.map(r => r.entity as VFSEntity)
|
||||
}
|
||||
}
|
||||
136
src/vfs/semantic/projections/RelationshipProjection.ts
Normal file
136
src/vfs/semantic/projections/RelationshipProjection.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
/**
|
||||
* Relationship Projection Strategy
|
||||
*
|
||||
* Maps relationship-based paths to files connected in the knowledge graph
|
||||
* Uses EXISTING GraphAdjacencyIndex for O(1) traversal
|
||||
*/
|
||||
|
||||
import { Brainy } from '../../../brainy.js'
|
||||
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
|
||||
import { FindParams } from '../../../types/brainy.types.js'
|
||||
import { VerbType } from '../../../types/graphTypes.js'
|
||||
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
|
||||
import { RelationshipValue } from '../SemanticPathParser.js'
|
||||
|
||||
/**
|
||||
* Relationship Projection: /related-to/<path>/depth-N/types-X,Y
|
||||
*
|
||||
* Uses EXISTING infrastructure:
|
||||
* - Brainy.getRelations() for graph traversal (REAL - line 803 in brainy.ts)
|
||||
* - GraphAdjacencyIndex for O(1) neighbor lookups (REAL)
|
||||
* - VerbType enum for relationship types (REAL - graphTypes.ts)
|
||||
*/
|
||||
export class RelationshipProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'relationship'
|
||||
|
||||
/**
|
||||
* Convert relationship value to Brainy FindParams
|
||||
* Note: Graph queries don't use FindParams, but we provide this for consistency
|
||||
*/
|
||||
toQuery(value: RelationshipValue, subpath?: string): FindParams {
|
||||
// This is informational - actual resolution uses getRelations()
|
||||
return {
|
||||
where: {
|
||||
vfsType: 'file'
|
||||
},
|
||||
connected: {
|
||||
to: value.targetPath,
|
||||
depth: value.depth || 1
|
||||
},
|
||||
limit: 1000
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve relationships using REAL Brainy.getRelations()
|
||||
* Uses GraphAdjacencyIndex for O(1) graph traversal
|
||||
*/
|
||||
async resolve(brain: Brainy, vfs: VirtualFileSystem, value: RelationshipValue): Promise<string[]> {
|
||||
// Step 1: Resolve target path to entity ID
|
||||
const targetId = await this.resolvePathToId(vfs, value.targetPath)
|
||||
if (!targetId) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Step 2: Get relationships using REAL Brainy graph
|
||||
const depth = value.depth || 1
|
||||
const visited = new Set<string>()
|
||||
const results: string[] = []
|
||||
|
||||
await this.traverseRelationships(
|
||||
brain,
|
||||
targetId,
|
||||
depth,
|
||||
visited,
|
||||
results,
|
||||
value.relationshipTypes
|
||||
)
|
||||
|
||||
// Filter to only files
|
||||
return await this.filterFiles(brain, results)
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursive graph traversal using REAL Brainy.getRelations()
|
||||
*/
|
||||
private async traverseRelationships(
|
||||
brain: Brainy,
|
||||
entityId: string,
|
||||
remainingDepth: number,
|
||||
visited: Set<string>,
|
||||
results: string[],
|
||||
types?: string[]
|
||||
): Promise<void> {
|
||||
if (remainingDepth <= 0 || visited.has(entityId)) {
|
||||
return
|
||||
}
|
||||
|
||||
visited.add(entityId)
|
||||
|
||||
// Get outgoing relationships (REAL method - line 803 in brainy.ts)
|
||||
const relations = await brain.getRelations({
|
||||
from: entityId,
|
||||
limit: 100
|
||||
})
|
||||
|
||||
for (const relation of relations) {
|
||||
// Filter by relationship type if specified
|
||||
if (types && types.length > 0) {
|
||||
const relationshipName = relation.type?.toLowerCase()
|
||||
if (!types.some(t => t.toLowerCase() === relationshipName)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Add to results
|
||||
if (!results.includes(relation.to)) {
|
||||
results.push(relation.to)
|
||||
}
|
||||
|
||||
// Recurse if depth remaining
|
||||
if (remainingDepth > 1) {
|
||||
await this.traverseRelationships(
|
||||
brain,
|
||||
relation.to,
|
||||
remainingDepth - 1,
|
||||
visited,
|
||||
results,
|
||||
types
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve path to entity ID
|
||||
* Helper to convert traditional path to entity ID
|
||||
*/
|
||||
private async resolvePathToId(vfs: VirtualFileSystem, path: string): Promise<string | null> {
|
||||
try {
|
||||
// Use REAL VFS public method
|
||||
return await vfs.resolvePath(path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
84
src/vfs/semantic/projections/SimilarityProjection.ts
Normal file
84
src/vfs/semantic/projections/SimilarityProjection.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Similarity Projection Strategy
|
||||
*
|
||||
* Maps similarity-based paths to files with similar content
|
||||
* Uses EXISTING HNSW Index for O(log n) vector similarity
|
||||
*/
|
||||
|
||||
import { Brainy } from '../../../brainy.js'
|
||||
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
|
||||
import { FindParams } from '../../../types/brainy.types.js'
|
||||
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
|
||||
import { SimilarityValue } from '../SemanticPathParser.js'
|
||||
|
||||
/**
|
||||
* Similarity Projection: /similar-to/<path>/threshold-N
|
||||
*
|
||||
* Uses EXISTING infrastructure:
|
||||
* - Brainy.similar() for vector similarity (REAL - line 680 in brainy.ts)
|
||||
* - HNSW Index for O(log n) nearest neighbor search (REAL)
|
||||
* - Cosine similarity for scoring (REAL)
|
||||
*/
|
||||
export class SimilarityProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'similar'
|
||||
|
||||
/**
|
||||
* Convert similarity value to Brainy FindParams
|
||||
* Note: Similarity uses brain.similar(), not find(), but we provide this for consistency
|
||||
*/
|
||||
toQuery(value: SimilarityValue, subpath?: string): FindParams {
|
||||
// This is informational - actual resolution uses brain.similar()
|
||||
return {
|
||||
where: {
|
||||
vfsType: 'file'
|
||||
},
|
||||
near: {
|
||||
id: value.targetPath,
|
||||
threshold: value.threshold || 0.7
|
||||
},
|
||||
limit: 50
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve similarity using REAL Brainy.similar()
|
||||
* Uses HNSW Index for O(log n) vector search
|
||||
*/
|
||||
async resolve(brain: Brainy, vfs: VirtualFileSystem, value: SimilarityValue): Promise<string[]> {
|
||||
// Step 1: Resolve target path to entity ID
|
||||
const targetId = await this.resolvePathToId(vfs, value.targetPath)
|
||||
if (!targetId) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Step 2: Get target entity to use its vector
|
||||
const targetEntity = await brain.get(targetId)
|
||||
if (!targetEntity) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Step 3: Find similar entities using REAL HNSW search
|
||||
// VERIFIED: brain.similar() exists at line 680 in brainy.ts
|
||||
const results = await brain.similar({
|
||||
to: targetEntity,
|
||||
threshold: value.threshold || 0.7,
|
||||
limit: 50,
|
||||
where: { vfsType: 'file' } // Only files
|
||||
})
|
||||
|
||||
// Extract IDs
|
||||
return this.extractIds(results)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve path to entity ID
|
||||
*/
|
||||
private async resolvePathToId(vfs: VirtualFileSystem, path: string): Promise<string | null> {
|
||||
try {
|
||||
// Use REAL VFS public method
|
||||
return await vfs.resolvePath(path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
82
src/vfs/semantic/projections/TagProjection.ts
Normal file
82
src/vfs/semantic/projections/TagProjection.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* Tag Projection Strategy
|
||||
*
|
||||
* Maps tag-based paths to files with those tags
|
||||
* Uses EXISTING MetadataIndexManager for O(log n) queries
|
||||
*/
|
||||
|
||||
import { Brainy } from '../../../brainy.js'
|
||||
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
|
||||
import { FindParams } from '../../../types/brainy.types.js'
|
||||
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
|
||||
import { VFSEntity } from '../../types.js'
|
||||
|
||||
/**
|
||||
* Tag Projection: /by-tag/<tagName>/<subpath>
|
||||
*
|
||||
* Uses EXISTING infrastructure:
|
||||
* - Brainy.find() with metadata filters (REAL)
|
||||
* - MetadataIndexManager for O(log n) tag queries (REAL)
|
||||
* - VFSMetadata.tags field (REAL - types.ts line 66)
|
||||
*/
|
||||
export class TagProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'tag'
|
||||
|
||||
/**
|
||||
* Convert tag name to Brainy FindParams
|
||||
*/
|
||||
toQuery(tagName: string, subpath?: string): FindParams {
|
||||
const query: FindParams = {
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
tags: { contains: tagName } // BFO operator for array contains
|
||||
},
|
||||
limit: 1000
|
||||
}
|
||||
|
||||
// Filter by filename if subpath specified
|
||||
if (subpath) {
|
||||
query.where = {
|
||||
...query.where,
|
||||
anyOf: [ // BFO logical operator
|
||||
{ name: subpath },
|
||||
{ path: { endsWith: subpath } } // BFO operator
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve tag to entity IDs using REAL Brainy.find()
|
||||
*/
|
||||
async resolve(brain: Brainy, vfs: VirtualFileSystem, tagName: string): Promise<string[]> {
|
||||
// Use REAL Brainy metadata filtering
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
tags: { contains: tagName } // BFO operator
|
||||
},
|
||||
limit: 1000
|
||||
})
|
||||
|
||||
return this.extractIds(results)
|
||||
}
|
||||
|
||||
/**
|
||||
* List all files with tags
|
||||
*/
|
||||
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
|
||||
// Get all files that have tags
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
tags: { exists: true } // BFO operator
|
||||
},
|
||||
limit
|
||||
})
|
||||
|
||||
return results.map(r => r.entity as VFSEntity)
|
||||
}
|
||||
}
|
||||
103
src/vfs/semantic/projections/TemporalProjection.ts
Normal file
103
src/vfs/semantic/projections/TemporalProjection.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* Temporal Projection Strategy
|
||||
*
|
||||
* Maps time-based paths to files modified at that time
|
||||
* Uses EXISTING MetadataIndexManager with range queries
|
||||
*/
|
||||
|
||||
import { Brainy } from '../../../brainy.js'
|
||||
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
|
||||
import { FindParams } from '../../../types/brainy.types.js'
|
||||
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
|
||||
import { VFSEntity } from '../../types.js'
|
||||
|
||||
/**
|
||||
* Temporal Projection: /as-of/<YYYY-MM-DD>/<subpath>
|
||||
*
|
||||
* Uses EXISTING infrastructure:
|
||||
* - Brainy.find() with range queries (REAL)
|
||||
* - MetadataIndexManager.$gte/$lte operators (REAL)
|
||||
* - VFSMetadata.modified field (REAL - types.ts line 49)
|
||||
*/
|
||||
export class TemporalProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'time'
|
||||
|
||||
/**
|
||||
* Convert date to Brainy FindParams with range query
|
||||
*/
|
||||
toQuery(date: Date, subpath?: string): FindParams {
|
||||
// Get start and end of day (24-hour window)
|
||||
const startOfDay = new Date(date)
|
||||
startOfDay.setHours(0, 0, 0, 0)
|
||||
|
||||
const endOfDay = new Date(date)
|
||||
endOfDay.setHours(23, 59, 59, 999)
|
||||
|
||||
const query: FindParams = {
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
modified: {
|
||||
greaterEqual: startOfDay.getTime(), // BFO operator
|
||||
lessEqual: endOfDay.getTime() // BFO operator
|
||||
}
|
||||
},
|
||||
limit: 1000
|
||||
}
|
||||
|
||||
// Filter by filename if subpath specified
|
||||
if (subpath) {
|
||||
query.where = {
|
||||
...query.where,
|
||||
anyOf: [ // BFO logical operator (not $or)
|
||||
{ name: subpath },
|
||||
{ path: { endsWith: subpath } } // BFO operator (not $regex)
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve date to entity IDs using REAL Brainy.find()
|
||||
* Uses MetadataIndexManager range queries for O(log n) performance
|
||||
*/
|
||||
async resolve(brain: Brainy, vfs: VirtualFileSystem, date: Date): Promise<string[]> {
|
||||
const startOfDay = new Date(date)
|
||||
startOfDay.setHours(0, 0, 0, 0)
|
||||
|
||||
const endOfDay = new Date(date)
|
||||
endOfDay.setHours(23, 59, 59, 999)
|
||||
|
||||
// Use REAL Brainy metadata filtering with range operators
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
modified: {
|
||||
greaterEqual: startOfDay.getTime(), // BFO operator
|
||||
lessEqual: endOfDay.getTime() // BFO operator
|
||||
}
|
||||
},
|
||||
limit: 1000
|
||||
})
|
||||
|
||||
return this.extractIds(results)
|
||||
}
|
||||
|
||||
/**
|
||||
* List recently modified files
|
||||
*/
|
||||
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
|
||||
const oneDayAgo = Date.now() - (24 * 60 * 60 * 1000)
|
||||
|
||||
const results = await brain.find({
|
||||
where: {
|
||||
vfsType: 'file',
|
||||
modified: { greaterEqual: oneDayAgo } // BFO operator
|
||||
},
|
||||
limit
|
||||
})
|
||||
|
||||
return results.map(r => r.entity as VFSEntity)
|
||||
}
|
||||
}
|
||||
Reference in a new issue