feat: add neural extraction APIs with NounType taxonomy

Add brain.extract() and brain.extractConcepts() methods that use
NeuralEntityExtractor with embeddings and sophisticated NounType
taxonomy (30+ entity types) for semantic entity and concept extraction.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-09-29 13:51:47 -07:00
parent 27cc699555
commit dd50d89ad6
41 changed files with 3807 additions and 7391 deletions

View file

@ -0,0 +1,83 @@
/**
* Author Projection Strategy
*
* Maps author-based paths to files owned by that author
* Uses EXISTING MetadataIndexManager for O(log n) queries
*/
import { Brainy } from '../../../brainy.js'
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
import { FindParams } from '../../../types/brainy.types.js'
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
import { VFSEntity } from '../../types.js'
/**
* Author Projection: /by-author/<authorName>/<subpath>
*
* Uses EXISTING infrastructure:
* - Brainy.find() with metadata filters (REAL)
* - MetadataIndexManager for O(log n) owner queries (REAL)
* - VFSMetadata.owner field (REAL - types.ts line 44)
*/
export class AuthorProjection extends BaseProjectionStrategy {
readonly name = 'author'
/**
* Convert author name to Brainy FindParams
*/
toQuery(authorName: string, subpath?: string): FindParams {
const query: FindParams = {
where: {
vfsType: 'file',
owner: authorName
},
limit: 1000
}
// Filter by filename if subpath specified
if (subpath) {
query.where = {
...query.where,
anyOf: [ // BFO logical operator (not $or)
{ name: subpath },
{ path: { endsWith: subpath } } // BFO operator (not $regex)
]
}
}
return query
}
/**
* Resolve author to entity IDs using REAL Brainy.find()
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, authorName: string): Promise<string[]> {
// Use REAL Brainy metadata filtering
const results = await brain.find({
where: {
vfsType: 'file',
owner: authorName
},
limit: 1000
})
return this.extractIds(results)
}
/**
* List all unique authors
* Uses aggregation over metadata
*/
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
// Get all files with owner metadata
const results = await brain.find({
where: {
vfsType: 'file',
owner: { $exists: true }
},
limit
})
return results.map(r => r.entity as VFSEntity)
}
}

View file

@ -0,0 +1,97 @@
/**
* Concept Projection Strategy
*
* Maps concept-based paths to files containing those concepts
* Uses EXISTING ConceptSystem and MetadataIndexManager
*/
import { Brainy } from '../../../brainy.js'
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
import { FindParams } from '../../../types/brainy.types.js'
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
import { VFSEntity } from '../../types.js'
/**
* Concept Projection: /by-concept/<conceptName>/<subpath>
*
* Uses EXISTING infrastructure:
* - Brainy.find() with metadata filters (REAL - line 580 in brainy.ts)
* - MetadataIndexManager for O(log n) concept queries (REAL)
* - ConceptSystem for concept extraction (REAL - ConceptSystem.ts)
*/
export class ConceptProjection extends BaseProjectionStrategy {
readonly name = 'concept'
/**
* Convert concept name to Brainy FindParams
* Uses EXISTING FindParams.where for metadata filtering
*
* Now uses flattened conceptNames array for O(log n) performance!
*/
toQuery(conceptName: string, subpath?: string): FindParams {
const query: FindParams = {
where: {
vfsType: 'file',
conceptNames: { contains: conceptName } // O(log n) indexed query
},
limit: 1000
}
// If subpath specified, also filter by filename
if (subpath) {
query.where = {
...query.where,
anyOf: [ // BFO logical operator
{ name: subpath },
{ path: { endsWith: subpath } } // BFO operator
]
}
}
return query
}
/**
* Resolve concept to entity IDs using REAL Brainy.find()
* VERIFIED: brain.find() exists at line 580 in brainy.ts
*
* NOW OPTIMIZED: Uses flattened conceptNames for O(log n) indexed queries!
* No more post-filtering - direct index lookup
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, conceptName: string): Promise<string[]> {
// Verify brain.find is a function (safety check)
if (typeof brain.find !== 'function') {
throw new Error('VERIFICATION FAILED: brain.find is not a function')
}
// Direct O(log n) query using flattened conceptNames array
// VFS automatically flattens concepts to conceptNames on write
const results = await brain.find({
where: {
vfsType: 'file',
conceptNames: { contains: conceptName } // Indexed array query
},
limit: 1000
})
return this.extractIds(results)
}
/**
* List all files with concept metadata
* Uses REAL Brainy.find() with metadata filter
*/
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
const results = await brain.find({
where: {
vfsType: 'file',
conceptNames: { exists: true } // Use flattened field
},
limit
})
// Convert to VFSEntity array
// VERIFIED: Result.entity exists in brainy.types.ts
return results.map(r => r.entity as VFSEntity)
}
}

View file

@ -0,0 +1,136 @@
/**
* Relationship Projection Strategy
*
* Maps relationship-based paths to files connected in the knowledge graph
* Uses EXISTING GraphAdjacencyIndex for O(1) traversal
*/
import { Brainy } from '../../../brainy.js'
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
import { FindParams } from '../../../types/brainy.types.js'
import { VerbType } from '../../../types/graphTypes.js'
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
import { RelationshipValue } from '../SemanticPathParser.js'
/**
* Relationship Projection: /related-to/<path>/depth-N/types-X,Y
*
* Uses EXISTING infrastructure:
* - Brainy.getRelations() for graph traversal (REAL - line 803 in brainy.ts)
* - GraphAdjacencyIndex for O(1) neighbor lookups (REAL)
* - VerbType enum for relationship types (REAL - graphTypes.ts)
*/
export class RelationshipProjection extends BaseProjectionStrategy {
readonly name = 'relationship'
/**
* Convert relationship value to Brainy FindParams
* Note: Graph queries don't use FindParams, but we provide this for consistency
*/
toQuery(value: RelationshipValue, subpath?: string): FindParams {
// This is informational - actual resolution uses getRelations()
return {
where: {
vfsType: 'file'
},
connected: {
to: value.targetPath,
depth: value.depth || 1
},
limit: 1000
}
}
/**
* Resolve relationships using REAL Brainy.getRelations()
* Uses GraphAdjacencyIndex for O(1) graph traversal
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, value: RelationshipValue): Promise<string[]> {
// Step 1: Resolve target path to entity ID
const targetId = await this.resolvePathToId(vfs, value.targetPath)
if (!targetId) {
return []
}
// Step 2: Get relationships using REAL Brainy graph
const depth = value.depth || 1
const visited = new Set<string>()
const results: string[] = []
await this.traverseRelationships(
brain,
targetId,
depth,
visited,
results,
value.relationshipTypes
)
// Filter to only files
return await this.filterFiles(brain, results)
}
/**
* Recursive graph traversal using REAL Brainy.getRelations()
*/
private async traverseRelationships(
brain: Brainy,
entityId: string,
remainingDepth: number,
visited: Set<string>,
results: string[],
types?: string[]
): Promise<void> {
if (remainingDepth <= 0 || visited.has(entityId)) {
return
}
visited.add(entityId)
// Get outgoing relationships (REAL method - line 803 in brainy.ts)
const relations = await brain.getRelations({
from: entityId,
limit: 100
})
for (const relation of relations) {
// Filter by relationship type if specified
if (types && types.length > 0) {
const relationshipName = relation.type?.toLowerCase()
if (!types.some(t => t.toLowerCase() === relationshipName)) {
continue
}
}
// Add to results
if (!results.includes(relation.to)) {
results.push(relation.to)
}
// Recurse if depth remaining
if (remainingDepth > 1) {
await this.traverseRelationships(
brain,
relation.to,
remainingDepth - 1,
visited,
results,
types
)
}
}
}
/**
* Resolve path to entity ID
* Helper to convert traditional path to entity ID
*/
private async resolvePathToId(vfs: VirtualFileSystem, path: string): Promise<string | null> {
try {
// Use REAL VFS public method
return await vfs.resolvePath(path)
} catch {
return null
}
}
}

View file

@ -0,0 +1,84 @@
/**
* Similarity Projection Strategy
*
* Maps similarity-based paths to files with similar content
* Uses EXISTING HNSW Index for O(log n) vector similarity
*/
import { Brainy } from '../../../brainy.js'
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
import { FindParams } from '../../../types/brainy.types.js'
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
import { SimilarityValue } from '../SemanticPathParser.js'
/**
* Similarity Projection: /similar-to/<path>/threshold-N
*
* Uses EXISTING infrastructure:
* - Brainy.similar() for vector similarity (REAL - line 680 in brainy.ts)
* - HNSW Index for O(log n) nearest neighbor search (REAL)
* - Cosine similarity for scoring (REAL)
*/
export class SimilarityProjection extends BaseProjectionStrategy {
readonly name = 'similar'
/**
* Convert similarity value to Brainy FindParams
* Note: Similarity uses brain.similar(), not find(), but we provide this for consistency
*/
toQuery(value: SimilarityValue, subpath?: string): FindParams {
// This is informational - actual resolution uses brain.similar()
return {
where: {
vfsType: 'file'
},
near: {
id: value.targetPath,
threshold: value.threshold || 0.7
},
limit: 50
}
}
/**
* Resolve similarity using REAL Brainy.similar()
* Uses HNSW Index for O(log n) vector search
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, value: SimilarityValue): Promise<string[]> {
// Step 1: Resolve target path to entity ID
const targetId = await this.resolvePathToId(vfs, value.targetPath)
if (!targetId) {
return []
}
// Step 2: Get target entity to use its vector
const targetEntity = await brain.get(targetId)
if (!targetEntity) {
return []
}
// Step 3: Find similar entities using REAL HNSW search
// VERIFIED: brain.similar() exists at line 680 in brainy.ts
const results = await brain.similar({
to: targetEntity,
threshold: value.threshold || 0.7,
limit: 50,
where: { vfsType: 'file' } // Only files
})
// Extract IDs
return this.extractIds(results)
}
/**
* Resolve path to entity ID
*/
private async resolvePathToId(vfs: VirtualFileSystem, path: string): Promise<string | null> {
try {
// Use REAL VFS public method
return await vfs.resolvePath(path)
} catch {
return null
}
}
}

View file

@ -0,0 +1,82 @@
/**
* Tag Projection Strategy
*
* Maps tag-based paths to files with those tags
* Uses EXISTING MetadataIndexManager for O(log n) queries
*/
import { Brainy } from '../../../brainy.js'
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
import { FindParams } from '../../../types/brainy.types.js'
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
import { VFSEntity } from '../../types.js'
/**
* Tag Projection: /by-tag/<tagName>/<subpath>
*
* Uses EXISTING infrastructure:
* - Brainy.find() with metadata filters (REAL)
* - MetadataIndexManager for O(log n) tag queries (REAL)
* - VFSMetadata.tags field (REAL - types.ts line 66)
*/
export class TagProjection extends BaseProjectionStrategy {
readonly name = 'tag'
/**
* Convert tag name to Brainy FindParams
*/
toQuery(tagName: string, subpath?: string): FindParams {
const query: FindParams = {
where: {
vfsType: 'file',
tags: { contains: tagName } // BFO operator for array contains
},
limit: 1000
}
// Filter by filename if subpath specified
if (subpath) {
query.where = {
...query.where,
anyOf: [ // BFO logical operator
{ name: subpath },
{ path: { endsWith: subpath } } // BFO operator
]
}
}
return query
}
/**
* Resolve tag to entity IDs using REAL Brainy.find()
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, tagName: string): Promise<string[]> {
// Use REAL Brainy metadata filtering
const results = await brain.find({
where: {
vfsType: 'file',
tags: { contains: tagName } // BFO operator
},
limit: 1000
})
return this.extractIds(results)
}
/**
* List all files with tags
*/
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
// Get all files that have tags
const results = await brain.find({
where: {
vfsType: 'file',
tags: { exists: true } // BFO operator
},
limit
})
return results.map(r => r.entity as VFSEntity)
}
}

View file

@ -0,0 +1,103 @@
/**
* Temporal Projection Strategy
*
* Maps time-based paths to files modified at that time
* Uses EXISTING MetadataIndexManager with range queries
*/
import { Brainy } from '../../../brainy.js'
import { VirtualFileSystem } from '../../VirtualFileSystem.js'
import { FindParams } from '../../../types/brainy.types.js'
import { BaseProjectionStrategy } from '../ProjectionStrategy.js'
import { VFSEntity } from '../../types.js'
/**
* Temporal Projection: /as-of/<YYYY-MM-DD>/<subpath>
*
* Uses EXISTING infrastructure:
* - Brainy.find() with range queries (REAL)
* - MetadataIndexManager.$gte/$lte operators (REAL)
* - VFSMetadata.modified field (REAL - types.ts line 49)
*/
export class TemporalProjection extends BaseProjectionStrategy {
readonly name = 'time'
/**
* Convert date to Brainy FindParams with range query
*/
toQuery(date: Date, subpath?: string): FindParams {
// Get start and end of day (24-hour window)
const startOfDay = new Date(date)
startOfDay.setHours(0, 0, 0, 0)
const endOfDay = new Date(date)
endOfDay.setHours(23, 59, 59, 999)
const query: FindParams = {
where: {
vfsType: 'file',
modified: {
greaterEqual: startOfDay.getTime(), // BFO operator
lessEqual: endOfDay.getTime() // BFO operator
}
},
limit: 1000
}
// Filter by filename if subpath specified
if (subpath) {
query.where = {
...query.where,
anyOf: [ // BFO logical operator (not $or)
{ name: subpath },
{ path: { endsWith: subpath } } // BFO operator (not $regex)
]
}
}
return query
}
/**
* Resolve date to entity IDs using REAL Brainy.find()
* Uses MetadataIndexManager range queries for O(log n) performance
*/
async resolve(brain: Brainy, vfs: VirtualFileSystem, date: Date): Promise<string[]> {
const startOfDay = new Date(date)
startOfDay.setHours(0, 0, 0, 0)
const endOfDay = new Date(date)
endOfDay.setHours(23, 59, 59, 999)
// Use REAL Brainy metadata filtering with range operators
const results = await brain.find({
where: {
vfsType: 'file',
modified: {
greaterEqual: startOfDay.getTime(), // BFO operator
lessEqual: endOfDay.getTime() // BFO operator
}
},
limit: 1000
})
return this.extractIds(results)
}
/**
* List recently modified files
*/
async list(brain: Brainy, vfs: VirtualFileSystem, limit = 100): Promise<VFSEntity[]> {
const oneDayAgo = Date.now() - (24 * 60 * 60 * 1000)
const results = await brain.find({
where: {
vfsType: 'file',
modified: { greaterEqual: oneDayAgo } // BFO operator
},
limit
})
return results.map(r => r.entity as VFSEntity)
}
}