David Snelling
38b0041464
Every committed generation now also appends a FACT — an after-image commit record (what each touched entity/relationship became, or a body-less tombstone for a removal) — to an append-only, crc32c-framed segment log under _generations/facts/. The before-image history and the canonical tree remain authoritative; the fact log gives consumers ONE sequential, self-verifying stream (index heals, incremental replays) in place of a per-entity directory walk. - Wire format: positional msgpack facts [generation, timestamp, ops, meta, blobHashes]; op = [kind u8, id bin16, record | nil tombstone]; 32-byte segment header (magic, formatVersion, firstGeneration, zeroed+verified reserved); length+crc32c frame per fact; zero-padded segment names so lexicographic order == generation order; JSON manifest with an atomic rename flip, manifest-first rotation. - Commit protocol: facts append+fsync BEFORE the commit point inside the existing durability window, so a crash can only leave the log AHEAD of committed truth — open() truncates back (torn tails detected by CRC). Absent generation = never committed; a scan can never see an uncommitted fact. transact() facts are durable-on-return; single-op facts ride the group-commit flush exactly like buffered history. A fact-append failure fails the write, loudly — a silent gap would be a lie a later replay discovers. - New public surface: brain.scanFacts() (sequential batches with heal telemetry: head/segments/approx up front, per-batch generation range + bytes + segment id, loud abort on gaps, summary cross-check) and brain.factSegmentPaths() (immutable sealed segments for zero-copy consumers; the mutable tail excluded). Exported types CommitFact, FactOp, FactScanBatch, FactScanHandle. - Storage: optional binary raw-byte primitives (appendRawBytes, readRawBytes, writeRawBytes, rawByteSize) on StorageAdapter — feature-detected; filesystem + memory adapters implement them; an adapter without them hosts no fact log. Fact segments are byte-copied (never hard-linked) into snapshots. The _generations/facts/ namespace is registered as a protected family (rebuildable: false): no sweeper or GC may delete under it. - New crc32c (Castagnoli) utility with RFC known-answer tests.
569 lines
13 KiB
TypeScript
569 lines
13 KiB
TypeScript
/**
|
|
* Brainy 8.0 - Your AI-Powered Second Brain
|
|
* 🧠⚛️ A multi-dimensional database with vector, graph, and relational storage
|
|
*
|
|
* Core Components:
|
|
* - Brainy: The unified database with Triple Intelligence
|
|
* - Triple Intelligence: Seamless fusion of vector + graph + field search
|
|
* - Db: Immutable, generation-pinned database values (now/transact/asOf/with)
|
|
* - Plugins: Extensible plugin system (cor, storage adapters)
|
|
* - Neural Import: AI-powered entity extraction & smart data import
|
|
*/
|
|
|
|
// Export main Brainy class
|
|
import { Brainy } from './brainy.js'
|
|
|
|
export { Brainy }
|
|
|
|
// The in-process change feed (brain.onChange) — event + listener types.
|
|
export type {
|
|
BrainyChangeEvent,
|
|
ChangeEventEntity,
|
|
ChangeEventRelation,
|
|
ChangeListener
|
|
} from './events/changeFeed.js'
|
|
|
|
// Temporal VFS — a file version entry (vfs.history / readFile({ asOf })).
|
|
export type { FileVersion } from './vfs/types.js'
|
|
|
|
// Export diagnostics result type
|
|
export type { DiagnosticsResult } from './brainy.js'
|
|
|
|
// Export Brainy configuration and types
|
|
export type {
|
|
BrainyConfig,
|
|
Entity,
|
|
Relation,
|
|
Result,
|
|
AddParams,
|
|
UpdateParams,
|
|
RelateParams,
|
|
UpdateRelationParams,
|
|
FindParams,
|
|
SimilarParams,
|
|
GetOptions,
|
|
RelatedParams,
|
|
AddManyParams,
|
|
UpdateManyParams,
|
|
RemoveManyParams,
|
|
RelateManyParams,
|
|
BatchResult,
|
|
SubtypeRegistry,
|
|
FillSubtypeRule,
|
|
FillSubtypeRules,
|
|
FillSubtypesResult,
|
|
AggregateDefinition,
|
|
AggregateMetricDef,
|
|
AggregateSource,
|
|
AggregateQueryParams,
|
|
AggregateResult,
|
|
AggregateGroupState,
|
|
MetricState,
|
|
AggregationOp,
|
|
TimeWindowGranularity,
|
|
GroupByDimension,
|
|
AggregationProvider
|
|
} from './types/brainy.types.js'
|
|
|
|
// Reserved-field contract — the canonical list of Brainy-owned field names
|
|
// that may never appear inside a `metadata` bag (see docs/concepts/consistency-model.md)
|
|
export {
|
|
RESERVED_ENTITY_FIELDS,
|
|
RESERVED_RELATION_FIELDS,
|
|
splitNounMetadataRecord,
|
|
splitVerbMetadataRecord
|
|
} from './types/reservedFields.js'
|
|
export type {
|
|
ReservedEntityField,
|
|
ReservedRelationField,
|
|
EntityMetadataInput,
|
|
EntityMetadataPatch,
|
|
RelationMetadataInput,
|
|
RelationMetadataPatch,
|
|
NoReservedEntityKeys,
|
|
NoReservedRelationKeys,
|
|
SplitMetadataRecord
|
|
} from './types/reservedFields.js'
|
|
|
|
// Export Aggregation Engine
|
|
export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js'
|
|
|
|
// Export Neural Import (AI data understanding)
|
|
export { NeuralImport } from './neural/neuralImport.js'
|
|
export type {
|
|
NeuralAnalysisResult,
|
|
DetectedEntity,
|
|
DetectedRelationship,
|
|
NeuralInsight,
|
|
NeuralImportOptions
|
|
} from './neural/neuralImport.js'
|
|
|
|
// Export Neural Entity Extraction
|
|
export { NeuralEntityExtractor } from './neural/entityExtractor.js'
|
|
export { SmartExtractor } from './neural/SmartExtractor.js'
|
|
export { SmartRelationshipExtractor } from './neural/SmartRelationshipExtractor.js'
|
|
export type {
|
|
ExtractedEntity
|
|
} from './neural/entityExtractor.js'
|
|
export type {
|
|
ExtractionResult,
|
|
SmartExtractorOptions,
|
|
FormatContext
|
|
} from './neural/SmartExtractor.js'
|
|
export type {
|
|
RelationshipExtractionResult,
|
|
SmartRelationshipExtractorOptions
|
|
} from './neural/SmartRelationshipExtractor.js'
|
|
|
|
// Export distance functions for convenience
|
|
import {
|
|
euclideanDistance,
|
|
cosineDistance,
|
|
manhattanDistance,
|
|
dotProductDistance
|
|
} from './utils/index.js'
|
|
|
|
export {
|
|
euclideanDistance,
|
|
cosineDistance,
|
|
manhattanDistance,
|
|
dotProductDistance
|
|
}
|
|
|
|
// Export version utilities
|
|
export { getBrainyVersion } from './utils/version.js'
|
|
|
|
// Export plugin system
|
|
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
|
|
export { PluginRegistry } from './plugin.js'
|
|
|
|
// Export migration system
|
|
export { MigrationRunner, MIGRATIONS } from './migration/index.js'
|
|
export type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './migration/index.js'
|
|
|
|
// Export optimistic-concurrency types (7.31.0)
|
|
export { RevisionConflictError } from './transaction/RevisionConflictError.js'
|
|
|
|
// Named not-found errors — thrown by update/relate/updateRelation/similar/
|
|
// transact/with() when a referenced entity or relation does not exist.
|
|
export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js'
|
|
|
|
// Base error + typed migration-lock error — thrown by any data-plane call while a
|
|
// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
|
|
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
|
|
export type { BrainyErrorType } from './errors/brainyError.js'
|
|
|
|
// ============= 8.0 Db API — generational MVCC =============
|
|
// Immutable database values: brain.now() / brain.transact() / brain.asOf() /
|
|
// db.with() / db.persist() / Brainy.load(). See src/db/ for the record layer.
|
|
export { Db } from './db/db.js'
|
|
|
|
// Portable graph export/import — db.export() / brain.export() / brain.import()
|
|
// (PortableGraph v1; identical wire format to the 7.x line).
|
|
export { isPortableGraph, validatePortableGraph } from './db/portableGraph.js'
|
|
export type {
|
|
PortableGraph,
|
|
PortableGraphEntity,
|
|
PortableGraphRelation,
|
|
ExportSelector,
|
|
ExportOptions,
|
|
ImportOptions,
|
|
ImportResult,
|
|
PortableGraphValidation
|
|
} from './db/portableGraph.js'
|
|
export {
|
|
GenerationConflictError,
|
|
SpeculativeOverlayError,
|
|
GenerationCompactedError,
|
|
StoreInconsistentError,
|
|
PendingFlushDurabilityError
|
|
} from './db/errors.js'
|
|
export type { UnreconciledRecord } from './db/errors.js'
|
|
export type {
|
|
TxOperation,
|
|
TxAddOperation,
|
|
TxUpdateOperation,
|
|
TxRemoveOperation,
|
|
TxRelateOperation,
|
|
TxUnrelateOperation,
|
|
TransactOptions,
|
|
TransactReceipt,
|
|
TxLogEntry,
|
|
CompactHistoryOptions,
|
|
CompactHistoryResult,
|
|
ChangedIds,
|
|
DiffResult,
|
|
HistoryVersion,
|
|
EntityHistory
|
|
} from './db/types.js'
|
|
// The generation fact log — sequential after-image scan surface
|
|
// (brain.scanFacts / brain.factSegmentPaths) for index heals and replays.
|
|
export type {
|
|
CommitFact,
|
|
FactOp,
|
|
FactScanBatch,
|
|
FactScanHandle
|
|
} from './db/factLog.js'
|
|
// Optional provider capability for generation-aware native indexes
|
|
export { isVersionedIndexProvider } from './plugin.js'
|
|
export type { VersionedIndexProvider } from './plugin.js'
|
|
export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js'
|
|
// Optional native graph-acceleration engine (cor 3.0) — the published provider
|
|
// contract + its columnar wire types. Brainy feature-detects an implementation
|
|
// and falls back to its pure-TS adjacency when absent.
|
|
export type {
|
|
GraphAccelerationProvider,
|
|
Subgraph,
|
|
OpaqueIdSet,
|
|
GraphTraversalDirection,
|
|
TraverseOptions,
|
|
EdgesForNodeOptions,
|
|
GraphCursorHandle,
|
|
GraphCursorOptions,
|
|
GraphCursorChunk,
|
|
GraphScores,
|
|
GraphCommunities,
|
|
GraphPath,
|
|
RankOptions,
|
|
CommunitiesOptions,
|
|
PathOptions,
|
|
SampleOptions,
|
|
MostConnectedOptions
|
|
} from './plugin.js'
|
|
|
|
// Export embedding functionality
|
|
import {
|
|
UniversalSentenceEncoder,
|
|
TransformerEmbedding,
|
|
createEmbeddingFunction,
|
|
defaultEmbeddingFunction,
|
|
batchEmbed,
|
|
embeddingFunctions
|
|
} from './utils/embedding.js'
|
|
|
|
// Export logging utilities
|
|
import {
|
|
logger,
|
|
LogLevel,
|
|
configureLogger,
|
|
createModuleLogger
|
|
} from './utils/logger.js'
|
|
|
|
// Chat system removed - was returning fake responses
|
|
|
|
|
|
// Export performance and optimization utilities
|
|
import {
|
|
getGlobalBackpressure,
|
|
AdaptiveBackpressure
|
|
} from './utils/adaptiveBackpressure.js'
|
|
|
|
import {
|
|
getGlobalPerformanceMonitor,
|
|
PerformanceMonitor
|
|
} from './utils/performanceMonitor.js'
|
|
|
|
// Export environment utilities
|
|
import { isNode } from './utils/environment.js'
|
|
|
|
export {
|
|
UniversalSentenceEncoder,
|
|
TransformerEmbedding,
|
|
createEmbeddingFunction,
|
|
defaultEmbeddingFunction,
|
|
batchEmbed,
|
|
embeddingFunctions,
|
|
|
|
// Environment utilities
|
|
isNode,
|
|
|
|
// Logging utilities
|
|
logger,
|
|
LogLevel,
|
|
configureLogger,
|
|
createModuleLogger,
|
|
|
|
// Performance and optimization utilities
|
|
getGlobalBackpressure,
|
|
AdaptiveBackpressure,
|
|
getGlobalPerformanceMonitor,
|
|
PerformanceMonitor
|
|
}
|
|
|
|
// Export storage adapters (Brainy 8.0 — filesystem + memory only).
|
|
import { MemoryStorage, createStorage } from './storage/storageFactory.js'
|
|
|
|
export { MemoryStorage, createStorage }
|
|
|
|
// FileSystemStorage is exported separately to avoid browser build issues.
|
|
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
|
|
|
|
// Export types
|
|
import type {
|
|
Vector,
|
|
VectorDocument,
|
|
SearchResult,
|
|
DistanceFunction,
|
|
EmbeddingFunction,
|
|
EmbeddingModel,
|
|
HNSWNoun,
|
|
HNSWVerb,
|
|
HNSWConfig,
|
|
StorageAdapter,
|
|
DerivedFamilyDeclaration
|
|
} from './coreTypes.js'
|
|
|
|
// Export vector index implementation (the JS HNSW path)
|
|
import { JsHnswVectorIndex } from './hnsw/hnswIndex.js'
|
|
|
|
export { JsHnswVectorIndex }
|
|
|
|
export type {
|
|
Vector,
|
|
VectorDocument,
|
|
SearchResult,
|
|
DistanceFunction,
|
|
EmbeddingFunction,
|
|
EmbeddingModel,
|
|
HNSWNoun,
|
|
HNSWVerb,
|
|
HNSWConfig,
|
|
StorageAdapter,
|
|
DerivedFamilyDeclaration
|
|
}
|
|
|
|
// Export graph types
|
|
import type {
|
|
GraphNoun,
|
|
GraphVerb,
|
|
EmbeddedGraphVerb,
|
|
Person,
|
|
Organization,
|
|
Location,
|
|
Thing,
|
|
Concept,
|
|
Event,
|
|
Agent,
|
|
Organism,
|
|
Substance,
|
|
Quality,
|
|
TimeInterval,
|
|
Function,
|
|
Proposition,
|
|
Document,
|
|
Media,
|
|
File,
|
|
Message,
|
|
Collection,
|
|
Dataset,
|
|
Product,
|
|
Service,
|
|
Task,
|
|
Project,
|
|
Process,
|
|
State,
|
|
Role,
|
|
Language,
|
|
Currency,
|
|
Measurement,
|
|
Hypothesis,
|
|
Experiment,
|
|
Contract,
|
|
Regulation,
|
|
Interface,
|
|
Resource,
|
|
Custom,
|
|
SocialGroup,
|
|
Institution,
|
|
Norm,
|
|
InformationContent,
|
|
InformationBearer,
|
|
Relationship
|
|
} from './types/graphTypes.js'
|
|
import { NounType, VerbType } from './types/graphTypes.js'
|
|
|
|
export type {
|
|
GraphNoun,
|
|
GraphVerb,
|
|
EmbeddedGraphVerb,
|
|
Person,
|
|
Organization,
|
|
Location,
|
|
Thing,
|
|
Concept,
|
|
Event,
|
|
Agent,
|
|
Organism,
|
|
Substance,
|
|
Quality,
|
|
TimeInterval,
|
|
Function,
|
|
Proposition,
|
|
Document,
|
|
Media,
|
|
File,
|
|
Message,
|
|
Collection,
|
|
Dataset,
|
|
Product,
|
|
Service,
|
|
Task,
|
|
Project,
|
|
Process,
|
|
State,
|
|
Role,
|
|
Language,
|
|
Currency,
|
|
Measurement,
|
|
Hypothesis,
|
|
Experiment,
|
|
Contract,
|
|
Regulation,
|
|
Interface,
|
|
Resource,
|
|
Custom,
|
|
SocialGroup,
|
|
Institution,
|
|
Norm,
|
|
InformationContent,
|
|
InformationBearer,
|
|
Relationship
|
|
}
|
|
// Export type utility functions
|
|
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
|
|
|
|
// Export BrainyTypes for type validation and lookup
|
|
import { BrainyTypes } from './utils/brainyTypes.js'
|
|
|
|
export {
|
|
NounType,
|
|
VerbType,
|
|
getNounTypes,
|
|
getVerbTypes,
|
|
getNounTypeMap,
|
|
getVerbTypeMap,
|
|
// BrainyTypes - type validation and lookup
|
|
BrainyTypes
|
|
}
|
|
|
|
// Export MCP (Model Control Protocol) components
|
|
import {
|
|
BrainyMCPAdapter,
|
|
BrainyMCPService
|
|
} from './mcp/index.js' // Import from mcp/index.js
|
|
import {
|
|
MCPRequest,
|
|
MCPResponse,
|
|
MCPDataAccessRequest,
|
|
MCPToolExecutionRequest,
|
|
MCPSystemInfoRequest,
|
|
MCPAuthenticationRequest,
|
|
MCPRequestType,
|
|
MCPServiceOptions,
|
|
MCPTool,
|
|
MCP_VERSION
|
|
} from './types/mcpTypes.js'
|
|
|
|
export {
|
|
// MCP classes
|
|
BrainyMCPAdapter,
|
|
BrainyMCPService,
|
|
|
|
// MCP types
|
|
MCPRequestType,
|
|
MCP_VERSION
|
|
}
|
|
|
|
export type {
|
|
MCPRequest,
|
|
MCPResponse,
|
|
MCPDataAccessRequest,
|
|
MCPToolExecutionRequest,
|
|
MCPSystemInfoRequest,
|
|
MCPAuthenticationRequest,
|
|
MCPServiceOptions,
|
|
MCPTool
|
|
}
|
|
|
|
// ============= Integration Hub =============
|
|
// Connect Brainy to Excel, Power BI, Google Sheets, and more
|
|
// Enable with: new Brainy({ integrations: true })
|
|
|
|
// Hub class (used internally by brain.hub, also available for advanced use)
|
|
export {
|
|
IntegrationHub,
|
|
createIntegrationHub
|
|
} from './integrations/index.js'
|
|
|
|
export type {
|
|
IntegrationHubConfig,
|
|
IntegrationRequest,
|
|
IntegrationResponse
|
|
} from './integrations/index.js'
|
|
|
|
// Re-export IntegrationsConfig from types (for TypeScript users)
|
|
export type { IntegrationsConfig } from './types/brainy.types.js'
|
|
|
|
// Core infrastructure
|
|
export {
|
|
EventBus,
|
|
TabularExporter,
|
|
IntegrationBase,
|
|
IntegrationLoader,
|
|
createIntegrationLoader,
|
|
detectEnvironment,
|
|
INTEGRATION_CATALOG
|
|
} from './integrations/index.js'
|
|
|
|
// Integration types
|
|
export type {
|
|
BrainyEvent,
|
|
EventFilter,
|
|
EventHandler,
|
|
EventSubscription,
|
|
TabularRow,
|
|
RelationTabularRow,
|
|
TabularExporterConfig,
|
|
IntegrationConfig,
|
|
IntegrationHealthStatus,
|
|
HTTPIntegration,
|
|
StreamingIntegration,
|
|
IntegrationType,
|
|
RuntimeEnvironment,
|
|
IntegrationInfo,
|
|
IntegrationLoaderConfig,
|
|
ODataQueryOptions,
|
|
WebhookRegistration,
|
|
WebhookDeliveryResult
|
|
} from './integrations/index.js'
|
|
|
|
// Concrete integrations
|
|
export {
|
|
GoogleSheetsIntegration,
|
|
ODataIntegration,
|
|
SSEIntegration,
|
|
WebhookIntegration
|
|
} from './integrations/index.js'
|
|
|
|
export type {
|
|
GoogleSheetsConfig,
|
|
ODataConfig,
|
|
SSEConfig,
|
|
WebhookConfig
|
|
} from './integrations/index.js'
|
|
|
|
// OData utilities (advanced)
|
|
export {
|
|
parseODataQuery,
|
|
parseFilter,
|
|
parseOrderBy,
|
|
parseSelect,
|
|
odataToFindParams,
|
|
applyFilter,
|
|
applySelect,
|
|
applyOrderBy,
|
|
applyPagination,
|
|
generateEdmx,
|
|
generateMetadataJson,
|
|
generateServiceDocument
|
|
} from './integrations/index.js'
|