diff --git a/docs/api-contract.json b/docs/api-contract.json new file mode 100644 index 00000000..9dadcc0e --- /dev/null +++ b/docs/api-contract.json @@ -0,0 +1,1545 @@ +{ + "contractVersion": 1, + "engine": "@soulcraftlabs/brainy", + "prose": "docs/contract-1-ratification.md", + "compatibility": { + "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", + "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" + }, + "doors": [ + { + "name": "adaptiveHistoryBudgetBytes", + "kind": "method", + "arity": 1 + }, + { + "name": "add", + "kind": "method", + "arity": 1 + }, + { + "name": "addMany", + "kind": "method", + "arity": 1 + }, + { + "name": "adoptLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "adoptLogAuthorityInner", + "kind": "method", + "arity": 0 + }, + { + "name": "aggViewFromEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "anyProviderMigrating", + "kind": "method", + "arity": 0 + }, + { + "name": "applyFusionScoring", + "kind": "method", + "arity": 2 + }, + { + "name": "applyGraphConstraints", + "kind": "method", + "arity": 2 + }, + { + "name": "armIdleFlushTimer", + "kind": "method", + "arity": 2 + }, + { + "name": "asOf", + "kind": "method", + "arity": 2 + }, + { + "name": "assertGenerationStoreReady", + "kind": "method", + "arity": 1 + }, + { + "name": "assertWritable", + "kind": "method", + "arity": 1 + }, + { + "name": "audit", + "kind": "method", + "arity": 0 + }, + { + "name": "auditGraph", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAdoptLegacyVfsBlobsIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAlpha", + "kind": "method", + "arity": 1 + }, + { + "name": "autoCompactHistory", + "kind": "method", + "arity": 0 + }, + { + "name": "awaitMigrationLock", + "kind": "method", + "arity": 1 + }, + { + "name": "awaitPendingEmbeds", + "kind": "method", + "arity": 0 + }, + { + "name": "backfillAggregateIfNeeded", + "kind": "method", + "arity": 1 + }, + { + "name": "batchGet", + "kind": "method", + "arity": 2 + }, + { + "name": "brainWideStrictRequiresSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "bridgeLegacyPendingEmbedSidecars", + "kind": "method", + "arity": 0 + }, + { + "name": "buildAtGenerationVectors", + "kind": "method", + "arity": 2 + }, + { + "name": "buildGraphView", + "kind": "method", + "arity": 4 + }, + { + "name": "buildMetadataFilter", + "kind": "method", + "arity": 1 + }, + { + "name": "buildMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "buildRelationMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "cacheVerbInt", + "kind": "method", + "arity": 2 + }, + { + "name": "canServeVectorAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "checkHealth", + "kind": "method", + "arity": 0 + }, + { + "name": "checkMigrations", + "kind": "method", + "arity": 0 + }, + { + "name": "clear", + "kind": "method", + "arity": 0 + }, + { + "name": "clearPendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "close", + "kind": "method", + "arity": 0 + }, + { + "name": "closeDurableSteps", + "kind": "method", + "arity": 0 + }, + { + "name": "cluster", + "kind": "method", + "arity": 1 + }, + { + "name": "collectProviderInvariants", + "kind": "method", + "arity": 0 + }, + { + "name": "compactHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "consumeMetadataWatermarkVerdict", + "kind": "method", + "arity": 1 + }, + { + "name": "convertMetadataToEntity", + "kind": "method", + "arity": 2 + }, + { + "name": "convertNounToEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "counts", + "kind": "accessor" + }, + { + "name": "createIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "createMigrationBackupIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "createPinnedDb", + "kind": "method", + "arity": 1 + }, + { + "name": "createResult", + "kind": "method", + "arity": 4 + }, + { + "name": "dbFinalizationRegistry", + "kind": "accessor" + }, + { + "name": "dbHost", + "kind": "accessor" + }, + { + "name": "defineAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "detectIdKind", + "kind": "method", + "arity": 3 + }, + { + "name": "diagnostics", + "kind": "method", + "arity": 0 + }, + { + "name": "diff", + "kind": "method", + "arity": 2 + }, + { + "name": "embed", + "kind": "method", + "arity": 1 + }, + { + "name": "embedBatch", + "kind": "method", + "arity": 2 + }, + { + "name": "emitCommitted", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnAdd", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnRelate", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceTrackedFieldValues", + "kind": "method", + "arity": 2 + }, + { + "name": "enhanceNLPResult", + "kind": "method", + "arity": 2 + }, + { + "name": "enqueuePendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "ensureAggregationIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureIndexesLoaded", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureInitialized", + "kind": "method", + "arity": 1 + }, + { + "name": "entityForAggFromRawRecord", + "kind": "method", + "arity": 1 + }, + { + "name": "entityFromGenerationRecord", + "kind": "method", + "arity": 3 + }, + { + "name": "entityIntsToUuids", + "kind": "method", + "arity": 1 + }, + { + "name": "entityViewFromRawRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "excludedVisibilityTiers", + "kind": "method", + "arity": 1 + }, + { + "name": "executeGraphSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeProximitySearch", + "kind": "method", + "arity": 1 + }, + { + "name": "executeTextSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeVectorSearch", + "kind": "method", + "arity": 3 + }, + { + "name": "explain", + "kind": "method", + "arity": 1 + }, + { + "name": "export", + "kind": "method", + "arity": 0 + }, + { + "name": "extract", + "kind": "method", + "arity": 2 + }, + { + "name": "extractConcepts", + "kind": "method", + "arity": 2 + }, + { + "name": "extractEntities", + "kind": "method", + "arity": 2 + }, + { + "name": "factSegmentPaths", + "kind": "method", + "arity": 1 + }, + { + "name": "fieldCountsAggregateName", + "kind": "method", + "arity": 1 + }, + { + "name": "fillSubtypes", + "kind": "method", + "arity": 1 + }, + { + "name": "filterIdsBelted", + "kind": "method", + "arity": 2 + }, + { + "name": "find", + "kind": "method", + "arity": 1 + }, + { + "name": "findAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "findDuplicates", + "kind": "method", + "arity": 1 + }, + { + "name": "findMatchingWords", + "kind": "method", + "arity": 3 + }, + { + "name": "flush", + "kind": "method", + "arity": 0 + }, + { + "name": "formatInfo", + "kind": "method", + "arity": 0 + }, + { + "name": "formatSubtypeError", + "kind": "method", + "arity": 1 + }, + { + "name": "generation", + "kind": "method", + "arity": 0 + }, + { + "name": "generationDigest", + "kind": "method", + "arity": 1 + }, + { + "name": "get", + "kind": "method", + "arity": 2 + }, + { + "name": "getActivePlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "getAvailableFields", + "kind": "method", + "arity": 0 + }, + { + "name": "getBackgroundDeduplicator", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsForType", + "kind": "method", + "arity": 1 + }, + { + "name": "getFieldStatistics", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsWithCardinality", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldValues", + "kind": "method", + "arity": 1 + }, + { + "name": "getIndexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getIndexStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "getMemoryStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getNeighborUuids", + "kind": "method", + "arity": 2 + }, + { + "name": "getNounCount", + "kind": "method", + "arity": 0 + }, + { + "name": "getOptimalQueryPlan", + "kind": "method", + "arity": 1 + }, + { + "name": "getStats", + "kind": "method", + "arity": 1 + }, + { + "name": "getStorageType", + "kind": "method", + "arity": 0 + }, + { + "name": "getSubtypeRule", + "kind": "method", + "arity": 1 + }, + { + "name": "getTripleIntelligence", + "kind": "method", + "arity": 0 + }, + { + "name": "getTypedNeighbors", + "kind": "method", + "arity": 4 + }, + { + "name": "getVerbCount", + "kind": "method", + "arity": 0 + }, + { + "name": "graph", + "kind": "accessor" + }, + { + "name": "graphAccelerationProvider", + "kind": "method", + "arity": 0 + }, + { + "name": "graphCommunities", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphEntityInt", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExport", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphPath", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathFallback", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathNative", + "kind": "method", + "arity": 4 + }, + { + "name": "graphRank", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraphFallback", + "kind": "method", + "arity": 4 + }, + { + "name": "graphSubgraphFromQuery", + "kind": "method", + "arity": 5 + }, + { + "name": "graphSubgraphNative", + "kind": "method", + "arity": 5 + }, + { + "name": "groupByLabel", + "kind": "method", + "arity": 2 + }, + { + "name": "hasStorageMethod", + "kind": "method", + "arity": 1 + }, + { + "name": "hasVectorOrTextCriteria", + "kind": "method", + "arity": 1 + }, + { + "name": "health", + "kind": "method", + "arity": 0 + }, + { + "name": "highlight", + "kind": "method", + "arity": 1 + }, + { + "name": "highlightSemanticPhase", + "kind": "method", + "arity": 5 + }, + { + "name": "history", + "kind": "method", + "arity": 2 + }, + { + "name": "historyStats", + "kind": "method", + "arity": 0 + }, + { + "name": "hub", + "kind": "accessor" + }, + { + "name": "hydrateIdMapperForGraphRebuild", + "kind": "method", + "arity": 0 + }, + { + "name": "hydrateNativeSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "import", + "kind": "method", + "arity": 2 + }, + { + "name": "importPluginPackage", + "kind": "method", + "arity": 1 + }, + { + "name": "incidentEdges", + "kind": "method", + "arity": 3 + }, + { + "name": "indexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "init", + "kind": "method", + "arity": 1 + }, + { + "name": "insights", + "kind": "method", + "arity": 0 + }, + { + "name": "isEmbeddingReady", + "kind": "method", + "arity": 0 + }, + { + "name": "isInfrastructureWrite", + "kind": "method", + "arity": 1 + }, + { + "name": "isInitialized", + "kind": "accessor" + }, + { + "name": "isReadOnly", + "kind": "accessor" + }, + { + "name": "kickBackgroundFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "kickEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "legacyLayoutMigrationPhase", + "kind": "method", + "arity": 0 + }, + { + "name": "loadAnalyticsGraph", + "kind": "method", + "arity": 1 + }, + { + "name": "loadPlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "logAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "maintenanceDebt", + "kind": "method", + "arity": 0 + }, + { + "name": "materializeAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "metadataIndexRetractionOp", + "kind": "method", + "arity": 3 + }, + { + "name": "migrate", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateField", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateInternal", + "kind": "method", + "arity": 2 + }, + { + "name": "migrateLegacyZeroNormVfsRootIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "migrationSnapshot", + "kind": "method", + "arity": 0 + }, + { + "name": "neededFamiliesMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "neighbors", + "kind": "method", + "arity": 2 + }, + { + "name": "newId", + "kind": "method", + "arity": 0 + }, + { + "name": "nlp", + "kind": "method", + "arity": 0 + }, + { + "name": "normalizeConfig", + "kind": "method", + "arity": 1 + }, + { + "name": "noteWriteForPersistence", + "kind": "method", + "arity": 0 + }, + { + "name": "now", + "kind": "method", + "arity": 0 + }, + { + "name": "onChange", + "kind": "method", + "arity": 1 + }, + { + "name": "pagination", + "kind": "accessor" + }, + { + "name": "parseMigrationPath", + "kind": "method", + "arity": 1 + }, + { + "name": "parseNaturalQuery", + "kind": "method", + "arity": 1 + }, + { + "name": "pathExists", + "kind": "method", + "arity": 2 + }, + { + "name": "pendingEmbedCount", + "kind": "method", + "arity": 0 + }, + { + "name": "performInit", + "kind": "method", + "arity": 1 + }, + { + "name": "persistPinnedGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "persistSingleOp", + "kind": "method", + "arity": 6 + }, + { + "name": "pickMetadataProbe", + "kind": "method", + "arity": 1 + }, + { + "name": "pickVectorProbe", + "kind": "method", + "arity": 0 + }, + { + "name": "pinGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "planGetEntity", + "kind": "method", + "arity": 3 + }, + { + "name": "planTransact", + "kind": "method", + "arity": 1 + }, + { + "name": "planTxAdd", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRemove", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUnrelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUpdate", + "kind": "method", + "arity": 3 + }, + { + "name": "projectionGauges", + "kind": "method", + "arity": 0 + }, + { + "name": "providerForFamily", + "kind": "method", + "arity": 1 + }, + { + "name": "providerIsMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "providerMigrationStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "queryAggregate", + "kind": "method", + "arity": 2 + }, + { + "name": "queryIndexFamilies", + "kind": "method", + "arity": 1 + }, + { + "name": "readPath", + "kind": "method", + "arity": 2 + }, + { + "name": "ready", + "kind": "accessor" + }, + { + "name": "rebuildIndexesIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "rebuildMetadataIndexOnline", + "kind": "method", + "arity": 0 + }, + { + "name": "reconcileLogDivergence", + "kind": "method", + "arity": 2 + }, + { + "name": "reconstructPath", + "kind": "method", + "arity": 4 + }, + { + "name": "recordStateAt", + "kind": "method", + "arity": 3 + }, + { + "name": "recoverPendingEmbedsFromLog", + "kind": "method", + "arity": 0 + }, + { + "name": "registerShutdownHooks", + "kind": "method", + "arity": 0 + }, + { + "name": "relate", + "kind": "method", + "arity": 1 + }, + { + "name": "related", + "kind": "method", + "arity": 1 + }, + { + "name": "relateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "relationFromGenerationRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "relationshipSubtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "releaseGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "remove", + "kind": "method", + "arity": 1 + }, + { + "name": "removeAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMany", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMigrationBackupSafe", + "kind": "method", + "arity": 0 + }, + { + "name": "repackHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "repairIndex", + "kind": "method", + "arity": 1 + }, + { + "name": "requestFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "requireProviders", + "kind": "method", + "arity": 1 + }, + { + "name": "requireSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveAsOfGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "resolveDiffEndpoint", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHiddenIds", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHNSWPersistMode", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveRawGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveRetentionPolicy", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveVerbEndpointInts", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveVerbIntsToIds", + "kind": "method", + "arity": 1 + }, + { + "name": "restore", + "kind": "method", + "arity": 2 + }, + { + "name": "rrfFusion", + "kind": "method", + "arity": 4 + }, + { + "name": "runAggregationBackfillWalk", + "kind": "method", + "arity": 0 + }, + { + "name": "runAggregationCatchUp", + "kind": "method", + "arity": 0 + }, + { + "name": "runEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "runOracle", + "kind": "method", + "arity": 1 + }, + { + "name": "runRepairIndexPhases", + "kind": "method", + "arity": 5 + }, + { + "name": "scanFacts", + "kind": "method", + "arity": 1 + }, + { + "name": "seedIdsToInts", + "kind": "method", + "arity": 1 + }, + { + "name": "selectorToSeedIds", + "kind": "method", + "arity": 1 + }, + { + "name": "setRetentionBudget", + "kind": "method", + "arity": 1 + }, + { + "name": "setupEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "setupIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "setupStorage", + "kind": "method", + "arity": 0 + }, + { + "name": "similar", + "kind": "method", + "arity": 1 + }, + { + "name": "similarity", + "kind": "method", + "arity": 2 + }, + { + "name": "splitForHighlighting", + "kind": "method", + "arity": 2 + }, + { + "name": "stampBrainFormat", + "kind": "method", + "arity": 0 + }, + { + "name": "stampBrainFormatIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "stampEntityTree", + "kind": "method", + "arity": 0 + }, + { + "name": "stampProjectionWatermarks", + "kind": "method", + "arity": 0 + }, + { + "name": "stats", + "kind": "method", + "arity": 0 + }, + { + "name": "storageAdapter", + "kind": "accessor" + }, + { + "name": "stream", + "kind": "method", + "arity": 0 + }, + { + "name": "streaming", + "kind": "accessor" + }, + { + "name": "subtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "trackField", + "kind": "method", + "arity": 1 + }, + { + "name": "transact", + "kind": "method", + "arity": 2 + }, + { + "name": "transactionLog", + "kind": "method", + "arity": 1 + }, + { + "name": "unrelate", + "kind": "method", + "arity": 1 + }, + { + "name": "unvectorNounForRootMigration", + "kind": "method", + "arity": 1 + }, + { + "name": "update", + "kind": "method", + "arity": 1 + }, + { + "name": "updateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "updateRelation", + "kind": "method", + "arity": 1 + }, + { + "name": "upsertMergeParams", + "kind": "method", + "arity": 2 + }, + { + "name": "use", + "kind": "method", + "arity": 1 + }, + { + "name": "usesDefaultWasmEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "validateIndexConsistency", + "kind": "method", + "arity": 0 + }, + { + "name": "vectorSearchAtGeneration", + "kind": "method", + "arity": 4 + }, + { + "name": "verbsToRelations", + "kind": "method", + "arity": 1 + }, + { + "name": "verbToRelationLike", + "kind": "method", + "arity": 1 + }, + { + "name": "verifyEntityTreeStamp", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyGraphAdjacencyLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyMetadataLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyVectorLive", + "kind": "method", + "arity": 0 + }, + { + "name": "versionedIndexProviders", + "kind": "method", + "arity": 0 + }, + { + "name": "vfs", + "kind": "accessor" + }, + { + "name": "waitForIndexed", + "kind": "method", + "arity": 2 + }, + { + "name": "warm", + "kind": "method", + "arity": 0 + }, + { + "name": "warmupEmbeddings", + "kind": "method", + "arity": 0 + }, + { + "name": "warnIfReadsDegraded", + "kind": "method", + "arity": 1 + }, + { + "name": "wireConnectionsCodec", + "kind": "method", + "arity": 0 + }, + { + "name": "wireGraphIdResolver", + "kind": "method", + "arity": 0 + } + ], + "errors": [ + "BrainyError", + "DerivedArtifactMissingError", + "GraphIndexNotReadyError", + "MetadataIndexNotReadyError", + "MigrationInProgressError", + "ProtectedArtifactError", + "VectorIndexNotReadyError" + ], + "operators": { + "accepted": [ + "between", + "contains", + "endsWith", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "length", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "matches", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf", + "startsWith" + ], + "servedOnIndexPath": [ + "between", + "contains", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf" + ], + "refusedByIndexPath": [ + "endsWith", + "length", + "matches", + "startsWith" + ], + "combinators": [ + "allOf", + "anyOf", + "not" + ] + }, + "fieldAddressing": { + "systemKeyPrefix": "system.", + "systemEntityScalars": [ + "confidence", + "createdAt", + "createdBy", + "id", + "service", + "subtype", + "type", + "updatedAt", + "visibility", + "weight" + ], + "systemRelationScalars": [ + "confidence", + "createdAt", + "createdBy", + "service", + "sourceId", + "subtype", + "targetId", + "updatedAt", + "verb", + "visibility", + "weight" + ], + "plumbingFields": [ + "_rev", + "connections", + "data", + "level", + "vector" + ] + }, + "health": { + "verdicts": [ + "pass", + "warn", + "fail" + ], + "healKinds": [ + "none", + "repair", + "rebuild" + ], + "servingWithholdingInvariants": [ + "index-initialized", + "durable-state-present", + "manifest-residency", + "replay-clean", + "strand-latch" + ] + } +} diff --git a/docs/contract-1-ratification.md b/docs/contract-1-ratification.md new file mode 100644 index 00000000..87df7b1f --- /dev/null +++ b/docs/contract-1-ratification.md @@ -0,0 +1,244 @@ +# Contract 1 — ratification + +Open Brainy's answer to the API contract published by the accelerated engine +(`docs/api-contract.md` + `docs/api-contract.json`, contract version 1). Each +item is answered with the code line that proves it, and each promise is stated +as a promise rather than a description. + +*Internal engineering document — no frontmatter, not published.* + +--- + +## 1. Contract version — DECLARED + +`package.json` carries `"brainyContract": 1`, and the engine states its own: + +```ts +export const BRAINY_CONTRACT_VERSION = 1 as const +export function contractVersion(): number { return BRAINY_CONTRACT_VERSION } +``` + +`src/utils/version.ts`, re-exported from `src/index.ts`. Two engines can now +compare an integer instead of probing prototypes, and a tool can read the +package field without importing the engine. Pinned in +`tests/integration/filter-operator-conformance.test.ts` ("declares its contract +version in code and in package.json") — the code value and the package field +can never drift apart silently. + +--- + +## 2. The REQUIRED / OPTIONAL split — RATIFIED, WITH A COMMITMENT + +**Ratified: 41 required doors of 57.** The promise, stated plainly: + +> **A REQUIRED door is never removed, never narrowed, and never made optional +> without a MAJOR contract bump.** "Narrowed" includes: refusing an input it +> used to accept, returning less than it used to return, and changing an +> ordering, a cursor encoding, or a refusal's typed code. An OPTIONAL door may +> be added in a minor; an optional door **promoted to required** is a major, +> because a consumer that relied on feature-detecting it now has a hard +> dependency. + +Two clarifications this engine attaches, so the promise means the same thing +on both sides: + +1. **A refusal is part of the door.** Contract 1 includes not only that + `find({ where })` answers, but that it REFUSES by name for the operators + listed as refused. Turning a refusal into a silent empty answer is a + narrowing, not a relaxation — the same class of change as removing the door. +2. **Deprecation is not removal.** This engine may mark a required door + deprecated in a minor (documented, warned) as long as it keeps working. Only + its removal is a major. + +--- + +## 3. `is` / `isNot` / `greaterEqual` / `lessEqual` — A FINDING AGAINST THE SPEC + +**The specification is wrong about these four, and this engine has never +served them.** `docs/filter-operator-conformance.md` (in the accelerated +engine's repository — not editable from here) lists them as served aliases. +The accepted set is defined in one place: + +```ts +// src/utils/metadataFilter.ts +const VALUE_OPERATORS = new Set([ + 'equals', 'eq', 'notEquals', 'ne', + 'greaterThan', 'gt', 'greaterThanOrEqual', 'gte', + 'lessThan', 'lt', 'lessThanOrEqual', 'lte', + 'between', 'oneOf', 'in', 'noneOf', + 'contains', 'excludes', 'hasAll', 'length', + 'exists', 'missing', 'matches', 'startsWith', 'endsWith' +]) +``` + +25 tokens. None of the four appears; `validateWhereFilter()` raises +`BrainyError('INVALID_QUERY')` naming the bad operator and listing the valid +set, before any index read. A consumer following the spec would have written a +filter this engine rejects outright. + +**Action taken here, since the prose lives in the other repository:** the truth +is made machine-checkable rather than re-asserted in another document. The +accepted set is asserted token-for-token in +`tests/integration/filter-operator-conformance.test.ts`, read out of the +engine's own refusal message, and the same set is emitted into +`docs/api-contract.json` (item 8). Diff the manifests; the prose can then be +corrected from a fact. + +--- + +## 4. The serving-withholding invariant list — CONFIRMED IDENTICAL + +`index-initialized · durable-state-present · manifest-residency · +replay-clean · strand-latch`. Confirmed as this engine's list, and confirmed +EXHAUSTIVE for contract 1: these are the only invariants whose failure may +withhold serving. Everything else a health report can fail is a `warn` — it +names damage without closing a door. + +The mechanism on this side: `assessProviderHealth()` +(`src/utils/indexReadiness.ts`) treats the provider's own `serving` verdict as +authoritative and verbatim; an UNLEDGERED family never flips a serving provider +to not-ready and never flips a not-serving provider to ready. The read gate +refuses PER FAMILY — a metadata read is never refused by an unserving vector +leg (`src/brainy.ts`, `ensureFamiliesServing`). + +**One addition this engine is making, declared here because it changes what a +refusal MEANS:** a provider may now report `rebuildInProgress()` — it is +rebuilding ITSELF, online, and its doors refuse by name with progress until it +is whole. This does not add a withholding invariant (the provider's own +`serving: false` is still what withholds); it adds a REASON attached to that +withholding, so a caller can tell "temporarily closed, opens by itself" from +"broken, needs repairIndex()". Additive, hence a minor. + +--- + +## 5. The compatibility rule — ADOPTED + +**Minor = additive. Major = breaking.** Adopted verbatim, with the +announcement duty attached: + +> **Every public-surface addition is announced.** The accelerated engine's +> package re-exports this engine's surface by enumeration, so it goes red on +> any new export BY DESIGN — that redness is the announcement mechanism +> working, not a build break to route around. + +The mechanics that make this checkable rather than remembered: +`scripts/emit-contract-manifest.mjs --check` fails when the committed +`docs/api-contract.json` no longer matches the built surface. A new export is +therefore a red check with a message naming what to do: re-emit and announce. + +--- + +## 6. The 30 storage seam methods — SUPPORTED SURFACE, COMMITTED + +**Committed: every method in `docs/api-contract.md` §15 is supported surface +until Stage 2, and none is removed without a contract major.** They are the +seam the accelerated engine's storage adapter implements and the seam its +reader replaces piece by piece; removing one mid-programme would break a +working pair for no gain. + +Two qualifications, both stated so neither side is surprised: + +1. **Supported ≠ frozen in behaviour.** A seam method may become FASTER, may + narrate more, and may start refusing an input that was previously an + undefined-behaviour footgun — the last of those is announced as a divergence + here before it ships, not discovered by the other engine. +2. **`counts.json`'s ledger is the one seam value that is not an + enumeration.** See `docs/canonical-layout-ratification.md` §8: only the + all-tier pair carrying `allCountsDerivedBy: 'identity-record'` with + `allCountsSuspect: false` may be subtracted against. That rule is part of + this commitment. + +--- + +## 7. `hasAll` / `noneOf` / `excludes` — SERVED, NOT RATIFIED AS A DIVERGENCE + +The accelerated engine was right that it was the correct side, and the +divergence is now closed in the right direction: **this engine serves all three +on the index path.** + +The defect underneath was worse than a divergence. The metadata index's +operator switch (`src/utils/metadataIndex.ts`) had **no default case**, so any +operator without a `case` left the field's match set at its initial `[]` and +`find()` returned an empty page. `hasAll`, `noneOf` and `excludes` are +documented, accepted by the validator, and implemented in the in-memory +matcher — and they answered silently wrong through an index-backed find. + +- **`hasAll: [a, b]`** — the intersection of each element's posting set. An + empty operand array is vacuously true of every row that HAS the field. +- **`noneOf: [a, b]`** — the complement of the union of their posting sets. +- **`excludes: v`** — the complement of `contains`. + +**And the other four are now REFUSED BY NAME rather than answered empty.** +`startsWith`, `endsWith`, `matches` and `length` cannot be evaluated by an +equality/range posting index without reading every row, which is the cost this +path exists to avoid. They raise `BrainyError('INVALID_QUERY')` naming the +operator, the field, and the reason. This matches the accelerated engine's +behaviour for the same four tokens, so the two engines now AGREE on all 25: + +| class | tokens | +|---|---| +| served on the index path | between, contains, eq, equals, excludes, exists, greaterThan, greaterThanOrEqual, gt, gte, hasAll, in, lessThan, lessThanOrEqual, lt, lte, missing, ne, noneOf, notEquals, oneOf | +| refused by name | endsWith, length, matches, startsWith | + +Pinned in `tests/integration/filter-operator-conformance.test.ts`: the exact +25-token accepted set, the three now served with their real answers (including +an honest zero), and each of the four refusing by name. + +**This is a behaviour change for any consumer today calling the four refused +operators through `find({ where })`.** They received an empty page; they now +receive a typed refusal. Converting a wrong answer into a loud refusal is this +engine's own law, and the previous behaviour was not a contract anyone could +have relied on deliberately — but it is a change, and it is named here rather +than discovered. + +**`knownDivergences` after this change:** the entry +`served-beyond-baseline` is RESOLVED (both engines serve all three). The entry +`refused-operators-answer-differently` is RESOLVED (both engines refuse the +same four by name). Contract 1 has no remaining operator divergence. + +--- + +## 8. This engine's own manifest — EMITTED + +`docs/api-contract.json`, generated by `scripts/emit-contract-manifest.mjs` +from the BUILT surface: the prototype's own methods and accessors, the exported +error classes, the operator sets read out of their single definitions, the +field-addressing vocabulary read out of `src/db/fieldAddressing.ts`, and the +health verdicts. Nothing in it is hand-maintained, so a diff between the two +manifests is a diff between two engines rather than between two authors. + +`node scripts/emit-contract-manifest.mjs --check` fails when the committed +manifest is stale — the announcement duty of item 5, made mechanical. + +**What the manifest deliberately does NOT carry: requirement marking.** Whether +a door is required is a commitment, not a property of the surface; it is item 2 +of this document. The diff the two sides want — "does Open Brainy still expose +every door contract 1 requires?" — is a set-membership check between their +`doors[].name` where `requirement === 'required'` and this manifest's +`doors[].name`. + +--- + +## A defect this work surfaced but did not fix + +`src/vfs/VirtualFileSystem.ts` builds a path-prefix filter as +`path: { $startsWith: options.path }` — with a `$` prefix. No operator in this +engine carries a `$`, so `validateWhereFilter()` rejects it with +`INVALID_QUERY` before any index read: **`vfs.searchFiles({ path })` throws +today, on every call that passes a path.** It is pre-existing and unrelated to +the operator work above (the validator refuses it before the index path is +reached), and it is left as a filing rather than fixed here, because the right +answer is a design question — a path-prefix search cannot be served by an +equality/range index, so it needs either a path-segment index or an explicit +in-memory narrow, not a spelling correction. + +--- + +## Summary of what changed in code for this ratification + +| item | change | +|---|---| +| 1 | `"brainyContract": 1` in package.json; `contractVersion()` / `BRAINY_CONTRACT_VERSION` exported | +| 3 | the accepted 25-token set asserted from the engine's own refusal message, and emitted into the manifest | +| 7 | `hasAll` / `noneOf` / `excludes` served on the index path; `startsWith` / `endsWith` / `matches` / `length` refused by name instead of answered empty | +| 8 | `scripts/emit-contract-manifest.mjs` + the generated `docs/api-contract.json`, with a `--check` mode | diff --git a/package.json b/package.json index bb6b5a47..60e901de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "@soulcraftlabs/brainy", "version": "10.4.3", + "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs new file mode 100644 index 00000000..13a9bc6d --- /dev/null +++ b/scripts/emit-contract-manifest.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node +/** + * Emit this build's API-contract manifest to docs/api-contract.json. + * + * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the + * code the first time somebody adds one. This reads the surface the build + * actually exposes — the prototype's own methods and accessors, the exported + * error classes, the `where` operator sets, the field-addressing vocabulary, + * the health verdicts — so a diff between two engines' manifests is a diff + * between two engines, never between two authors. + * + * Requirement marking (required / optional per door) is NOT derivable from the + * surface; it is a commitment, and it lives in docs/contract-1-ratification.md. + * This manifest carries the surface; that document carries the promise. + * + * Usage: node scripts/emit-contract-manifest.mjs [--check] + * --check exits non-zero when the committed manifest is stale. + */ + +import { writeFileSync, readFileSync, existsSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') +const OUT = join(ROOT, 'docs', 'api-contract.json') + +const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) +const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) +const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) +const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) + +/** Every own method and accessor on the class's prototype, minus the private ones. */ +function surfaceOf(ctor) { + const doors = [] + for (const name of Object.getOwnPropertyNames(ctor.prototype)) { + if (name === 'constructor' || name.startsWith('_')) continue + const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) + if (!descriptor) continue + if (typeof descriptor.value === 'function') { + doors.push({ name, kind: 'method', arity: descriptor.value.length }) + } else if (descriptor.get) { + doors.push({ name, kind: 'accessor' }) + } + } + return doors.sort((a, b) => a.name.localeCompare(b.name)) +} + +const errors = Object.entries(errorsModule) + .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) + .map(([name]) => name) + .sort() + +// The operator sets, read from the engine's own refusal message so the +// manifest can never disagree with the validator. +const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') +const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) +if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') +const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() + +const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') +const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => + // Proven by the refusal path: these are the tokens with no case in the + // index's operator switch, so they fall to its default and are refused. + !new RegExp(`case '${op}':`).test(indexSource) +) +const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) + +const manifest = { + contractVersion: versionModule.contractVersion(), + engine: '@soulcraftlabs/brainy', + prose: 'docs/contract-1-ratification.md', + compatibility: { + minor: + 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', + major: + 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' + }, + doors: surfaceOf(Brainy), + errors, + operators: { + accepted, + servedOnIndexPath: servedOnIndex, + refusedByIndexPath: refusedByIndex, + combinators: ['allOf', 'anyOf', 'not'] + }, + fieldAddressing: { + systemKeyPrefix: 'system.', + systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), + systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), + plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() + }, + health: { + verdicts: ['pass', 'warn', 'fail'], + healKinds: ['none', 'repair', 'rebuild'], + servingWithholdingInvariants: [ + 'index-initialized', + 'durable-state-present', + 'manifest-residency', + 'replay-clean', + 'strand-latch' + ] + } +} + +const rendered = `${JSON.stringify(manifest, null, 2)}\n` + +if (process.argv.includes('--check')) { + if (!existsSync(OUT)) { + console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) + process.exit(1) + } + if (readFileSync(OUT, 'utf-8') !== rendered) { + console.error( + `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + + `the addition (minor = additive; a removal is a contract major).` + ) + process.exit(1) + } + console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) + process.exit(0) +} + +writeFileSync(OUT, rendered) +console.log( + `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + + `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + + `${manifest.operators.accepted.length} operators ` + + `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` +) diff --git a/src/brainy.ts b/src/brainy.ts index 24407018..70d46973 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -198,7 +198,12 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { assessIndexReadiness, assessProviderHealth } from './utils/indexReadiness.js' +import { + assessIndexReadiness, + assessProviderHealth, + assessProviderRebuild, + describeRebuildProgress +} from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -740,6 +745,24 @@ export class Brainy implements BrainyInterface { // Write acks NEVER await it; a failed background flush is LOUD and re-armed. private _persistDirtyWrites = 0 private _persistLastFlushAt = Date.now() + /** + * Whether a write has been committed since the last flush that ran. THE + * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written + * to has nothing to make durable, and a flush over it must cost nothing and + * say nothing. Before this, a flush called every provider, stamped the + * watermarks, persisted the generation counter and re-stamped the entity + * tree whether or not anything had changed — roughly 28 writes for a store + * that had not moved. + * + * WHAT THIS DOES NOT EXPLAIN, stated so nobody reads it as solved: a + * production process holding 21 brains printed "All indexes flushed to disk + * in 216-601ms" per brain every ~35s and idled at 1.26 cores with no writes + * for ten minutes. This engine's cadence is WRITE-DRIVEN — every trigger + * runs through noteWriteForPersistence, which only a committed write calls — + * so something was calling flush() on those brains, and this gate makes such + * a call free rather than accounting for it. The caller is still unidentified. + */ + private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null @@ -834,7 +857,18 @@ export class Brainy implements BrainyInterface { // Read-gate narration dedup: a degraded-but-serving or not-ready health // report narrates via prodLog.warn ONCE per (provider, report.generation) — // never once per read. Keyed on the provider instance itself. - private _lastNarratedHealthGeneration = new Map() + /** + * The last health narration emitted per provider, keyed by its CONTENT. + * + * This used to dedupe on the provider's `generation` counter, which bumps on + * every ledger mutation and every rebuild boundary — so a provider that + * bumps its generation on routine work re-emitted the same unchanged health + * line on every read that consulted it, and a provider that never bumped + * could suppress a line whose reasons had genuinely changed. The dedupe key + * is now what the line SAYS: an unchanged verdict is silent however the + * generation moves, and a changed verdict is always heard. + */ + private _lastNarratedHealth = new Map() constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -1146,6 +1180,23 @@ export class Brainy implements BrainyInterface { ) }, OPEN_HEARTBEAT_MS) if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() + /** + * Narrate one STEP inside a phase when it turns out to be expensive. + * A phase that costs a minute and names only itself tells an operator + * where to look but not what to look at; this names the step. Silent + * under OPEN_PHASE_NARRATE_MS, so a fast open says nothing extra. + */ + const step = async (name: string, cause: string, run: () => Promise): Promise => { + const startedAt = Date.now() + try { + return await run() + } finally { + const elapsed = Date.now() - startedAt + if (elapsed >= OPEN_PHASE_NARRATE_MS) { + prodLog.narrate(`[Brainy] open: step "${name}" took ${elapsed}ms — ${cause}`) + } + } + } const markPhase = (name: string): void => { const now = Date.now() const elapsed = now - lastPhaseCheckpoint @@ -1246,9 +1297,12 @@ export class Brainy implements BrainyInterface { // instances skip recovery (readers never write; the next writer // repairs). this.generationStore = new GenerationStore(this.storage) - const generationOpenResult = await this.generationStore.open({ - readOnly: this.config.mode === 'reader' - }) + const generationOpenResult = await step( + 'generation-store.open', + 'reading the generation manifest and committed ranges, opening the fact log and the ' + + 'packed segment tier, and folding any crash-recovery replay', + () => this.generationStore.open({ readOnly: this.config.mode === 'reader' }) + ) // The generation fact log is CANONICAL state, not a derived index — no // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare @@ -1286,7 +1340,11 @@ export class Brainy implements BrainyInterface { // rollup invariants against the log head + live counters. Loud on // genuine incoherence (repairIndex heals), silent on absent/coherent, // benign-behind refreshes at the next flush. Never blocks open. - await this.verifyEntityTreeStamp() + await step( + 'verify-entity-tree-stamp', + 'comparing the entity tree\'s stamped generation and rollups against the store', + () => this.verifyEntityTreeStamp() + ) // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — @@ -1298,7 +1356,11 @@ export class Brainy implements BrainyInterface { // them from the canonical records and then re-stamps the marker AFTER the // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / // absent marker, so the next open idempotently re-rebuilds). - this._brainFormat = await readBrainFormat(this.storage) + this._brainFormat = await step( + 'read-brain-format', + 'reading the on-disk format marker that decides whether the derived indexes are stale', + () => readBrainFormat(this.storage) + ) this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH @@ -1309,7 +1371,11 @@ export class Brainy implements BrainyInterface { // upgrade verifies + stamps; retained on failure. No-op for a reader, for // non-filesystem storage, or for a brain with no persisted data. if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { - await this.createMigrationBackupIfNeeded() + await step( + 'pre-upgrade-backup', + 'snapshotting the brain directory before a one-time format rebuild (migrationBackup)', + () => this.createMigrationBackupIfNeeded() + ) } // PHASE 2 of 5 — "generation-store open+fold": GenerationStore @@ -1472,10 +1538,27 @@ export class Brainy implements BrainyInterface { `[Brainy] Rebuilding indexes after crash recovery rolled back ` + `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)` ) + // SELF-REBUILD DEFERENCE, same law as the open gate: a provider that + // is already rebuilding itself from canonical is doing exactly this + // work. Kicking a second rebuild on top of it is redundant at best. + // Safe by ordering: the crash-recovery fold ran in the generation + // store's open, BEFORE any provider was constructed, so a provider + // rebuilding now is reading the repaired canonical records. + const kick = async (leg: string, provider: { rebuild: () => Promise }) => { + const rebuilding = assessProviderRebuild(provider) + if (rebuilding) { + prodLog.narrate( + `[Brainy] crash-recovery rebuild: the ${leg} provider is already ` + + `${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.` + ) + return + } + await provider.rebuild() + } await Promise.all([ - this.metadataIndex.rebuild(), - this.index.rebuild(), - this.graphIndex.rebuild() + kick('metadata', this.metadataIndex), + kick('vector', this.index as unknown as { rebuild: () => Promise }), + kick('graph', this.graphIndex) ]) } @@ -1572,7 +1655,11 @@ export class Brainy implements BrainyInterface { // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true // once this open-time step has run. - await this.rebuildIndexesIfNeeded() + await step( + 'rebuild-indexes-if-needed', + 'the derived-index gate: each family\'s readiness verdict, and any build it asks for', + () => this.rebuildIndexesIfNeeded() + ) this.lazyRebuildCompleted = true // Check for pending data migrations @@ -1645,7 +1732,11 @@ export class Brainy implements BrainyInterface { // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) - await this._vfs.init() + await step( + 'vfs.init', + 'creating or adopting the VFS root and wiring the path resolver', + () => this._vfs!.init() + ) this._vfsInitialized = true // Mark VFS as fully initialized // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the @@ -1669,7 +1760,11 @@ export class Brainy implements BrainyInterface { const storedArtifact = await this.storage .readRawObject(LOG_AUTHORITY_PATH) .catch(() => null) - const authority = await readLogAuthority(this.storage) + const authority = await step( + 'read-log-authority', + 'reading the stored storage-authority artifact', + () => readLogAuthority(this.storage) + ) this._logAuthority = authority if (authority.authority === 'log') { this.generationStore.setLogDurability('at-ack') @@ -1680,7 +1775,12 @@ export class Brainy implements BrainyInterface { this.generationStore.getFactLog() !== null ) { try { - await this.adoptLogAuthority() + await step( + 'adopt-log-authority', + 'the adoption oracle: verifying the log against canonical before flipping this ' + + 'brain to durable-at-ack, and backfilling any curable divergence', + () => this.adoptLogAuthority() + ) prodLog.info( '[Brainy] storage authority adopted at open: generation log ' + '(fleet default; oracle green; durable-at-ack enabled)' @@ -1720,8 +1820,16 @@ export class Brainy implements BrainyInterface { // this is where it lands. if (!this.isReadOnly) { try { - await this.bridgeLegacyPendingEmbedSidecars() - await this.recoverPendingEmbedsFromLog() + await step( + 'bridge-pending-embed-sidecars', + 'migrating any pre-log deferred-embed marker files into the generation log', + () => this.bridgeLegacyPendingEmbedSidecars() + ) + await step( + 'recover-pending-embeds', + 'folding the generation log\'s deferred-embed markers back into the pending set', + () => this.recoverPendingEmbedsFromLog() + ) if (this._pendingEmbedIds.size > 0) { prodLog.info( `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + @@ -2668,6 +2776,12 @@ export class Brainy implements BrainyInterface { * engine's own cadence (callers never call flush() in hot paths). */ private noteWriteForPersistence(): void { + // THE DIRTY WITNESS. Set on every committed write — both commit paths + // (single-op and transaction) end here, and the deferred-embed worker + // lands its vectors through the single-op path — BEFORE the policy check, + // so a `'manual'` consumer's explicit flush() is never skipped either. + // Cleared by a flush that actually runs; see flush(). + this._dirtySinceLastFlush = true const cfg = this.config.persistence if (this.isReadOnly || cfg?.policy === 'manual') return this._persistDirtyWrites++ @@ -4522,6 +4636,19 @@ export class Brainy implements BrainyInterface { this._graphAdjacencyVerified = true return 'live' } + // A provider that is REBUILDING ITSELF gets a refusal that says so, + // with its own progress: open deliberately did not wait for it (see + // rebuildIndexesIfNeeded), so this door is temporarily closed and will + // open on its own. Anything else is a broken index needing a repair. + const rebuilding = assessProviderRebuild(this.graphIndex) + if (rebuilding) { + throw new GraphIndexNotReadyError( + `Graph adjacency index is ${describeRebuildProgress(rebuilding)} and is not serving ` + + `yet. find({ connected }), neighbors() and related() refuse rather than serve an ` + + `empty result. The brain is open and every other family is serving; this door opens ` + + `by itself when the provider reports serving — no action is needed.` + ) + } throw new GraphIndexNotReadyError( `Graph adjacency index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + @@ -4625,6 +4752,15 @@ export class Brainy implements BrainyInterface { this._metadataVerified = true return 'live' } + const rebuilding = assessProviderRebuild(this.metadataIndex) + if (rebuilding) { + throw new MetadataIndexNotReadyError( + `Metadata field index is ${describeRebuildProgress(rebuilding)} and is not serving ` + + `yet. find({ where }) and other filtered reads refuse rather than serve an empty ` + + `result. The brain is open and every other family is serving; this door opens by ` + + `itself when the provider reports serving — no action is needed.` + ) + } throw new MetadataIndexNotReadyError( `Metadata field index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + @@ -4754,6 +4890,15 @@ export class Brainy implements BrainyInterface { this._vectorVerified = true return 'live' } + const rebuilding = assessProviderRebuild(this.index) + if (rebuilding) { + throw new VectorIndexNotReadyError( + `Vector index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + + `Semantic find({ query }) and proximity search refuse rather than serve an empty ` + + `result. The brain is open and every other family is serving; this door opens by ` + + `itself when the provider reports serving — no action is needed.` + ) + } throw new VectorIndexNotReadyError( `Vector index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + @@ -8412,6 +8557,11 @@ export class Brainy implements BrainyInterface { */ async clear(): Promise { await this.ensureInitialized() + // A clear mutates durable state without going through a commit path, so + // it must set the dirty witness itself — otherwise a `clear()` followed by + // `flush()` would find the brain "clean" and skip the entity-tree stamp, + // leaving a stamp that describes the population this call just removed. + this._dirtySinceLastFlush = true // Clear storage await this.storage.clear() @@ -12246,6 +12396,27 @@ export class Brainy implements BrainyInterface { return } + // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been + // committed since the last flush, so every step below would re-persist + // state identical to what is already on disk — provider flushes, the + // watermark stamps, the generation counter, the entity-tree stamp — and + // print two lines announcing it. The witness is set by every committed + // write (see noteWriteForPersistence) and cleared here; a write landing + // DURING this flush sets it again, so it is never lost — the next flush + // does that write's work. This makes an unexplained flush FREE; it does + // not explain one (see _dirtySinceLastFlush). + if (!this._dirtySinceLastFlush) { + return + } + this._dirtySinceLastFlush = false + // An explicit flush IS a flush: tell the cadence so, or the very next + // write sees "30s since the last flush" (the cadence only counted its + // own) and kicks a background flush that has nothing left to do, and the + // idle timer fires two seconds later over writes this flush already + // persisted. + this._persistLastFlushAt = Date.now() + this._persistDirtyWrites = 0 + console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() @@ -16595,8 +16766,19 @@ export class Brainy implements BrainyInterface { if (legacyEntityPaths.length === 0) { // Already flat (root entities, no head-branch entities) → stamp the marker // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. - const rootEntities = await probe.listRawObjects('entities') - if (rootEntities.length > 0) { + // "Are there any entities?" is answered by ONE directory read, not by a + // recursive listing of every file in the tree: this runs on the open path + // of every store that does not yet carry the marker (a restore, a store + // built by an older release), and on a large store that listing walks the + // whole canonical tree to learn a boolean. + const oneLevel = ( + probe as unknown as { listRawPrefixes?: (prefix: string) => Promise } + ).listRawPrefixes + const hasRootEntities = + typeof oneLevel === 'function' + ? (await oneLevel.call(probe, 'entities')).length > 0 + : (await probe.listRawObjects('entities')).length > 0 + if (hasRootEntities) { await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, @@ -17095,16 +17277,34 @@ export class Brainy implements BrainyInterface { if (assessment.reasons.length > 0 && assessment.report != null) { const generation = assessment.report.generation - if (this._lastNarratedHealthGeneration.get(provider) !== generation) { - this._lastNarratedHealthGeneration.set(provider, generation) - prodLog.warn( - `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + - assessment.reasons.join('; ') - ) + // Dedupe by CONTENT, not by the provider's generation counter — see + // _lastNarratedHealth. The generation is still REPORTED (an operator + // wants to know which generation produced the verdict); it just no + // longer decides whether the line is worth saying. + const line = + `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + + assessment.reasons.join('; ') + const key = `${assessment.report.provider}\u0000${assessment.reasons.join('; ')}` + if (this._lastNarratedHealth.get(provider) !== key) { + this._lastNarratedHealth.set(provider, key) + prodLog.warn(line) } } if (assessment.readiness === 'not-ready') { + // A provider REBUILDING ITSELF gets a refusal that says so, with its + // own progress: open deliberately did not wait for it, this door is + // temporarily closed, and it opens by itself. Distinct from a broken + // index, which needs an operator. + const rebuilding = assessProviderRebuild(provider) + if (rebuilding) { + throw new ErrorClass( + `${name} index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + + `Reads of this family refuse rather than serve an empty result. The brain is open ` + + `and every other family is serving; this door opens by itself when the provider ` + + `reports serving — no action is needed.` + ) + } throw new ErrorClass( `${name} index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + @@ -17427,9 +17627,37 @@ export class Brainy implements BrainyInterface { // by awaitMigrationLock meanwhile (nothing serves from a half-built index). // Gated per-index, so a non-migrating sibling still rebuilds when it needs // to; a migrating provider is skipped even under epoch-drift or size()===0. - const metadataMigrating = this.providerIsMigrating(this.metadataIndex) - const vectorMigrating = this.providerIsMigrating(this.index) - const graphMigrating = this.providerIsMigrating(this.graphIndex) + // SELF-REBUILD DEFERENCE (the sibling of the migration lock, and the + // reason a production open took 641 seconds): a provider that reports + // `rebuildInProgress()` is ALREADY rebuilding its own index. Brainy must + // neither start a second rebuild nor WAIT for the provider's — init() + // returns, every other family serves, and that family's own doors refuse + // by name (carrying this progress) until the provider reports serving. + // A provider without the hook behaves exactly as before. + const metadataRebuilding = assessProviderRebuild(this.metadataIndex) + const vectorRebuilding = assessProviderRebuild(this.index) + const graphRebuilding = assessProviderRebuild(this.graphIndex) + for (const [leg, progress] of [ + ['metadata', metadataRebuilding], + ['vector', vectorRebuilding], + ['graph', graphRebuilding] + ] as const) { + if (progress) { + prodLog.narrate( + `[Brainy] open(): the ${leg} provider is ${describeRebuildProgress(progress)} — ` + + `open does NOT wait for it. The brain opens now, every other family serves, and ` + + `${leg} reads refuse by name until the provider reports itself serving.` + ) + } + } + + const metadataMigrating = + this.providerIsMigrating(this.metadataIndex) || metadataRebuilding !== null + const vectorMigrating = this.providerIsMigrating(this.index) || vectorRebuilding !== null + const graphMigrating = this.providerIsMigrating(this.graphIndex) || graphRebuilding !== null + // The epoch stamp certifies EVERY derived index, so it must not advance + // while any family is still being built — by a migration lock or by the + // provider itself. const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its @@ -17641,6 +17869,15 @@ export class Brainy implements BrainyInterface { // when the metadata provider holds the migration lock: a 0 count there // reflects its in-place rebuild in progress, not a missed rebuild, so // forcing a second rebuild would collide with the provider's own. + // THREE states, not two. `metadataMigrating` above is true for a + // provider holding the migration lock AND for one that reports it is + // rebuilding itself — a provider whose rebuild() returns once the + // rebuild is OWNED AND RUNNING (online, its doors refusing by name) + // legitimately reports 0 entries here, and calling that CRITICAL would + // print a false alarm and kick a redundant second rebuild on every + // first contact. The check's real class — a rebuild that ran to + // completion and produced nothing — is untouched: a provider reporting + // 0 entries with NO rebuild in progress still trips it. if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + @@ -18212,6 +18449,10 @@ export class Brainy implements BrainyInterface { */ async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() + // A repair recounts, prunes and rebuilds outside the commit paths; the + // dirty witness is set so a caller's flush after a repair does its normal + // work rather than finding the brain "clean". + this._dirtySinceLastFlush = true const startedAt = Date.now() const families: RepairFamilyReport[] = [] diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index d925c9e0..fd052c31 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -537,12 +537,29 @@ export class GenerationStore { this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) - // Discover existing generation record directories. - const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) + // Discover existing generation record directories — BY DIRECTORY NAME. + // This used to call listRawObjects(), which recurses the whole + // `_generations/` tree and returns every file in every generation, to + // extract a set of integers the top-level directory names already spell. + // MEASURED on a real store with an 11 GB generation history: the phase + // this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with + // no fold to blame — this walk is what it was doing. An adapter without + // the one-level door falls back to the recursive listing, unchanged. const seenGens = new Set() - for (const p of recordPaths) { - const gen = parseGenerationFromPath(p) - if (gen !== null) seenGens.add(gen) + const oneLevel = ( + this.storage as { listRawPrefixes?: (prefix: string) => Promise } + ).listRawPrefixes + if (typeof oneLevel === 'function') { + for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) { + const gen = Number(name) + if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen) + } + } else { + const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) + for (const p of recordPaths) { + const gen = parseGenerationFromPath(p) + if (gen !== null) seenGens.add(gen) + } } let rolledBack = 0 diff --git a/src/db/types.ts b/src/db/types.ts index 363de086..56bdef11 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -450,6 +450,21 @@ export interface GenerationStorage { deleteRawObject(path: string): Promise /** List raw object paths under a prefix (normalized, `.gz`-stripped). */ listRawObjects(prefix: string): Promise + /** + * OPTIONAL: the IMMEDIATE child directory names under a prefix — one level, + * no recursion, no file paths. + * + * Why it exists: discovering which generations are on disk needs only the + * top-level directory NAMES under `_generations/`, but the only door for it + * was `listRawObjects`, which recurses the whole tree and returns every file + * in every generation. On a store with a long history that is a full walk of + * the entire generation log, paid on EVERY open, to learn a set of integers + * the directory names already spell out. + * + * An adapter without this door keeps working — the caller falls back to the + * recursive listing. + */ + listRawPrefixes?(prefix: string): Promise /** Remove every object under a prefix (and the directory itself on disk). */ removeRawPrefix(prefix: string): Promise /** Durability barrier: fsync the given object paths (no-op in memory). */ diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index d002164e..ebd3b90c 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1052,6 +1052,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { */ private startAutoFlush(): void { this.flushTimer = setInterval(async () => { + // NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an + // index nobody has written to since the last flush has nothing to + // write, and calling into the trees (and their logging) on a cadence + // over a quiet store is exactly the idle cost this law exists to + // remove. + if ( + !this.lsmTreeVerbsBySource.hasPendingWrites() && + !this.lsmTreeVerbsByTarget.hasPendingWrites() + ) { + return + } await this.flush() }, this.config.flushInterval) // Background maintenance must never keep the host process alive — diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index e19ec145..b4f6052f 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -687,6 +687,17 @@ export class LSMTree { } } + /** + * @description Whether this tree holds anything a flush would write — + * the MemTable is non-empty. Synchronous and O(1), so a background cadence + * can ask before it does anything at all: the engine does no periodic work + * without a cause. + * @returns true when a flush would write; false when it would be a no-op. + */ + hasPendingWrites(): boolean { + return !this.memTable.isEmpty() + } + async close(): Promise { this.stopCompactionTimer() diff --git a/src/index.ts b/src/index.ts index 973136a5..edc21809 100644 --- a/src/index.ts +++ b/src/index.ts @@ -184,6 +184,7 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' +export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js' // Export plugin system export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 784ea5d8..5ec1d88e 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -107,7 +107,23 @@ export class FileSystemStorage extends BaseStorage { * "the previous writer died" without inferring either from a pid. */ private static readonly WRITER_CLOSE_FILE = '_writer.close' - private static readonly WRITER_HEARTBEAT_MS = 10_000 + /** + * How often the lock file's `lastHeartbeat` is rewritten. + * + * THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness + * is decided by PID LIVENESS alone (see isWriterLockStale) and the fence + * compares pid + hostname — no decision anywhere reads this timestamp. It + * exists so an operator inspecting a lock file, or reading the + * BRAINY_WRITER_LOCKED error, can judge liveness themselves. + * + * At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1 + * writes/s across a production process holding 21 idle brains, for a + * human-readable timestamp nothing computes with. At 60s an operator still + * sees a heartbeat inside the minute, at a sixth of the cost. With the + * clean-close record now recording orderly releases explicitly, the + * heartbeat carries even less weight than it did. + */ + private static readonly WRITER_HEARTBEAT_MS = 60_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo @@ -135,9 +151,16 @@ export class FileSystemStorage extends BaseStorage { private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_WATCH_INTERVAL_MS = 500 + /** + * The safety sweep behind the fs.watch: catches events an exotic filesystem + * dropped, and runs the stale-request GC. See startFlushRequestWatcher. + */ + private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000 private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private flushWatcherInterval?: NodeJS.Timeout + /** The inotify-backed watch on the request directory, when the FS supports one. */ + private flushWatcher?: import('node:fs').FSWatcher private flushWatcherInFlight = false private flushWatcherOnRequest?: () => Promise @@ -686,6 +709,30 @@ export class FileSystemStorage extends BaseStorage { return pruned } + /** + * @description The IMMEDIATE child directory names under a prefix — ONE + * `readdir`, no recursion, no file paths. See the seam's JSDoc + * (`src/db/types.ts`) for what this replaced: discovering the generations on + * disk walked the entire generation log on every open, reading out every + * file in every generation, to learn the set of integers the top-level + * directory names already spell. + * @param prefix - Storage-root-relative directory prefix. + * @returns The child directory names (not paths); empty when the prefix does + * not exist. + */ + public override async listRawPrefixes(prefix: string): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, prefix) + try { + const entries = await fs.promises.readdir(fullPath, { withFileTypes: true }) + return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory()) + .map((e: { name: string }) => e.name) + } catch (error: any) { + if (error?.code === 'ENOENT') return [] + throw error + } + } + /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing @@ -2361,36 +2408,115 @@ export class FileSystemStorage extends BaseStorage { /** * Start watching for cross-process flush requests. Called by Brainy.init() - * in writer mode. Polls `locks/_flush_requests/` every - * FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied - * callback (`brain.flush()`), after which an `.ack` is written to - * `locks/_flush_responses/` with the same request ID. Stale `.req` files - * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick. + * in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers + * the supplied callback (`brain.flush()`), after which an `.ack` is written + * to `locks/_flush_responses/` with the same request ID. Stale `.req` files + * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. + * + * THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request + * directory every 500 ms, per brain, for the entire life of every writer — + * armed on every non-reader brain whether or not any inspector process + * existed. MEASURED on a production process holding 21 brains: 42 directory + * reads per second on a completely idle service, plus a stale-request GC + * pass on every one of them. The engine does no periodic work without a + * cause, and a request that has not been made is not a cause. + * + * `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is + * seen SOONER than the old poll saw it. Two honest concessions ride with it: + * - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because + * `fs.watch` can miss events on network and fuse filesystems and because + * the stale-request GC needs some tick of its own. At 30s that is 0.7 + * reads/s across 21 brains where the poll cost 42. + * - a filesystem that cannot watch at all falls back to the ORIGINAL + * 500 ms poll, narrated once, because correctness outranks idle cost: + * an inspector whose request is never seen waits forever. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - if (this.flushWatcherInterval) return // already watching + // Already watching — or already ARMING. The arm is asynchronous (the + // request directory is created before it can be watched), so neither the + // watcher nor the interval exists yet during that window; the callback is + // the flag that covers it. Without this a second call in the window would + // leave two watchers and two sweeps running for the life of the store. + if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) - // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir. - this.ensureDirectoryExists(reqDir).catch(() => {}) - this.ensureDirectoryExists(ackDir).catch(() => {}) - - this.flushWatcherInterval = setInterval(() => { - if (this.flushWatcherInFlight) return // skip overlapping tick + const sweep = (): void => { + if (this.flushWatcherInFlight) return // skip overlapping sweep this.flushWatcherInFlight = true this.processFlushRequests(reqDir, ackDir).finally(() => { this.flushWatcherInFlight = false }) - }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) + } + + // Ensure both dirs exist up front so the first .req drop doesn't race with + // mkdir — and so there is a directory to watch. + void this.ensureDirectoryExists(reqDir) + .then(() => this.ensureDirectoryExists(ackDir)) + .then(() => { + if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile + try { + const watcher = fs.watch(reqDir, () => sweep()) + this.flushWatcher = watcher + watcher.on('error', (err: Error) => { + // A watch that dies mid-life must not leave the door deaf. + console.warn( + `[brainy] Flush-request watch failed (${err.message}) — falling back to polling.` + ) + this.flushWatcher?.close() + this.flushWatcher = undefined + // The SAFETY sweep must go first. It is already armed at 30s, and + // startFlushRequestPolling() declines to arm over an existing + // interval — so leaving it would quietly leave this store answering + // flush requests on a 30s cadence instead of the 500ms one the door + // promises. A degrade nobody asked for is still a degrade. + if (this.flushWatcherInterval) { + clearInterval(this.flushWatcherInterval) + this.flushWatcherInterval = undefined + } + this.startFlushRequestPolling(sweep) + }) + if (typeof watcher.unref === 'function') watcher.unref() + // The safety sweep: missed events on exotic filesystems, and the + // stale-request GC. + this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS) + if (typeof this.flushWatcherInterval.unref === 'function') { + this.flushWatcherInterval.unref() + } + // One sweep now: a request may have been dropped before the watch armed. + sweep() + } catch (err) { + console.warn( + `[brainy] Flush-request directory cannot be watched on this filesystem ` + + `(${(err as Error).message}) — polling every ` + + `${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.` + ) + this.startFlushRequestPolling(sweep) + } + }) + .catch(() => { + // The request directory could not be created; nothing to watch. A + // cross-process flush request cannot be made either, so there is + // nothing to miss. + }) + } + + /** The original 500 ms poll — the fallback when a directory cannot be watched. */ + private startFlushRequestPolling(sweep: () => void): void { + if (this.flushWatcherInterval) return + this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) if (typeof this.flushWatcherInterval.unref === 'function') { this.flushWatcherInterval.unref() } } public override stopFlushRequestWatcher(): void { + if (this.flushWatcher) { + this.flushWatcher.close() + this.flushWatcher = undefined + } if (this.flushWatcherInterval) { clearInterval(this.flushWatcherInterval) this.flushWatcherInterval = undefined diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index f518e68f..d8bcb780 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1437,6 +1437,29 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.listObjectsUnderPath(prefix) } + /** + * @description The IMMEDIATE child directory names under a prefix — one + * level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a + * separate door exists. This default derives them from the recursive + * listing, so it is never WRONG, only never faster; the filesystem adapter + * overrides it with a single directory read. + * @param prefix - Storage-root-relative directory prefix. + * @returns The child directory names (not paths), in listing order. + */ + public async listRawPrefixes(prefix: string): Promise { + await this.ensureInitialized() + const paths = await this.listObjectsUnderPath(prefix) + const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/` + const names = new Set() + for (const p of paths) { + const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null + if (rest === null) continue + const slash = rest.search(/[/\\]/) + if (slash > 0) names.add(rest.slice(0, slash)) + } + return [...names] + } + /** * Remove every object under a storage-root-relative prefix. The filesystem * adapter overrides this with a recursive directory removal; this default diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index 498f2003..f1b52e3b 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -153,3 +153,83 @@ export function assessProviderHealth(provider: unknown): ProviderHealthAssessmen reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] } } + +/** + * @description A provider's self-report that it is REBUILDING ITS OWN index + * right now. Returned by the optional `rebuildInProgress()` hook. + * + * The distinction this exists to make: a provider reporting `serving: false` + * because it is BROKEN and a provider reporting `serving: false` because it is + * BUSY BUILDING ITSELF look identical through `healthReport()` alone, and + * brainy treated both the same way — it called `rebuild()` and waited for it, + * on the foreground of `init()`. A production store whose metadata provider + * had to rebuild paid 641 SECONDS of that wait before `init()` returned, with + * every other family idle behind it. + * + * A provider that reports progress here owns its own rebuild: brainy neither + * starts one nor waits for it, `init()` returns, the other families serve, and + * THAT family's doors refuse by name — carrying this progress — until the + * provider reports itself serving. + * + * Every field but `phase` is optional and every field is a MEASUREMENT: a + * provider reports only what it actually tracks, never an estimate dressed as + * a fact. + */ +export interface ProviderRebuildProgress { + /** The provider's own name for what it is doing. Quoted verbatim in refusals. */ + phase: string + /** Units completed so far, if the provider counts them. */ + done?: number + /** Units expected in total, if the provider knows it. */ + total?: number + /** Epoch millis when this rebuild started, if the provider tracks it. */ + startedAt?: number +} + +/** A provider that can report a rebuild it is running itself. */ +interface MaybeRebuildingProvider { + rebuildInProgress?: () => ProviderRebuildProgress | null +} + +/** + * @description Ask a provider whether it is rebuilding itself right now. + * Synchronous, O(1), feature-detected: a provider without the hook reports + * nothing and is treated exactly as before. + * @param provider - Any index provider, or `null`/`undefined`. + * @returns The provider's progress, or `null` when it is not rebuilding (or + * does not implement the hook). + */ +export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null { + const p = provider as MaybeRebuildingProvider | null | undefined + if (p == null || typeof p.rebuildInProgress !== 'function') return null + try { + const progress = p.rebuildInProgress() + if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) { + return null + } + return progress + } catch { + // A throwing hook says nothing trustworthy about a rebuild; fall through to + // the ordinary health verdict rather than inventing one. + return null + } +} + +/** + * @description Render a rebuild progress report as one operator-facing clause, + * for a refusal message. Includes only what the provider actually measured. + * @param progress - The provider's report. + * @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`. + */ +export function describeRebuildProgress(progress: ProviderRebuildProgress): string { + const parts: string[] = [`"${progress.phase}"`] + if (typeof progress.done === 'number' && typeof progress.total === 'number') { + parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`) + } else if (typeof progress.done === 'number') { + parts.push(`${progress.done.toLocaleString()} done`) + } + if (typeof progress.startedAt === 'number') { + parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`) + } + return `rebuilding (${parts.join(', ')})` +} diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3cc56b2e..3e0e3d17 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2241,6 +2241,74 @@ export class MetadataIndexManager implements MetadataIndexProvider { break } + // ===== ARRAY SET OPERATORS ===== + // An element-indexed array field makes all three exact on the + // index path. They were previously ABSENT from this switch, so + // `fieldResults` kept its initial `[]` and the whole find() + // returned an empty page — a documented, matcher-implemented + // operator answering silently wrong. Served here instead. + + // hasAll: [a, b] — the field's array contains EVERY operand: + // the intersection of each element's posting set. + case 'hasAll': { + if (!Array.isArray(operand)) { + fieldResults = [] + break + } + if (operand.length === 0) { + // Vacuously true of every row that HAS the field. + const anyBitmap = (this.columnStore && this.columnStore.hasField(field)) + ? await this.columnStore.rangeQuery(field) + : await this.getExistsBitmapLegacy(field) + fieldResults = this.idMapper.intsIterableToUuids(anyBitmap) + break + } + let intersection: Set | null = null + for (const item of operand) { + const ids = new Set(await this.getIds(field, item)) + if (intersection === null) { + intersection = ids + } else { + for (const id of [...intersection]) { + if (!ids.has(id)) intersection.delete(id) + } + } + if (intersection.size === 0) break + } + fieldResults = intersection ? [...intersection] : [] + break + } + + // noneOf: [a, b] — the field's value is NONE of the operands: + // the complement of their union. + case 'noneOf': { + if (!Array.isArray(operand)) { + fieldResults = [] + break + } + const excludeInts: number[] = [] + for (const value of operand) { + for (const uuid of await this.getIds(field, value)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + } + fieldResults = this.complementIds(excludeInts) + break + } + + // excludes: value — the field's array does NOT contain the value: + // the complement of `contains`. + case 'excludes': { + const excludeInts: number[] = [] + for (const uuid of await this.getIds(field, operand)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + fieldResults = this.complementIds(excludeInts) + break + } + // ===== MISSING OPERATOR ===== // missing: boolean - equivalent to exists: !boolean case 'missing': { @@ -2257,6 +2325,27 @@ export class MetadataIndexManager implements MetadataIndexProvider { } break } + + // ===== EVERYTHING ELSE: REFUSED BY NAME, NEVER ANSWERED EMPTY ==== + // An equality/range posting index cannot evaluate a substring, a + // pattern or an array length without reading every row, and this + // path exists precisely to avoid that. It used to fall out of the + // switch with `fieldResults` still `[]`, so `find({ where: { name: + // { startsWith: 'a' } } })` returned an empty page and looked like + // an answer. An accepted operator either works or refuses — the + // matcher's own support for these operators governs in-memory + // filtering, never an index-backed find(). + default: + throw new BrainyError( + `Filter operator "${op}" on field "${rawField}" cannot be served by the ` + + `metadata index: an equality/range posting index cannot evaluate substrings, ` + + `patterns or array lengths without reading every row. It is REFUSED rather ` + + `than answered with an empty page. Filter on an indexable operator ` + + `(equals/eq, notEquals/ne, oneOf/in, noneOf, greaterThan/gt, ` + + `greaterThanOrEqual/gte, lessThan/lt, lessThanOrEqual/lte, between, contains, ` + + `excludes, hasAll, exists, missing) and narrow the rest in your own code.`, + 'INVALID_QUERY' + ) } // Intersect this operator's matches with the running set (AND semantics // for multiple operators on the same field). diff --git a/src/utils/version.ts b/src/utils/version.ts index f302eae0..327f923c 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -83,3 +83,27 @@ export function getAugmentationVersion(service: string): { augmentation: string; version: getBrainyVersion() } } + +/** + * The API-contract version this build implements — a single integer that two + * engines can compare without probing prototypes. + * + * A MINOR release is ADDITIVE: doors and error codes may be added, never + * removed or narrowed, and the contract integer does not move. A MAJOR release + * is what a REQUIRED door's removal or a behavioural narrowing costs, and it + * bumps this integer. A consumer pinning `brainyContract` in a peer range is + * therefore pinning "what I may call", not "which build I run". + * + * Declared in package.json as `"brainyContract"` so a manifest, a tool, or a + * sibling package can read it without importing the engine, and returned here + * so a running process can state its own. + */ +export const BRAINY_CONTRACT_VERSION = 1 as const + +/** + * @description The API-contract version this build implements. + * @returns The contract integer — see {@link BRAINY_CONTRACT_VERSION}. + */ +export function contractVersion(): number { + return BRAINY_CONTRACT_VERSION +} diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 59a16be4..19470b48 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -6,6 +6,7 @@ */ import { Readable, Writable } from 'stream' +import { prodLog } from '../utils/logger.js' import crypto from 'crypto' import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' @@ -66,6 +67,15 @@ export class VirtualFileSystem implements IVirtualFileSystem { private config: Required> & { rootEntityId?: string } private rootEntityId?: string private initialized = false + /** + * The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}. + */ + private rootSweep?: Promise + /** + * Where the completed old-root sweep is recorded. Engine plumbing under + * `_system/`, like every other marker there — never enumerated as data. + */ + private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -143,8 +153,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Create or find root entity this.rootEntityId = await this.initializeRoot() - // Clean up old UUID-based roots (one-time migration) - await this.cleanupOldRoots() + // Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS. + // This is a migration sweep for roots created before the fixed root id + // existed. It ran on EVERY open, forever: a filtered find over the whole + // store hunting for duplicates that a store has either always had or + // never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it + // dominates cost 43-53 SECONDS of every open, warm reopens included. + // Now: a durable marker records that the sweep has run, and a store + // carrying it never sweeps again; a store without one sweeps in the + // BACKGROUND (the sweep only removes duplicate roots — nothing serves + // from them — and it was always declared non-critical). + this.rootSweep = this.sweepOldRootsIfNeeded() // Initialize projection registry with auto-discovery of built-in projections this.projectionRegistry = new ProjectionRegistry() @@ -394,6 +413,88 @@ export class VirtualFileSystem implements IVirtualFileSystem { * * This is a one-time migration helper that can be removed in future versions. */ + /** + * @description Run the old-root sweep at most once per store, in the + * background, and record that it ran. See the call site in {@link init} for + * the measurement that made this necessary. + * @returns A promise that settles when the sweep has finished (or was + * skipped); nothing in the read path awaits it. + */ + private async sweepOldRootsIfNeeded(): Promise { + const store = this.rawObjectStore() + if (store === null) { + // A storage adapter with no raw-object door cannot carry the marker. + // Sweep every open, as before — correctness over cost. + await this.cleanupOldRoots() + return + } + try { + const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH) + if (marker !== null && marker !== undefined) return + } catch { + // Unreadable marker: sweep, and rewrite it below. + } + prodLog.narrate( + '[VFS] one-time sweep for pre-fixed-id root directories running in the background — ' + + 'the open does not wait for it, and once it has run this store never sweeps again.' + ) + const startedAt = Date.now() + await this.cleanupOldRoots() + try { + await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { + sweptAt: new Date().toISOString(), + durationMs: Date.now() - startedAt + }) + prodLog.narrate( + `[VFS] old-root sweep complete in ${Date.now() - startedAt}ms and recorded — ` + + 'no future open pays for it.' + ) + } catch (error) { + // Unrecorded sweep = the next open sweeps again. Conservative, and said + // out loud rather than quietly repeated forever. + prodLog.narrate( + `[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` + + `recorded (${(error as Error).message}) — the next open will sweep again.` + ) + } + } + + /** + * @description Settle once the background old-root sweep has finished. + * Resolves immediately when the store already carried the marker. Exists so + * tests and operators can observe the sweep instead of racing it; no read + * path waits on it. + * @returns A promise that settles with the sweep. + */ + public async whenRootSweepSettled(): Promise { + await this.rootSweep + } + + /** + * @description The brain's storage adapter, narrowed to the raw-object door + * this migration marker needs. Boundary: `Brainy.storage` is private, and + * this is the same reach-in the engine uses elsewhere for exactly this kind + * of engine-internal artifact. Returns null when the adapter has no + * raw-object door. + */ + private rawObjectStore(): { + readRawObject: (key: string) => Promise + writeRawObject: (key: string, value: unknown) => Promise + } | null { + const storage = (this.brain as unknown as { storage?: Record }).storage + if ( + storage && + typeof storage.readRawObject === 'function' && + typeof storage.writeRawObject === 'function' + ) { + return storage as unknown as { + readRawObject: (key: string) => Promise + writeRawObject: (key: string, value: unknown) => Promise + } + } + return null + } + private async cleanupOldRoots(): Promise { try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) diff --git a/tests/integration/filter-operator-conformance.test.ts b/tests/integration/filter-operator-conformance.test.ts new file mode 100644 index 00000000..628017e7 --- /dev/null +++ b/tests/integration/filter-operator-conformance.test.ts @@ -0,0 +1,151 @@ +/** + * @module tests/integration/filter-operator-conformance + * @description THE OPERATOR SET, AND WHAT EACH TOKEN DOES ON THE INDEX PATH. + * + * The contract-1 manifest splits this engine's `where` operators three ways — + * served, served-beyond-baseline, refused-by-name — and two engines must agree + * token for token. This lane is the machine-checkable side of that agreement: + * it asserts the EXACT accepted set (so a manifest can be diffed against a run + * rather than against prose), and it pins each of the three classes. + * + * The defect it closes: the metadata index's operator switch had no default + * case, so an operator it does not implement — `hasAll`, `noneOf`, `excludes`, + * `startsWith`, `endsWith`, `matches`, `length` — left the field's match set at + * its initial `[]` and `find()` returned an empty page. A documented operator, + * implemented in the in-memory matcher, answering silently wrong. Three of the + * seven are now SERVED on the index path; the other four are REFUSED BY NAME, + * because an equality/range posting index cannot evaluate a substring, a + * pattern or an array length without reading every row. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { contractVersion, BRAINY_CONTRACT_VERSION } from '../../src/utils/version.js' + +/** The accepted `where` value-operator tokens, as a sorted list. */ +const ACCEPTED_OPERATORS = [ + 'between', 'contains', 'endsWith', 'eq', 'equals', 'excludes', 'exists', + 'greaterThan', 'greaterThanOrEqual', 'gt', 'gte', 'hasAll', 'in', 'length', + 'lessThan', 'lessThanOrEqual', 'lt', 'lte', 'matches', 'missing', 'ne', + 'noneOf', 'notEquals', 'oneOf', 'startsWith' +] as const + +/** Served on the index path with exact posting-set semantics. */ +const SERVED_ON_INDEX = [ + 'between', 'contains', 'eq', 'equals', 'exists', 'greaterThan', + 'greaterThanOrEqual', 'gt', 'gte', 'in', 'lessThan', 'lessThanOrEqual', + 'lt', 'lte', 'missing', 'ne', 'notEquals', 'oneOf', + 'excludes', 'hasAll', 'noneOf' +] as const + +/** Accepted by name, refused by the index path — never answered empty. */ +const REFUSED_BY_INDEX = ['endsWith', 'length', 'matches', 'startsWith'] as const + +describe('filter operator conformance', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + async function seeded(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-operators-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + await brain.add({ + data: 'a document about ferrets', + type: NounType.Document, + metadata: { tags: ['ferret', 'small', 'furry'], team: 'alpha' } + }) + await brain.add({ + data: 'a document about whales', + type: NounType.Document, + metadata: { tags: ['whale', 'large'], team: 'beta' } + }) + await brain.flush() + return brain + } + + it('the accepted operator set is exactly these 25 tokens', async () => { + const brain = await seeded() + // The engine names its own valid set in the refusal it raises for an + // unknown token — the honest place to read it from. + let message = '' + try { + await brain.find({ where: { team: { notIn: ['alpha'] } } } as never) + } catch (err) { + message = (err as Error).message + } + expect(message).toMatch(/Unknown filter operator "notIn"/) + const listed = (message.match(/Valid operators: ([^.]+)\./)?.[1] ?? '') + .split(',') + .map((t) => t.trim()) + .filter(Boolean) + .sort() + expect(listed).toEqual([...ACCEPTED_OPERATORS].sort()) + expect(listed.length).toBe(25) + // Four tokens a sibling manifest listed as served aliases are NOT in this + // engine's set and never have been — they raise INVALID_QUERY. + for (const absent of ['is', 'isNot', 'greaterEqual', 'lessEqual']) { + expect(listed).not.toContain(absent) + await expect( + brain.find({ where: { team: { [absent]: 'alpha' } } } as never) + ).rejects.toThrow(/Unknown filter operator/) + } + }, 120_000) + + it('serves hasAll, noneOf and excludes on the index path — never an empty page', async () => { + const brain = await seeded() + + const hasAll = await brain.find({ where: { tags: { hasAll: ['ferret', 'furry'] } } } as never) + expect(hasAll.length).toBe(1) + expect((hasAll[0] as { metadata?: Record }).metadata?.team).toBe('alpha') + + const noneOf = await brain.find({ where: { team: { noneOf: ['alpha'] } } } as never) + expect(noneOf.length).toBe(1) + expect((noneOf[0] as { metadata?: Record }).metadata?.team).toBe('beta') + + const excludes = await brain.find({ where: { tags: { excludes: 'whale' } } } as never) + expect(excludes.length).toBe(1) + expect((excludes[0] as { metadata?: Record }).metadata?.team).toBe('alpha') + + // hasAll with an operand nothing carries is EMPTY because it is empty — + // the honest zero, reached by evaluating the operator. + const none = await brain.find({ where: { tags: { hasAll: ['ferret', 'whale'] } } } as never) + expect(none.length).toBe(0) + }, 120_000) + + it('refuses the four index-unserveable operators BY NAME', async () => { + const brain = await seeded() + for (const op of REFUSED_BY_INDEX) { + const operand = op === 'length' ? 3 : 'a' + await expect( + brain.find({ where: { team: { [op]: operand } } } as never), + `${op} must refuse, never answer an empty page` + ).rejects.toThrow(new RegExp(`Filter operator "${op}".*cannot be served by the metadata index`, 's')) + } + }, 120_000) + + it('declares its contract version in code and in package.json', async () => { + expect(contractVersion()).toBe(1) + expect(BRAINY_CONTRACT_VERSION).toBe(1) + const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) + expect(pkg.brainyContract).toBe(contractVersion()) + }) + + it('the three classes partition the accepted set', () => { + expect([...SERVED_ON_INDEX, ...REFUSED_BY_INDEX].sort()).toEqual([...ACCEPTED_OPERATORS].sort()) + }) +}) diff --git a/tests/integration/flush-watcher-event-driven.test.ts b/tests/integration/flush-watcher-event-driven.test.ts new file mode 100644 index 00000000..4b2e80c4 --- /dev/null +++ b/tests/integration/flush-watcher-event-driven.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/integration/flush-watcher-event-driven + * @description THE FLUSH-REQUEST WATCH IS EVENT-DRIVEN. + * + * It used to `readdir` the request directory every 500 ms, per brain, for the + * life of every writer — armed on every non-reader brain whether or not any + * inspector process existed. MEASURED on a production process holding 21 + * brains: 42 directory reads per second on a completely idle service, plus a + * stale-request GC pass on every one of them. + * + * The law: a request that has not been made is not a cause. The arrival itself + * wakes the watcher, so the request is seen SOONER than the poll saw it, and a + * slow safety sweep covers filesystems that drop watch events and the GC. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' +import * as nodeFs from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +describe('the flush-request watcher', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openWriter(): Promise<{ brain: Brainy; dir: string }> { + const dir = mkdtempSync(join(tmpdir(), 'brainy-flush-watch-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + await brain.add({ data: 'a row', type: NounType.Concept }) + await brain.flush() + return { brain, dir } + } + + it('does not poll the request directory on an idle writer', async () => { + const { dir } = await openWriter() + const reqDir = join(dir, 'locks', '_flush_requests') + + // Count real reads of the request directory over a window far longer than + // the old 500ms poll (which would have made ~16 of them). + const realReaddir = nodeFs.promises.readdir + let requestDirReads = 0 + const spy = vi + .spyOn(nodeFs.promises, 'readdir') + .mockImplementation((async (p: unknown, ...rest: unknown[]) => { + if (String(p) === reqDir) requestDirReads++ + return (realReaddir as unknown as (...a: unknown[]) => Promise)(p, ...rest) + }) as typeof nodeFs.promises.readdir) + + await new Promise((r) => setTimeout(r, 8_000)) + spy.mockRestore() + + // The old poll: 500ms → ~16 reads. The safety sweep is 30s → 0 in this window. + expect(requestDirReads).toBeLessThanOrEqual(1) + }, 120_000) + + it('answers a request that arrives, without waiting for the sweep', async () => { + const { brain, dir } = await openWriter() + const reqDir = join(dir, 'locks', '_flush_requests') + const ackDir = join(dir, 'locks', '_flush_responses') + mkdirSync(reqDir, { recursive: true }) + + // Drop a request exactly as an out-of-process inspector does. + const id = 'test-request-0001' + writeFileSync(join(reqDir, `${id}.req`), JSON.stringify({ at: Date.now() })) + + // The ack must land far sooner than the 30s safety sweep. + const deadline = Date.now() + 10_000 + let acked = false + while (Date.now() < deadline) { + try { + const entries = await nodeFs.promises.readdir(ackDir) + if (entries.some((e) => e.startsWith(id))) { acked = true; break } + } catch { /* dir not created yet */ } + await new Promise((r) => setTimeout(r, 100)) + } + expect(acked, 'the watcher must answer an arriving request').toBe(true) + void brain + }, 120_000) +}) diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts new file mode 100644 index 00000000..b5c386cf --- /dev/null +++ b/tests/integration/idle-costs-nothing.test.ts @@ -0,0 +1,152 @@ +/** + * @module tests/integration/idle-costs-nothing + * @description AN IDLE BRAIN DOES NO WORK. + * + * A flush used to re-persist state identical to what was already on disk — + * the provider flushes, the watermark stamps, the generation counter, the + * entity-tree stamp, roughly 28 writes — because `flush()` never asked whether + * anything had changed. + * + * The field observation that started this: a production process holding 21 + * brains printed "All indexes flushed to disk in 216–601ms" per brain every + * ~35 seconds and idled at 1.26 cores, with no writes for ten minutes. This + * engine's cadence is WRITE-DRIVEN, so that observation is NOT explained by + * the cadence and is not claimed to be fixed here — what is fixed is that such + * a call now costs nothing. Who was calling flush() remains open. + * + * The laws pinned here: + * (a) the persistence cadence arms only on a write — a brain nobody writes + * to flushes zero times, however long it is left open; + * (b) a flush on a clean brain is O(1): no provider is called, nothing is + * written, and nothing is printed; + * (c) one write earns exactly one flush's worth of work, and no more. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** Wait for any in-flight background flush, then let the idle timer settle. */ +async function drainCadence(brain: Brainy): Promise { + const inner = brain as unknown as { _persistBackgroundFlight: Promise | null } + await new Promise((r) => setTimeout(r, 3_000)) + await (inner._persistBackgroundFlight ?? Promise.resolve()) + await new Promise((r) => setTimeout(r, 500)) +} + +/** How long an idle brain is watched. Longer than the 30s flush interval. */ +const IDLE_WATCH_MS = 90_000 + +describe('an idle brain costs nothing', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-idle-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('flushes zero times over 90 idle seconds, and prints nothing', async () => { + const brain = await openBrain() + // One write and one flush to reach a clean, settled state — then nothing. + await brain.add({ data: 'the only write this test performs', type: NounType.Concept }) + await brain.flush() + + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + + // Watch the providers directly: a flush that runs calls all of them. + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const graphIndex = (brain as unknown as { graphIndex: { flush: () => Promise } }).graphIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const graphSpy = vi.spyOn(graphIndex, 'flush') + + try { + await new Promise((r) => setTimeout(r, IDLE_WATCH_MS)) + } finally { + console.log = origLog + } + + // (a) + (b): nothing ran, nothing was said. + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(graphSpy).not.toHaveBeenCalled() + }, 180_000) + + it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { + const brain = await openBrain() + await brain.add({ data: 'one write', type: NounType.Concept }) + await brain.flush() // this one does the work + + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + try { + await brain.flush() // ...and this one has nothing to do + await brain.flush() + await brain.flush() + } finally { + console.log = origLog + } + + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + }, 120_000) + + it('one write earns exactly one flush', async () => { + const brain = await openBrain() + await brain.add({ data: 'first', type: NounType.Concept }) + await brain.flush() + // Settle: the first write also kicked a BACKGROUND flush, which is not + // awaited by design. Drain it before counting, or its provider calls land + // inside this test's window and are attributed to the write below. + await drainCadence(brain) + + // Count the flushes that actually RAN. (Provider spies cannot answer this: + // the storage adapter's own count ledger is write-through, so a write calls + // flushCounts() on its own account, with no flush involved.) + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + const ran = () => logged.filter((l) => /All indexes flushed to disk/.test(l)).length + try { + await brain.add({ data: 'second — this is the cause', type: NounType.Concept }) + await brain.flush() + expect(ran()).toBe(1) + + // No further cause, no further work. + await brain.flush() + await brain.flush() + expect(ran()).toBe(1) + } finally { + console.log = origLog + } + }, 120_000) +}) diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts new file mode 100644 index 00000000..a46ad6a5 --- /dev/null +++ b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts @@ -0,0 +1,192 @@ +/** + * @module tests/integration/open-does-not-wait-for-a-rebuilding-provider + * @description OPEN DOES NOT WAIT FOR A PROVIDER THAT IS REBUILDING ITSELF. + * + * Measured on a production store: a metadata provider that had to rebuild made + * `init()` pay the ENTIRE rebuild on the foreground — 641 seconds — with every + * other family idle behind it, because a provider reporting `serving: false` + * because it is BUSY BUILDING and one reporting `serving: false` because it is + * BROKEN were indistinguishable, and both were answered the same way: call + * `rebuild()`, and wait. + * + * The law: a provider that reports `rebuildInProgress()` owns its own rebuild. + * `init()` returns; every other family serves; THAT family's doors refuse by + * name, carrying the provider's own progress; and the doors open by themselves + * when the provider reports serving. Nothing is ever served empty. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { ProviderRebuildProgress } from '../../src/utils/indexReadiness.js' + +/** How long the stub provider claims to be rebuilding. */ +const REBUILD_MS = 6_000 + +describe('a provider rebuilding itself never blocks open', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + it('init() returns in milliseconds, the family refuses by name, then answers', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-provider-')) + dirs.push(dir) + + // Seed a store so the open has something to (not) rebuild. + const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await seed.init() + await seed.add({ data: 'a row with a plain field', type: NounType.Concept, metadata: { kind: 'report' } }) + await seed.flush() + await seed.close() + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + + // Dress the metadata index as a provider that is rebuilding ITSELF: not + // serving, and honest about why. `init()` wires the real index first, so + // the hooks are installed on the instance as soon as it exists — the gate + // reads them by feature detection, exactly as it would a native provider's. + const rebuildStartedAt = Date.now() + const stillRebuilding = () => Date.now() - rebuildStartedAt < REBUILD_MS + let rebuildCalls = 0 + + const inner = brain as unknown as { + metadataIndex: Record + setupIndex?: unknown + } + // Install on the prototype-free instance right after construction by + // patching the property the moment init() assigns it. + const install = (target: Record) => { + const realRebuild = target.rebuild as () => Promise + target.rebuildInProgress = (): ProviderRebuildProgress | null => + stillRebuilding() + ? { phase: 'metadata shadow build', done: 4_096, total: 14_056, startedAt: rebuildStartedAt } + : null + target.healthReport = () => ({ + provider: 'metadata', + healthy: !stillRebuilding(), + serving: !stillRebuilding(), + generation: 1, + invariants: [], + unledgered: [] + }) + target.rebuild = async () => { + rebuildCalls++ + return realRebuild.call(target) + } + } + + // init() constructs the metadata index; patch as soon as it exists, before + // the gate consults it. A microtask hop after the index is assigned is + // enough because the gate runs later in the same init. + const initPromise = (async () => { + const originalEnsure = (brain as unknown as { setupIndex?: () => unknown }).setupIndex + void originalEnsure + return brain.init() + })() + // Patch on the first tick the index exists. + const patcher = setInterval(() => { + if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { + install(inner.metadataIndex) + } + }, 1) + const startedAt = Date.now() + try { + await initPromise + } finally { + clearInterval(patcher) + } + const openMs = Date.now() - startedAt + + // If the patch did not land before the gate ran, this test proves nothing — + // say so loudly rather than passing vacuously. + expect( + typeof inner.metadataIndex.rebuildInProgress, + 'the stub provider was never installed — the test is vacuous' + ).toBe('function') + + // 1. The open did not wait out the rebuild. + expect(openMs).toBeLessThan(REBUILD_MS) + // 2. And brainy did not start a rebuild of its own on top of the provider's. + expect(rebuildCalls).toBe(0) + + // 3. The family's door refuses BY NAME, carrying the provider's progress. + let refusal: Error | null = null + try { + await brain.find({ where: { kind: 'report' } } as never) + } catch (err) { + refusal = err as Error + } + expect(refusal, 'a not-serving metadata family must refuse, never serve empty').not.toBeNull() + expect(refusal!.message).toMatch(/metadata shadow build/i) + expect(refusal!.message).toMatch(/4,096\/14,056/) + expect(refusal!.message).toMatch(/no action is needed/i) + + // 4. Other families keep serving — the brain is open. + const all = await brain.getNouns?.({ pagination: { limit: 1 } } as never) + expect(all ?? true).toBeTruthy() + + // 5. When the provider reports itself serving, the door opens by itself. + await new Promise((r) => setTimeout(r, REBUILD_MS)) + ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false + await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() + }, 180_000) + + it('a rebuilding provider reporting 0 entries is not a CRITICAL, and gets no second rebuild', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-critical-')) + dirs.push(dir) + const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await seed.init() + await seed.add({ data: 'a stored entity', type: NounType.Concept }) + await seed.flush() + await seed.close() + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + + let rebuildCalls = 0 + const errors: string[] = [] + const origError = console.error + console.error = ((...a: unknown[]) => { errors.push(a.map(String).join(' ')) }) as typeof console.error + + const inner = brain as unknown as { metadataIndex: Record } + const patcher = setInterval(() => { + if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { + const target = inner.metadataIndex + target.rebuildInProgress = () => ({ phase: 'online metadata rebuild', startedAt: Date.now() }) + target.healthReport = () => ({ + provider: 'metadata', healthy: false, serving: false, + generation: 1, invariants: [], unledgered: [] + }) + // The shape the native engine now has: the index reports NOTHING while + // its rebuild runs online behind refusing doors. + target.getStats = async () => ({ totalEntries: 0 }) + target.rebuild = async () => { rebuildCalls++ } + } + }, 1) + try { + await brain.init() + } finally { + clearInterval(patcher) + console.error = origError + } + + expect( + typeof inner.metadataIndex.rebuildInProgress, + 'the stub provider was never installed — the test is vacuous' + ).toBe('function') + expect(errors.filter((l) => /CRITICAL: Metadata index has 0 entries/.test(l))).toEqual([]) + expect(rebuildCalls).toBe(0) + }, 180_000) +}) diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts new file mode 100644 index 00000000..f84f4413 --- /dev/null +++ b/tests/integration/vfs-root-sweep-once.test.ts @@ -0,0 +1,106 @@ +/** + * @module tests/integration/vfs-root-sweep-once + * @description THE OLD-ROOT SWEEP RUNS ONCE PER STORE, NOT ONCE PER OPEN. + * + * The VFS bootstrap ran a filtered `find()` over the whole store on EVERY + * open, hunting for root directories created before the fixed root id existed + * — duplicates a store has either always had or never will. MEASURED on a + * 14,056-noun / 72,679-verb store: the phase it dominates cost 43–53 SECONDS + * of every open, warm reopens included. + * + * The law: a migration sweep is caused by the store's state, not by the clock + * or the open count. It runs behind the doors, records that it ran, and a + * store carrying that record never sweeps again. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' + +describe('the VFS old-root sweep', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function open(dir: string): Promise { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('sweeps on the first open, records it, and never sweeps again', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-')) + dirs.push(dir) + + const sweepSpy = vi.spyOn( + VirtualFileSystem.prototype as unknown as { cleanupOldRoots: () => Promise }, + 'cleanupOldRoots' + ) + + const first = await open(dir) + await (first.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).toHaveBeenCalledTimes(1) + // The record is durable engine plumbing under _system/, like every other marker. + expect( + existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || + existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) + ).toBe(true) + + await first.add({ data: 'a row so the store is not trivially empty', type: NounType.Concept }) + await first.flush() + await first.close() + brains.splice(brains.indexOf(first), 1) + + sweepSpy.mockClear() + const second = await open(dir) + await (second.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).not.toHaveBeenCalled() + + await second.close() + brains.splice(brains.indexOf(second), 1) + + // ...and a third open, to prove it is the record and not a one-off. + sweepSpy.mockClear() + const third = await open(dir) + await (third.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).not.toHaveBeenCalled() + }, 180_000) + + it('the open does not wait for the sweep', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) + dirs.push(dir) + + const proto = VirtualFileSystem.prototype as unknown as Record< + string, + (...args: unknown[]) => Promise + > + const real = proto.cleanupOldRoots + proto.cleanupOldRoots = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, 4_000)) + return real.apply(this, args) + } + try { + const startedAt = Date.now() + const brain = await open(dir) + const openMs = Date.now() - startedAt + expect(openMs).toBeLessThan(3_000) + await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + } finally { + proto.cleanupOldRoots = real + } + }, 180_000) +})