diff --git a/scripts/release.sh b/scripts/release.sh index ec861913..0e6a9c43 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -27,8 +27,18 @@ for arg in "$@"; do esac done +# An explicit version (e.g. "8.0.0-rc.1") may be passed in place of patch/minor/major. +EXPLICIT_VERSION="" +if [[ "$RELEASE_TYPE" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + EXPLICIT_VERSION="$RELEASE_TYPE" +fi + +# Release on whatever branch we're on β€” RC lines live on feature branches, not main. +CURRENT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" + echo -e "${BLUE}πŸš€ Brainy Release Script${NC}" -echo -e "${BLUE}Release type: ${RELEASE_TYPE}${NC}\n" +echo -e "${BLUE}Release type: ${RELEASE_TYPE}${NC}" +echo -e "${BLUE}Branch: ${CURRENT_BRANCH}${NC}\n" if [ "$DRY_RUN" = true ]; then echo -e "${YELLOW}⚠️ DRY RUN MODE - No changes will be made${NC}\n" @@ -57,7 +67,8 @@ echo -e "${GREEN}βœ… Build successful${NC}\n" if [ "$SKIP_TESTS" = false ]; then echo -e "${BLUE}3️⃣ Running tests...${NC}" if [ "$DRY_RUN" = false ]; then - npm test + # Full CI gate: unit + integration (test:ci = test:ci-unit && test:ci-integration). + npm run test:ci fi echo -e "${GREEN}βœ… Tests passed${NC}\n" else @@ -74,24 +85,41 @@ MAJOR="${VERSION_PARTS[0]}" MINOR="${VERSION_PARTS[1]}" PATCH="${VERSION_PARTS[2]}" -case $RELEASE_TYPE in - major) - NEW_VERSION="$((MAJOR + 1)).0.0" - ;; - minor) - NEW_VERSION="${MAJOR}.$((MINOR + 1)).0" - ;; - patch) - NEW_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" - ;; - *) - echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" - echo "Usage: ./scripts/release.sh [patch|minor|major] [--dry-run]" - exit 1 - ;; -esac +if [ -n "$EXPLICIT_VERSION" ]; then + NEW_VERSION="$EXPLICIT_VERSION" +else + case $RELEASE_TYPE in + major) + NEW_VERSION="$((MAJOR + 1)).0.0" + ;; + minor) + NEW_VERSION="${MAJOR}.$((MINOR + 1)).0" + ;; + patch) + NEW_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" + ;; + *) + echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" + echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run]" + exit 1 + ;; + esac +fi -echo -e "${BLUE}New version: ${NEW_VERSION}${NC}\n" +# A version with a hyphen (e.g. 8.0.0-rc.1) is a prerelease: publish under the 'rc' +# dist-tag (NOT 'latest') and mark the GitHub release as a prerelease. +PRERELEASE=false +NPM_TAG="latest" +if [[ "$NEW_VERSION" == *"-"* ]]; then + PRERELEASE=true + NPM_TAG="rc" +fi + +echo -e "${BLUE}New version: ${NEW_VERSION}${NC}" +if [ "$PRERELEASE" = true ]; then + echo -e "${YELLOW}⚠️ Prerelease β†’ npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" +fi +echo "" if [ "$DRY_RUN" = true ]; then echo -e "${YELLOW}DRY RUN: Would release version ${NEW_VERSION}${NC}" @@ -149,17 +177,23 @@ echo -e "${GREEN}βœ… Tag created${NC}\n" # Step 9: Push to GitHub echo -e "${BLUE}8️⃣ Pushing to GitHub...${NC}" -git push --follow-tags origin main +git push --follow-tags origin "$CURRENT_BRANCH" echo -e "${GREEN}βœ… Pushed to GitHub${NC}\n" # Step 10: Publish to npm -echo -e "${BLUE}9️⃣ Publishing to npm...${NC}" -npm publish +echo -e "${BLUE}9️⃣ Publishing to npm (dist-tag: ${NPM_TAG})...${NC}" +npm publish --tag "$NPM_TAG" +# Brainy is the only PUBLIC @soulcraft package β€” verify visibility after every publish. +npm access get status @soulcraft/brainy || true echo -e "${GREEN}βœ… Published to npm${NC}\n" # Step 11: Create GitHub release echo -e "${BLUE}πŸ”Ÿ Creating GitHub release...${NC}" -gh release create "v${NEW_VERSION}" --generate-notes +if [ "$PRERELEASE" = true ]; then + gh release create "v${NEW_VERSION}" --generate-notes --prerelease +else + gh release create "v${NEW_VERSION}" --generate-notes +fi echo -e "${GREEN}βœ… GitHub release created${NC}\n" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index 2edcd9a7..bf9dc5de 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -200,9 +200,11 @@ function updateMetricAdd(state: MetricState, val: number, op: AggregationOp): vo state.m2 = (state.m2 ?? 0) + (val - oldMean) * (val - mean) } - // Percentile: track the numeric value multiset (matches Cortex's value_counts) for the - // exact-percentile computation. (distinctCount is tracked separately, on raw values.) - if (op === 'percentile') { + // Track the numeric value multiset for percentile AND min/max. min/max need it + // to recompute the extreme after a delete WITHOUT scanning entities β€” that scan + // was the 7.x infinite-loop / hang (materialized aggregates fed back in). + // (distinctCount is tracked separately, on raw values.) + if (op === 'percentile' || op === 'min' || op === 'max') { if (!state.valueCounts) state.valueCounts = {} const key = String(val) state.valueCounts[key] = (state.valueCounts[key] ?? 0) + 1 @@ -214,9 +216,9 @@ function updateMetricAdd(state: MetricState, val: number, op: AggregationOp): vo * Note: removing from Welford's is the inverse update. */ function updateMetricRemove(state: MetricState, val: number, op: AggregationOp): void { - // Percentile: decrement the numeric value multiset (drop the key at zero). - // (distinctCount is decremented separately, on raw values.) - if (op === 'percentile' && state.valueCounts) { + // Decrement the numeric value multiset for percentile AND min/max (drop the key + // at zero). (distinctCount is decremented separately, on raw values.) + if ((op === 'percentile' || op === 'min' || op === 'max') && state.valueCounts) { const key = String(val) const c = state.valueCounts[key] if (c !== undefined) { @@ -241,6 +243,29 @@ function updateMetricRemove(state: MetricState, val: number, op: AggregationOp): } } +/** + * Recompute a metric's min/max from its value multiset β€” O(distinct values), fully + * in-memory, NO entity scan. Called lazily on query when a delete of the current + * extreme marked it stale. (Scanning entities to recompute is what hung 7.x.) + */ +function recomputeMinMaxFromCounts(state: MetricState): void { + const keys = state.valueCounts ? Object.keys(state.valueCounts) : [] + if (keys.length === 0) { + state.min = Infinity + state.max = -Infinity + return + } + let mn = Infinity + let mx = -Infinity + for (const k of keys) { + const n = Number(k) + if (n < mn) mn = n + if (n > mx) mx = n + } + state.min = mn + state.max = mx +} + /** * Exact percentile over a value multiset, using linear interpolation between closest ranks * (numpy 'linear' / type-7): `rank = pΒ·(nβˆ’1)`, interpolating between the floor and ceil @@ -610,7 +635,7 @@ export class AggregationIndex { // Collect all groups let results: AggregateResult[] = [] - for (const group of stateMap.values()) { + for (const [serialized, group] of stateMap.entries()) { // Skip empty groups (all metrics at zero count) const hasData = Object.values(group.metrics).some(m => m.count > 0) if (!hasData) continue @@ -639,11 +664,24 @@ export class AggregationIndex { metrics[metricName] = state.count > 0 ? state.sum / state.count : 0 break case 'min': - metrics[metricName] = state.min === Infinity ? 0 : state.min - break - case 'max': - metrics[metricName] = state.max === -Infinity ? 0 : state.max + case 'max': { + // Lazily recompute the extreme from the value multiset if a delete of + // the current min/max marked it stale (hang-free; no entity scan). + const staleSet = this.staleMinMax.get(params.name) + if (staleSet && staleSet.has(`${serialized}:${metricName}`)) { + recomputeMinMaxFromCounts(state) + staleSet.delete(`${serialized}:${metricName}`) + } + metrics[metricName] = + metricDef.op === 'min' + ? state.min === Infinity + ? 0 + : state.min + : state.max === -Infinity + ? 0 + : state.max break + } case 'variance': metrics[metricName] = state.count > 1 ? (state.m2 ?? 0) / (state.count - 1) : 0 break diff --git a/src/brainy.ts b/src/brainy.ts index 3bc1e871..2f6a5293 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -5,7 +5,8 @@ * NO STUBS, NO MOCKS, REAL IMPLEMENTATION */ -import { v4 as uuidv4 } from './universal/uuid.js' +import { v4 as uuidv4, v7 as uuidv7 } from './universal/uuid.js' +import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from './utils/idNormalization.js' import { JsHnswVectorIndex } from './hnsw/hnswIndex.js' // TypeAwareHNSWIndex removed from default path β€” single unified JsHnswVectorIndex is faster // for the 99% of queries that don't filter by type (avoids searching 42 separate graphs) @@ -1206,6 +1207,20 @@ export class Brainy implements BrainyInterface { * }) * ``` */ + /** + * @description Mint a fresh, time-ordered entity id (UUID v7) WITHOUT a write. + * Assign ids client-side to reference an entity before it exists β€” forward + * references in `transact()`, deterministic `relate()`, no writeβ†’getβ†’relate + * round-trip. Time-ordered, so ids sort by creation time. + * @returns A new UUID v7 string. + * @example + * const id = brain.newId() + * await brain.add({ id, type: NounType.Document, data: 'draft' }) + */ + newId(): string { + return uuidv7() + } + async add(params: AddParams): Promise { this.assertWritable('add') await this.ensureInitialized() @@ -1235,17 +1250,22 @@ export class Brainy implements BrainyInterface { // missing-subtype check via the `isVFSEntity` marker. this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata) + // Id normalization (8.0): a natural string key (e.g. 'user-123') is coerced + // to a STABLE UUID (v5) so the engine only ever sees a UUID, while the + // caller's original string is preserved under ORIGINAL_ID_KEY for reads. + // A real UUID passes through; no id mints a fresh v7. Done BEFORE the + // ifAbsent check so a natural-key upsert is idempotent against the same key. + const { id, originalId } = coerceNewEntityId(params.id) + // ifAbsent (7.31.0) β€” by-ID idempotent insert. Only meaningful when a custom id // is supplied; a freshly generated UUID can never collide. Returns the existing - // id without writing if the entity is already present (no throw, no overwrite). + // (canonical) id without writing if the entity is already present (no throw, + // no overwrite). if (params.id && params.ifAbsent) { - const existing = await this.storage.getNounMetadata(params.id) - if (existing) return params.id + const existing = await this.storage.getNounMetadata(id) + if (existing) return id } - // Generate ID if not provided - const id = params.id || uuidv4() - // Get or compute vector const vector = params.vector || (await this.embed(params.data)) @@ -1263,6 +1283,9 @@ export class Brainy implements BrainyInterface { // Only metadata fields are queryable via find({ where }). const storageMetadata = { ...params.metadata, + // Preserve the caller's original (non-UUID) id when normalized, so reads + // can surface it. A real UUID passes through with no _originalId. + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), @@ -1299,8 +1322,12 @@ export class Brainy implements BrainyInterface { service: params.service, data: params.data, ...(params.createdBy && { createdBy: params.createdBy }), - // Only custom fields in metadata - metadata: params.metadata || {} + // Only custom fields in metadata (plus the preserved original id, which + // is a custom field β€” surfaced on read under ORIGINAL_ID_KEY). + metadata: { + ...(params.metadata || {}), + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } } // Execute atomically with transaction system @@ -1477,6 +1504,14 @@ export class Brainy implements BrainyInterface { async get(id: string, options?: GetOptions): Promise | null> { await this.ensureInitialized() + // Id normalization (8.0): a caller may read by their natural key β€” resolve + // it to the same canonical UUID add() stored. A real UUID passes through. + // Only a non-empty string is normalized; empty/null/undefined fall through + // to the storage layer's structural-validation throw (preserved contract). + if (typeof id === 'string' && id !== '') { + id = resolveEntityId(id) + } + // Route to metadata-only or full entity based on options const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) @@ -1527,6 +1562,12 @@ export class Brainy implements BrainyInterface { async batchGet(ids: string[], options?: GetOptions): Promise>> { await this.ensureInitialized() + // Id normalization (8.0): resolve each id to its canonical UUID so callers + // can batch-read by natural key. resolveEntityId is idempotent on real + // UUIDs, so internal callers passing canonical ids are unaffected. The + // returned map is keyed by the canonical (stored) id. + ids = ids.map((id) => resolveEntityId(id)) + const results = new Map>() if (ids.length === 0) return results @@ -2003,6 +2044,10 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateUpdateParams(params) + // Id normalization (8.0): resolve a natural key to the canonical UUID add() + // stored, so update() targets the same entity. A real UUID passes through. + params = { ...params, id: resolveEntityId(params.id) } + // Reserved fields arriving via the metadata patch are remapped to their // canonical top-level location, mirroring add()'s lift. Without this the // patch value survived the merge but was then clobbered by the @@ -2221,6 +2266,10 @@ export class Brainy implements BrainyInterface { await this.ensureInitialized() + // Id normalization (8.0): resolve a natural key to the canonical UUID add() + // stored, so remove() deletes the same entity. A real UUID passes through. + id = resolveEntityId(id) + // Get entity metadata and related verbs before deletion const metadata = await this.storage.getNounMetadata(id) const noun = await this.storage.getNoun(id) @@ -2536,6 +2585,12 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateRelateParams(params) + // Id normalization (8.0): resolve BOTH endpoints so a caller may relate by + // natural key on either side. Each maps to the same canonical UUID add() + // stored; real UUIDs pass through. (The relationship's own id is an + // engine-minted UUID β€” relation ids are never caller-supplied here.) + params = { ...params, from: resolveEntityId(params.from), to: resolveEntityId(params.to) } + // Reserved fields arriving via the metadata bag are normalized to their // canonical top-level params before enforcement β€” mirror of add()'s lift. params = this.remapReservedRelateMetadata(params) @@ -2924,10 +2979,19 @@ export class Brainy implements BrainyInterface { await this.ensureInitialized() // Handle string ID shorthand: related(id) -> related({ from: id }) - const params = typeof paramsOrId === 'string' + const rawParams = typeof paramsOrId === 'string' ? { from: paramsOrId } : (paramsOrId || {}) + // Id normalization (8.0): resolve the anchor id(s) so a caller may traverse + // by natural key. Each maps to the canonical UUID add() stored; real UUIDs + // pass through. Mutates a copy so the caller's params object is untouched. + const params: RelatedParams = { + ...rawParams, + ...(rawParams.from !== undefined && { from: resolveEntityId(rawParams.from) }), + ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }) + } + const limit = params.limit || 100 const offset = params.offset || 0 @@ -3668,9 +3732,24 @@ export class Brainy implements BrainyInterface { await this.ensureIndexesLoaded() // Parse natural language queries - const params: FindParams = + let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query + // Id normalization (8.0): resolve the graph-traversal anchor id(s) so a + // caller may constrain by natural key. Each maps to the canonical UUID + // add() stored; real UUIDs pass through. Done once here so every downstream + // consumer of params.connected sees canonical ids. + if (params.connected && (params.connected.from !== undefined || params.connected.to !== undefined)) { + params = { + ...params, + connected: { + ...params.connected, + ...(params.connected.from !== undefined && { from: resolveEntityId(params.connected.from) }), + ...(params.connected.to !== undefined && { to: resolveEntityId(params.connected.to) }) + } + } + } + // Zero-config validation (static import for performance) validateFindParams(params) @@ -4503,7 +4582,10 @@ export class Brainy implements BrainyInterface { let idsToDelete: string[] = [] if (params.ids) { - idsToDelete = params.ids + // Id normalization (8.0): resolve each supplied id to the canonical UUID + // add() stored, so callers may delete by natural key. The find()-derived + // path below already yields canonical ids. + idsToDelete = params.ids.map((id) => resolveEntityId(id)) } else if (params.type || params.where) { // Find entities to delete const entities = await this.find({ @@ -6206,15 +6288,20 @@ export class Brainy implements BrainyInterface { this.enforceTrackedFieldValues({ subtype: params.subtype } as Record, 'top-level') this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata) + // Id normalization (8.0) β€” mirror of add(): a natural key coerces to a + // STABLE UUID (v5), preserving the original under ORIGINAL_ID_KEY; a real + // UUID passes through; no id mints a fresh v7. Done BEFORE ifAbsent so a + // natural-key upsert is idempotent against the same key. + const { id, originalId } = coerceNewEntityId(params.id) + // ifAbsent β€” idempotent by-id insert, resolved against batch + store. if (params.id && params.ifAbsent) { - const existing = await this.planGetEntity(state, params.id) + const existing = await this.planGetEntity(state, id) if (existing) { - return params.id + return id } } - const id = params.id || uuidv4() const vector = params.vector || (await this.embed(params.data)) if (!this.dimensions) { this.dimensions = vector.length @@ -6234,6 +6321,9 @@ export class Brainy implements BrainyInterface { const now = Date.now() const storageMetadata = { ...params.metadata, + // Preserve the caller's original (non-UUID) id when normalized β€” mirror + // of add(). A real UUID passes through with no _originalId. + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), @@ -6264,7 +6354,10 @@ export class Brainy implements BrainyInterface { service: params.service, data: params.data, ...(params.createdBy && { createdBy: params.createdBy }), - metadata: params.metadata || {} + metadata: { + ...(params.metadata || {}), + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } } plan.operations.push( @@ -6297,6 +6390,9 @@ export class Brainy implements BrainyInterface { // remap to their dedicated param (top-level wins), system-managed fields // drop with a one-shot warning. const params = this.remapReservedUpdateMetadata(rawParams as UpdateParams) + // Id normalization (8.0) β€” mirror of update(): a natural key resolves to the + // canonical UUID add() stored. A real UUID passes through. + params.id = resolveEntityId(params.id) this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') if (params.subtype !== undefined) { this.enforceTrackedFieldValues({ subtype: params.subtype } as Record, 'top-level') @@ -6447,7 +6543,9 @@ export class Brainy implements BrainyInterface { if (!op.id || typeof op.id !== 'string') { throw new Error(`transact(): remove operation requires an entity id (got ${JSON.stringify(op.id)})`) } - const id = op.id + // Id normalization (8.0) β€” mirror of remove(): a natural key resolves to the + // canonical UUID add() stored. A real UUID passes through. + const id = resolveEntityId(op.id) const pending = state.nouns.get(id) const metadata = pending ? pending.metadata : await this.storage.getNounMetadata(id) @@ -6531,6 +6629,11 @@ export class Brainy implements BrainyInterface { validateRelateParams(rawParams as RelateParams) // Same reserved-field normalization as relate(). const params = this.remapReservedRelateMetadata(rawParams as RelateParams) + // Id normalization (8.0) β€” mirror of relate(): resolve BOTH endpoints to the + // canonical UUID add() stored, so a relate op may reference either side by + // natural key. Real UUIDs pass through. (Relationship ids are engine-minted.) + params.from = resolveEntityId(params.from) + params.to = resolveEntityId(params.to) this.enforceSubtypeOnRelate('relate', params.type, params.subtype, params.metadata) const fromEntity = await this.planGetEntity(state, params.from) @@ -11451,7 +11554,7 @@ export class Brainy implements BrainyInterface { // undefined (default) β†’ no auto-detection (safe default) // false β†’ no auto-detection // [] β†’ no auto-detection - // ['@soulcraft/cortex'] β†’ load only these explicitly listed packages + // ['@soulcraft/cor'] β†’ load only these explicitly listed packages // Note: plugins registered via brain.use() are always activated regardless of config const pluginConfig = this.config.plugins if (Array.isArray(pluginConfig) && pluginConfig.length > 0) { diff --git a/src/db/db.ts b/src/db/db.ts index 928d2560..ac6e4a2e 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -64,6 +64,7 @@ import { splitVerbMetadataRecord } from '../types/reservedFields.js' import { v4 as uuidv4 } from '../universal/uuid.js' +import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from '../utils/idNormalization.js' import { EntityNotFoundError } from '../errors/notFound.js' import { SpeculativeOverlayError } from './errors.js' import type { GenerationStore } from './generationStore.js' @@ -268,6 +269,11 @@ export class Db { async get(id: string, options?: GetOptions): Promise | null> { this.assertUsable('get') + // Id normalization (8.0): resolve a natural key to the canonical UUID the + // write path stored, so this view reads by natural key consistently. A real + // UUID passes through. Overlay keys are canonical (see with()). + id = resolveEntityId(id) + if (this.overlay && this.overlay.nouns.has(id)) { return this.overlay.nouns.get(id) ?? null } @@ -447,12 +453,19 @@ export class Db { async related(paramsOrId?: string | RelatedParams): Promise[]> { this.assertUsable('related') - const params: RelatedParams = + const rawParams: RelatedParams = typeof paramsOrId === 'string' ? { from: paramsOrId } : (paramsOrId ?? {}) + // Id normalization (8.0): resolve the anchor id(s) so this view traverses by + // natural key consistently with the live engine. Real UUIDs pass through. + const params: RelatedParams = { + ...rawParams, + ...(rawParams.from !== undefined && { from: resolveEntityId(rawParams.from) }), + ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }) + } const historical = this.isHistorical() if (!historical && !this.overlay) { - return this.host.related(paramsOrId) + return this.host.related(params) } if (params.cursor !== undefined) { @@ -587,7 +600,11 @@ export class Db { const service = op.service ?? (typeof reserved.service === 'string' ? reserved.service : undefined) - const id = op.id ?? uuidv4() + // Id normalization (8.0) β€” mirror of the committed transact() add + // path: a natural key coerces to a STABLE UUID (v5), preserving the + // original under ORIGINAL_ID_KEY; a real UUID passes through; no id + // mints a fresh id so the overlay keys match the durable path. + const { id, originalId } = coerceNewEntityId(op.id) const now = Date.now() overlay.nouns.set(id, { id, @@ -595,7 +612,10 @@ export class Db { type: op.type, ...(subtype !== undefined && { subtype }), data: op.data, - metadata: custom as T, + metadata: { + ...(custom as object), + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } as T, ...(service !== undefined && { service }), createdAt: now, updatedAt: now, @@ -606,11 +626,14 @@ export class Db { break } case 'update': { - const base = await speculativeGet(op.id) + // Id normalization (8.0): resolve a natural key to the canonical UUID + // the write path stored. A real UUID passes through. + const updateId = resolveEntityId(op.id) + const base = await speculativeGet(updateId) if (!base) { throw new EntityNotFoundError( - op.id, - `with(): entity ${op.id} not found at generation ${this.gen}` + updateId, + `with(): entity ${updateId} not found at generation ${this.gen}` ) } // Same reserved-field normalization as the committed update path. @@ -627,7 +650,7 @@ export class Db { op.merge !== false ? ({ ...(base.metadata as object), ...custom } as T) : ((op.metadata !== undefined ? custom : base.metadata) as T) - overlay.nouns.set(op.id, { + overlay.nouns.set(updateId, { ...base, ...(op.type !== undefined && { type: op.type }), ...(subtype !== undefined && { subtype }), @@ -642,38 +665,45 @@ export class Db { break } case 'remove': { - overlay.nouns.set(op.id, null) + // Id normalization (8.0): resolve a natural key to the canonical UUID + // the write path stored. A real UUID passes through. + const removeId = resolveEntityId(op.id) + overlay.nouns.set(removeId, null) // Tombstone overlay relations touching the removed entity; base // relations are cascade-filtered at read time in related(). for (const [verbId, relation] of overlay.verbs) { - if (relation && (relation.from === op.id || relation.to === op.id)) { + if (relation && (relation.from === removeId || relation.to === removeId)) { overlay.verbs.set(verbId, null) } } break } case 'relate': { - const from = await speculativeGet(op.from) - const to = await speculativeGet(op.to) + // Id normalization (8.0) β€” mirror of relate(): resolve BOTH endpoints + // to the canonical UUID the write path stored. Real UUIDs pass through. + const relFrom = resolveEntityId(op.from) + const relTo = resolveEntityId(op.to) + const from = await speculativeGet(relFrom) + const to = await speculativeGet(relTo) if (!from) { - throw new EntityNotFoundError(op.from, `with(): source entity ${op.from} not found`) + throw new EntityNotFoundError(relFrom, `with(): source entity ${relFrom} not found`) } if (!to) { - throw new EntityNotFoundError(op.to, `with(): target entity ${op.to} not found`) + throw new EntityNotFoundError(relTo, `with(): target entity ${relTo} not found`) } // Dedupe against the view (overlay first, then the edges visible // at this generation via the record layer) β€” mirror of relate(). let duplicate: Relation | undefined for (const relation of overlay.verbs.values()) { - if (relation && relation.from === op.from && relation.to === op.to && relation.type === op.type) { + if (relation && relation.from === relFrom && relation.to === relTo && relation.type === op.type) { duplicate = relation break } } if (!duplicate) { - const existing = await this.related({ from: op.from, type: op.type }) - duplicate = existing.find((relation) => relation.to === op.to) + const existing = await this.related({ from: relFrom, type: op.type }) + duplicate = existing.find((relation) => relation.to === relTo) } if (duplicate) break @@ -694,8 +724,8 @@ export class Db { const id = uuidv4() overlay.verbs.set(id, { id, - from: op.from, - to: op.to, + from: relFrom, + to: relTo, type: op.type, ...(subtype !== undefined && { subtype }), weight: weight ?? 1.0, @@ -709,8 +739,8 @@ export class Db { const reverseId = uuidv4() overlay.verbs.set(reverseId, { id: reverseId, - from: op.to, - to: op.from, + from: relTo, + to: relFrom, type: op.type, ...(subtype !== undefined && { subtype }), weight: weight ?? 1.0, diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 9d7c0931..61e6a86a 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1425,10 +1425,10 @@ export interface BrainyConfig { // Plugin configuration // Controls which plugins are loaded during init() - // - undefined (default): Auto-detect installed plugins (@soulcraft/cortex, etc.) + // - undefined (default): Auto-detect installed plugins (@soulcraft/cor, etc.) // - false: No plugins β€” skip auto-detection entirely // - []: No plugins β€” skip auto-detection entirely - // - ['@soulcraft/cortex']: Load only specified plugins, no auto-detection + // - ['@soulcraft/cor']: Load only specified plugins, no auto-detection plugins?: string[] | false // Logging configuration diff --git a/src/universal/uuid.ts b/src/universal/uuid.ts index 72932f1a..38432393 100644 --- a/src/universal/uuid.ts +++ b/src/universal/uuid.ts @@ -1,24 +1,222 @@ /** - * Universal UUID implementation - * Framework-friendly: Works in all environments + * @module universal/uuid + * @description Framework-friendly UUID utilities used across Brainy β€” works in + * Node, Bun, Deno, and browsers (relies only on the global Web Crypto API, with + * pure-JS fallbacks). + * + * Three generators + helpers: + * - {@link v4} β€” random UUID (legacy default). + * - {@link v7} β€” time-ordered UUID (RFC 9562). The 8.0 default for new ids: + * lexicographically sortable by creation time, which keeps the id↔int mapper + * and any range scan locality-friendly. + * - {@link v5} β€” deterministic namespaced UUID (RFC 4122, SHA-1). Used by the + * 8.0 id-normalization layer to map a caller's non-UUID string key to a STABLE + * UUID, so a native engine (which requires UUID ids for its int mapper) always + * sees a valid UUID while the caller can keep using their natural key. */ +/** Render 16 bytes as a canonical hyphenated UUID string. */ +function bytesToUuid(b: Uint8Array): string { + const h: string[] = [] + for (let i = 0; i < 16; i++) h.push(b[i].toString(16).padStart(2, '0')) + return ( + `${h[0]}${h[1]}${h[2]}${h[3]}-${h[4]}${h[5]}-${h[6]}${h[7]}-` + + `${h[8]}${h[9]}-${h[10]}${h[11]}${h[12]}${h[13]}${h[14]}${h[15]}` + ) +} + +/** Parse a hyphenated UUID string into its 16 bytes. */ +function uuidToBytes(uuid: string): Uint8Array { + const hex = uuid.replace(/-/g, '') + const b = new Uint8Array(16) + for (let i = 0; i < 16; i++) b[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) + return b +} + +/** Fill a byte array with cryptographically-strong (or Math.random fallback) randomness. */ +function randomBytes(n: number): Uint8Array { + const out = new Uint8Array(n) + if (typeof crypto !== 'undefined' && crypto.getRandomValues) { + crypto.getRandomValues(out) + } else { + for (let i = 0; i < n; i++) out[i] = Math.floor(Math.random() * 256) + } + return out +} + +/** + * @description Canonical UUID-format check (any version/variant). NOT a version + * assertion β€” just "is this string shaped like a UUID". + * @param value - The string to test. + * @returns `true` if `value` matches the 8-4-4-4-12 hex UUID format. + */ +export function isUUID(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value) +} + +/** + * @description Random (version 4) UUID. + * @returns A random UUID string. + */ export function v4(): string { - // Use crypto.randomUUID if available (Node.js 19+, modern browsers) if (typeof crypto !== 'undefined' && crypto.randomUUID) { return crypto.randomUUID() } - - // Fallback implementation for older environments return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { - const r = Math.random() * 16 | 0 - const v = c === 'x' ? r : (r & 0x3 | 0x8) + const r = (Math.random() * 16) | 0 + const v = c === 'x' ? r : (r & 0x3) | 0x8 return v.toString(16) }) } -// Named export to match uuid package API -export { v4 as uuidv4 } +/** + * @description Time-ordered (version 7) UUID β€” 48-bit Unix-ms timestamp prefix + * + random tail. Two ids minted in order sort in order as strings. + * @returns A version-7 UUID string. + */ +export function v7(): string { + const ms = Date.now() + const b = new Uint8Array(16) + // 48-bit big-endian millisecond timestamp. + b[0] = Math.floor(ms / 0x10000000000) & 0xff + b[1] = Math.floor(ms / 0x100000000) & 0xff + b[2] = Math.floor(ms / 0x1000000) & 0xff + b[3] = Math.floor(ms / 0x10000) & 0xff + b[4] = Math.floor(ms / 0x100) & 0xff + b[5] = ms & 0xff + const rand = randomBytes(10) + b[6] = (rand[0] & 0x0f) | 0x70 // version 7 + b[7] = rand[1] + b[8] = (rand[2] & 0x3f) | 0x80 // variant 10xx + for (let i = 9; i < 16; i++) b[i] = rand[i - 6] // rand[3..9] + return bytesToUuid(b) +} -// Default export for convenience -export default { v4 } \ No newline at end of file +/** Left-rotate a 32-bit word. */ +function rotl(n: number, s: number): number { + return ((n << s) | (n >>> (32 - s))) >>> 0 +} + +/** + * @description SHA-1 of a byte array β†’ 20 bytes. Pure JS so it runs synchronously + * everywhere (Web Crypto's digest is async-only). Used solely by {@link v5}. + * @param msg - Input bytes. + * @returns The 20-byte SHA-1 digest. + */ +function sha1(msg: Uint8Array): Uint8Array { + const ml = msg.length * 8 + const withByte = msg.length + 1 + const padLen = withByte % 64 <= 56 ? 56 - (withByte % 64) : 120 - (withByte % 64) + const total = msg.length + 1 + padLen + 8 + const buf = new Uint8Array(total) + buf.set(msg, 0) + buf[msg.length] = 0x80 + const hi = Math.floor(ml / 0x100000000) + const lo = ml >>> 0 + buf[total - 8] = (hi >>> 24) & 0xff + buf[total - 7] = (hi >>> 16) & 0xff + buf[total - 6] = (hi >>> 8) & 0xff + buf[total - 5] = hi & 0xff + buf[total - 4] = (lo >>> 24) & 0xff + buf[total - 3] = (lo >>> 16) & 0xff + buf[total - 2] = (lo >>> 8) & 0xff + buf[total - 1] = lo & 0xff + + let h0 = 0x67452301 + let h1 = 0xefcdab89 + let h2 = 0x98badcfe + let h3 = 0x10325476 + let h4 = 0xc3d2e1f0 + + const w = new Array(80) + for (let i = 0; i < total; i += 64) { + for (let t = 0; t < 16; t++) { + w[t] = + ((buf[i + t * 4] << 24) | + (buf[i + t * 4 + 1] << 16) | + (buf[i + t * 4 + 2] << 8) | + buf[i + t * 4 + 3]) >>> + 0 + } + for (let t = 16; t < 80; t++) { + w[t] = rotl(w[t - 3] ^ w[t - 8] ^ w[t - 14] ^ w[t - 16], 1) + } + let a = h0 + let b = h1 + let c = h2 + let d = h3 + let e = h4 + for (let t = 0; t < 80; t++) { + let f: number + let k: number + if (t < 20) { + f = (b & c) | (~b & d) + k = 0x5a827999 + } else if (t < 40) { + f = b ^ c ^ d + k = 0x6ed9eba1 + } else if (t < 60) { + f = (b & c) | (b & d) | (c & d) + k = 0x8f1bbcdc + } else { + f = b ^ c ^ d + k = 0xca62c1d6 + } + const tmp = (rotl(a, 5) + (f >>> 0) + e + k + w[t]) >>> 0 + e = d + d = c + c = rotl(b, 30) + b = a + a = tmp + } + h0 = (h0 + a) >>> 0 + h1 = (h1 + b) >>> 0 + h2 = (h2 + c) >>> 0 + h3 = (h3 + d) >>> 0 + h4 = (h4 + e) >>> 0 + } + + const out = new Uint8Array(20) + const hs = [h0, h1, h2, h3, h4] + for (let i = 0; i < 5; i++) { + out[i * 4] = (hs[i] >>> 24) & 0xff + out[i * 4 + 1] = (hs[i] >>> 16) & 0xff + out[i * 4 + 2] = (hs[i] >>> 8) & 0xff + out[i * 4 + 3] = hs[i] & 0xff + } + return out +} + +/** + * @description Deterministic version-5 (SHA-1, namespaced) UUID. The same + * `(name, namespace)` always yields the same UUID β€” the property the + * id-normalization layer relies on to turn a stable string key into a stable + * UUID id. + * @param name - The name to hash (e.g. a caller-supplied string id). + * @param namespace - A UUID-format namespace (defaults to {@link BRAINY_ID_NAMESPACE}). + * @returns A version-5 UUID string. + */ +export function v5(name: string, namespace: string = BRAINY_ID_NAMESPACE): string { + const ns = uuidToBytes(namespace) + const nameBytes = new TextEncoder().encode(name) + const data = new Uint8Array(16 + nameBytes.length) + data.set(ns, 0) + data.set(nameBytes, 16) + const hash = sha1(data) + const b = hash.slice(0, 16) + b[6] = (b[6] & 0x0f) | 0x50 // version 5 + b[8] = (b[8] & 0x3f) | 0x80 // variant 10xx + return bytesToUuid(b) +} + +/** + * @description The fixed namespace for Brainy's stringβ†’UUID normalization. A + * stable constant (spells "brainy" in the first bytes) so the same caller string + * maps to the same UUID across processes and machines. + */ +export const BRAINY_ID_NAMESPACE = '62726169-6e79-5000-8000-000000000000' + +// Named export to match the `uuid` package API. +export { v4 as uuidv4, v7 as uuidv7, v5 as uuidv5 } + +export default { v4, v7, v5, isUUID, BRAINY_ID_NAMESPACE } diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index 5c02ce1b..7a54846f 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -73,7 +73,7 @@ export class EntityIdSpaceExceeded extends Error { `EntityIdMapper: nextId ${attempted} would exceed u32::MAX ` + `(${U32_ENTITY_ID_MAX}). The JS fallback mapper caps at u32 to ` + `match the metadata index's Roaring32 bitmap width. For >4.29 B ` + - `entities, install @soulcraft/cortex and configure the binary ` + + `entities, install @soulcraft/cor and configure the binary ` + `mapper with idSpace: 'u64' (mmap-backed extendible-hash KV).`, ) this.name = 'EntityIdSpaceExceeded' diff --git a/src/utils/idNormalization.ts b/src/utils/idNormalization.ts new file mode 100644 index 00000000..16ed1f5d --- /dev/null +++ b/src/utils/idNormalization.ts @@ -0,0 +1,65 @@ +/** + * @module utils/idNormalization + * @description The 8.0 entity-id normalization layer. Every id that enters + * Brainy is coerced to a valid UUID, because a native engine maps ids to compact + * integers and requires real UUIDs (`encode()` rejects arbitrary strings). + * + * The rules (decided as AR.1 = "normalize"): + * - **No id supplied** β†’ a fresh time-ordered {@link v7} UUID (sortable by + * creation time; friendly to the id↔int mapper and range scans). + * - **A valid UUID** β†’ used as-is. + * - **Any other string** (a natural key like `'user-123'`) β†’ a STABLE + * {@link v5} UUID derived from it, so the same key always maps to the same id + * and the caller can keep using their natural key on every read/relate while + * the engine only ever sees a UUID. The original string is preserved so reads + * can surface it. + * + * Two entry points: {@link coerceNewEntityId} at creation (add/relate), and + * {@link resolveEntityId} at lookup (get/update/remove/relate endpoints). + */ +import { v5, v7, isUUID } from '../universal/uuid.js' + +/** Metadata key under which a caller's original (non-UUID) id is preserved. */ +export const ORIGINAL_ID_KEY = '_originalId' + +/** + * @description The result of normalizing an id at creation time. + */ +export interface CoercedId { + /** The canonical UUID to store and hand to the engine. */ + id: string + /** The caller's original string, present only when it was normalized (non-UUID). */ + originalId?: string +} + +/** + * @description Normalize an id supplied at CREATION (add/relate). Generates a + * time-ordered UUID when none is given, passes a real UUID through, and maps any + * other string to a stable namespaced UUID (recording the original). + * @param id - The caller-supplied id, or undefined to auto-generate. + * @returns The canonical UUID and, when normalized, the original string. + * @example + * coerceNewEntityId() // β†’ { id: } + * coerceNewEntityId('user-123') // β†’ { id: , originalId: 'user-123' } + */ +export function coerceNewEntityId(id?: string | null): CoercedId { + if (id === undefined || id === null || id === '') { + return { id: v7() } + } + if (isUUID(id)) { + return { id } + } + return { id: v5(id), originalId: id } +} + +/** + * @description Normalize an id used for LOOKUP (get/update/remove/relate + * endpoints) to the same canonical UUID `coerceNewEntityId` produced β€” so a + * caller can read/relate by their natural key transparently. A real UUID passes + * through; any other string maps to its stable namespaced UUID. + * @param id - The id to resolve. + * @returns The canonical UUID. + */ +export function resolveEntityId(id: string): string { + return isUUID(id) ? id : v5(id) +} diff --git a/tests/integration/aggregation-delete.test.ts b/tests/integration/aggregation-delete.test.ts new file mode 100644 index 00000000..6d4d10e7 --- /dev/null +++ b/tests/integration/aggregation-delete.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/integration/aggregation-delete + * @description Regression for BR-AGG-DELETE-HANG (consumer-reported on 7.x): once a + * brain had ever deleted an entity, raw `queryAggregate` never resolved (7.x recomputed + * MIN/MAX after a delete by SCANNING entities, which fed materialized-aggregate entities + * back into aggregation β†’ infinite loop). 8.0 must (a) RESOLVE after deletes β€” never hang β€” + * and (b) return the CORRECT new min/max after deleting the current extreme (recomputed + * from the in-memory value multiset, no entity scan). + */ +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** Deterministic 384-dim vectors so the embedder is never invoked. */ +function vec(seed: number): number[] { + return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100) +} + +/** Resolve a query or fail loudly if it hangs (the 7.x symptom). */ +function noHang(p: Promise, label: string, ms = 4000): Promise { + return Promise.race([ + p, + new Promise((_, rej) => + setTimeout(() => rej(new Error(`${label} HUNG (did not resolve in ${ms}ms)`)), ms) + ) + ]) +} + +describe('queryAggregate across deletes β€” no hang + correct min/max', () => { + const brains: Brainy[] = [] + afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + }) + + it('resolves after deletes and recomputes min/max from the multiset', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + brains.push(brain) + await brain.init() + + brain.defineAggregate({ + name: 'stats_by_cat', + source: { type: NounType.Thing }, + groupBy: ['category'], + metrics: { + total: { op: 'sum', field: 'amount' }, + lo: { op: 'min', field: 'amount' }, + hi: { op: 'max', field: 'amount' }, + n: { op: 'count' } + } + }) + + const ids: Record = {} + for (const [label, amount] of [ + ['a', 10], + ['b', 20], + ['c', 30], + ['d', 40] + ] as const) { + ids[label] = await brain.add({ + data: `e-${label}`, + type: NounType.Thing, + vector: vec(amount), + metadata: { category: 'x', amount } + }) + } + + const pick = (rows: any[]) => rows.find((r) => r.groupKey.category === 'x') + + let g = pick(await noHang(brain.queryAggregate('stats_by_cat'), 'queryAggregate (initial)')) + expect(g).toBeDefined() + expect(g.metrics.total).toBe(100) + expect(g.metrics.lo).toBe(10) + expect(g.metrics.hi).toBe(40) + expect(g.metrics.n).toBe(4) + + // Delete the CURRENT min (10) AND the CURRENT max (40) β€” the case that hung in 7.x. + await brain.remove(ids.a) + await brain.remove(ids.d) + + g = pick(await noHang(brain.queryAggregate('stats_by_cat'), 'queryAggregate (after delete)')) + expect(g).toBeDefined() + expect(g.metrics.n).toBe(2) + expect(g.metrics.total).toBe(50) // 20 + 30 + expect(g.metrics.lo).toBe(20) // new min after deleting 10 + expect(g.metrics.hi).toBe(30) // new max after deleting 40 + + // A further delete still resolves and stays correct. + await brain.remove(ids.b) + g = pick(await noHang(brain.queryAggregate('stats_by_cat'), 'queryAggregate (after 2nd delete)')) + expect(g.metrics.n).toBe(1) + expect(g.metrics.lo).toBe(30) + expect(g.metrics.hi).toBe(30) + }) +}) diff --git a/tests/integration/find-unified-integration.test.ts b/tests/integration/find-unified-integration.test.ts index f826aaa0..4370295e 100644 --- a/tests/integration/find-unified-integration.test.ts +++ b/tests/integration/find-unified-integration.test.ts @@ -9,6 +9,10 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' +// 8.0 id normalization: the test factory seeds natural-key ids ('alice', …). +// The engine maps each to a stable UUID (v5) and preserves the original under +// _originalId, so entity.id is the canonical UUID and lookups round-trip by key. +import { v5 } from '../../src/universal/uuid' import { createTestEntity, createTestRelation, @@ -110,8 +114,9 @@ describe('Unified Find() Integration Tests', () => { }) expect(results).toHaveLength(3) - // First result should be Alice herself with high similarity - expect(results[0].entity.id).toBe('alice') + // First result should be Alice herself with high similarity (her id is + // the canonical v5 mapping of the natural key 'alice'). + expect(results[0].entity.id).toBe(v5('alice')) expect(results[0].score).toBeGreaterThan(0.95) }) @@ -138,8 +143,8 @@ describe('Unified Find() Integration Tests', () => { expect(results.length).toBeGreaterThan(0) // Should find Bob and Charlie (Alice's friends) - const bobResult = results.find((r: any) => r.entity.id === 'bob') - const charlieResult = results.find((r: any) => r.entity.id === 'charlie') + const bobResult = results.find((r: any) => r.entity.id === v5('bob')) + const charlieResult = results.find((r: any) => r.entity.id === v5('charlie')) expect(bobResult || charlieResult).toBeDefined() }) @@ -154,7 +159,7 @@ describe('Unified Find() Integration Tests', () => { expect(results.length).toBeGreaterThan(0) // Should find Alice (who is friends with Bob) - const aliceResult = results.find((r: any) => r.entity.id === 'alice') + const aliceResult = results.find((r: any) => r.entity.id === v5('alice')) expect(aliceResult).toBeDefined() }) @@ -169,8 +174,8 @@ describe('Unified Find() Integration Tests', () => { expect(results.length).toBeGreaterThan(0) // Should find both Bob and Charlie - const bobResult = results.find((r: any) => r.entity.id === 'bob') - const charlieResult = results.find((r: any) => r.entity.id === 'charlie') + const bobResult = results.find((r: any) => r.entity.id === v5('bob')) + const charlieResult = results.find((r: any) => r.entity.id === v5('charlie')) expect(bobResult).toBeDefined() expect(charlieResult).toBeDefined() }) @@ -204,9 +209,10 @@ describe('Unified Find() Integration Tests', () => { }) expect(results.length).toBeGreaterThan(0) - // Should find direct connections (Bob, Charlie) and second-degree (Diana) - const directConnections = ['bob', 'charlie'] - const secondDegreeConnections = ['diana'] + // Should find direct connections (Bob, Charlie) and second-degree (Diana). + // Compare against the canonical v5 ids the engine stored for each key. + const directConnections = [v5('bob'), v5('charlie')] + const secondDegreeConnections = [v5('diana')] const hasDirect = results.some((r: any) => directConnections.includes(r.entity.id)) const hasSecondDegree = results.some((r: any) => secondDegreeConnections.includes(r.entity.id)) @@ -326,7 +332,7 @@ describe('Unified Find() Integration Tests', () => { expect(results.length).toBeGreaterThan(0) // Charlie should be highly ranked due to direct connection - const charlieResult = results.find((r: any) => r.entity.id === 'charlie') + const charlieResult = results.find((r: any) => r.entity.id === v5('charlie')) if (charlieResult) { expect(charlieResult.score).toBeGreaterThan(0.5) } diff --git a/tests/integration/id-normalization.test.ts b/tests/integration/id-normalization.test.ts new file mode 100644 index 00000000..1ea1a221 --- /dev/null +++ b/tests/integration/id-normalization.test.ts @@ -0,0 +1,225 @@ +/** + * @module tests/integration/id-normalization + * @description Correctness gate for the 8.0 transparent entity-id normalization + * layer (#18). A caller may use a natural string key (e.g. `'user-1'`) anywhere + * an id is accepted; it is mapped to a STABLE UUID (`v5('user-1')`) on creation + * AND on every lookup, so it round-trips transparently while the engine only + * ever sees a UUID. The caller's original string is preserved under + * {@link ORIGINAL_ID_KEY} in the stored entity metadata. A real UUID passes + * through untouched (no `_originalId`). + * + * The layer is ALL-OR-NOTHING: every creation AND lookup path must normalize + * identically, or round-trips break. These tests prove the contract holds + * end-to-end across `add` / `get` / `update` / `remove` / `relate` / `related` + * / `find({ connected })` / `transact` / `addMany` / `relateMany`, plus + * determinism (same key β†’ same UUID, upsert not duplicate), UUID passthrough, + * and the v7 defaults for no-id `add()` and `newId()`. + * + * All entities carry explicit 384-dim vectors so no test invokes the embedder. + */ + +import { describe, it, expect } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { v5, v7, isUUID } from '../../src/universal/uuid.js' +import { ORIGINAL_ID_KEY } from '../../src/utils/idNormalization.js' + +/** Deterministic 384-dim vector so no test ever invokes the embedder. */ +function vec(seed: number): number[] { + return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100) +} + +/** Fresh in-memory brain (no embedder, no subtype enforcement). */ +async function makeBrain(): Promise { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + return brain +} + +describe('id normalization β€” transparent string-key round-trips', () => { + it('1. add() returns v5(key); get(key) and get(returnedId) both resolve; _originalId preserved', async () => { + const brain = await makeBrain() + + const returnedId = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) + + // The returned id is the stable v5 mapping, a real UUID. + expect(returnedId).toBe(v5('user-1')) + expect(isUUID(returnedId)).toBe(true) + + // get() by the natural key AND by the canonical id both resolve. + const byKey = await brain.get('user-1') + const byId = await brain.get(returnedId) + expect(byKey).not.toBeNull() + expect(byId).not.toBeNull() + expect(byKey!.id).toBe(returnedId) + expect(byId!.id).toBe(returnedId) + + // The caller's original string is surfaced under ORIGINAL_ID_KEY. + expect(byKey!.metadata[ORIGINAL_ID_KEY]).toBe('user-1') + }) + + it('2. relate() by string keys; related(key) and related({from:key}) return the edge to v5(toKey)', async () => { + const brain = await makeBrain() + + await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) + await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document }) + + await brain.relate({ from: 'user-1', to: 'doc-1', type: VerbType.Creates }) + + const expectedTo = v5('doc-1') + + const viaString = await brain.related('user-1') + expect(viaString.length).toBe(1) + expect(viaString[0].from).toBe(v5('user-1')) + expect(viaString[0].to).toBe(expectedTo) + + const viaFrom = await brain.related({ from: 'user-1' }) + expect(viaFrom.length).toBe(1) + expect(viaFrom[0].to).toBe(expectedTo) + + // And the inbound anchor resolves too. + const viaTo = await brain.related({ to: 'doc-1' }) + expect(viaTo.length).toBe(1) + expect(viaTo[0].from).toBe(v5('user-1')) + }) + + it('3. update() by string key reflects on get(key)', async () => { + const brain = await makeBrain() + + await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { role: 'admin' } }) + await brain.update({ id: 'user-1', metadata: { role: 'owner' } }) + + const e = await brain.get('user-1') + expect(e).not.toBeNull() + expect((e!.metadata as any).role).toBe('owner') + // Original id still surfaced after update. + expect(e!.metadata[ORIGINAL_ID_KEY]).toBe('user-1') + }) + + it('4. remove() by string key deletes; get(key) is null', async () => { + const brain = await makeBrain() + + await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) + expect(await brain.get('user-1')).not.toBeNull() + + await brain.remove('user-1') + expect(await brain.get('user-1')).toBeNull() + // The canonical id is gone too. + expect(await brain.get(v5('user-1'))).toBeNull() + }) + + it('5. find({ connected: { from: key } }) resolves the anchor key', async () => { + const brain = await makeBrain() + + await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) + await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document }) + await brain.relate({ from: 'user-1', to: 'doc-1', type: VerbType.Creates }) + + const results = await brain.find({ connected: { from: 'user-1' } }) + const ids = results.map((r) => r.id) + expect(ids).toContain(v5('doc-1')) + }) + + it('6. transact() add+relate by string keys round-trips with consistent canonical ids', async () => { + const brain = await makeBrain() + + // Seed user-1 so the relate op has a target to point at. + await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) + + const db = await brain.transact([ + { op: 'add', id: 'u-2', vector: vec(3), type: NounType.Person }, + { op: 'relate', from: 'u-2', to: 'user-1', type: VerbType.RelatedTo } + ]) + + // Receipt ids resolve to the canonical mappings. + expect(db.receipt.ids[0]).toBe(v5('u-2')) + + // get('u-2') works and surfaces the original id. + const u2 = await brain.get('u-2') + expect(u2).not.toBeNull() + expect(u2!.id).toBe(v5('u-2')) + expect(u2!.metadata[ORIGINAL_ID_KEY]).toBe('u-2') + + // The relation links the right canonical ids. + const rels = await brain.related('u-2') + expect(rels.length).toBe(1) + expect(rels[0].from).toBe(v5('u-2')) + expect(rels[0].to).toBe(v5('user-1')) + }) + + it('7. addMany() + relateMany() with string ids round-trip', async () => { + const brain = await makeBrain() + + const added = await brain.addMany({ + items: [ + { id: 'a-1', vector: vec(10), type: NounType.Person }, + { id: 'b-1', vector: vec(11), type: NounType.Document } + ] + }) + expect(added.successful).toContain(v5('a-1')) + expect(added.successful).toContain(v5('b-1')) + + const relIds = await brain.relateMany({ + items: [{ from: 'a-1', to: 'b-1', type: VerbType.Creates }] + }) + expect(relIds.length).toBe(1) + + const a1 = await brain.get('a-1') + expect(a1).not.toBeNull() + expect(a1!.metadata[ORIGINAL_ID_KEY]).toBe('a-1') + + const rels = await brain.related('a-1') + expect(rels.length).toBe(1) + expect(rels[0].to).toBe(v5('b-1')) + }) + + it('8. determinism: same key maps to same UUID β€” two adds upsert ONE entity, not two', async () => { + const brain = await makeBrain() + + const id1 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 1 } }) + const id2 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 2 } }) + + // Stable mapping across calls. + expect(id1).toBe(id2) + expect(id1).toBe(v5('user-1')) + + // Only ONE entity exists (the second add overwrote the first). + const count = await brain.getNounCount() + expect(count).toBe(1) + + const e = await brain.get('user-1') + expect((e!.metadata as any).n).toBe(2) + }) + + it('9. valid-UUID passthrough: a real UUID is kept verbatim with NO _originalId', async () => { + const brain = await makeBrain() + + const realUuid = v7() + const returnedId = await brain.add({ id: realUuid, vector: vec(5), type: NounType.Thing }) + + expect(returnedId).toBe(realUuid) + + const e = await brain.get(realUuid) + expect(e).not.toBeNull() + expect(e!.id).toBe(realUuid) + expect(e!.metadata[ORIGINAL_ID_KEY]).toBeUndefined() + }) + + it('10. no-id add() mints a v7; newId() mints a v7', async () => { + const brain = await makeBrain() + + const autoId = await brain.add({ vector: vec(6), type: NounType.Thing }) + expect(isUUID(autoId)).toBe(true) + // v7 carries version nibble '7' at the canonical position. + expect(autoId[14]).toBe('7') + + const minted = brain.newId() + expect(isUUID(minted)).toBe(true) + expect(minted[14]).toBe('7') + + // The auto-id entity round-trips by its returned canonical id. + const e = await brain.get(autoId) + expect(e).not.toBeNull() + expect(e!.metadata[ORIGINAL_ID_KEY]).toBeUndefined() + }) +}) diff --git a/tests/unit/brainy/update-reserved-metadata-remap.test.ts b/tests/unit/brainy/update-reserved-metadata-remap.test.ts index ab1750e5..8e1a97c3 100644 --- a/tests/unit/brainy/update-reserved-metadata-remap.test.ts +++ b/tests/unit/brainy/update-reserved-metadata-remap.test.ts @@ -252,7 +252,10 @@ describe('reserved-field metadata remap (8.0 contract)', () => { const entity = await speculative.get('spec-entity') expect(entity?.confidence).toBe(0.65) - expect(entity?.metadata).toEqual({ custom: 'spec' }) + // 8.0 id normalization: a natural-key id is mapped to a stable UUID and + // the caller's original string is preserved under _originalId β€” surfaced + // here exactly as the durable transact()/add() paths do. + expect(entity?.metadata).toEqual({ custom: 'spec', _originalId: 'spec-entity' }) await speculative.release() await base.release() }) diff --git a/tests/unit/universal/uuid.test.ts b/tests/unit/universal/uuid.test.ts new file mode 100644 index 00000000..309cc10c --- /dev/null +++ b/tests/unit/universal/uuid.test.ts @@ -0,0 +1,47 @@ +/** + * @module tests/unit/universal/uuid + * @description Pins the in-house UUID utilities (v4/v5/v7 + isUUID). v5 is checked + * against a known RFC/python `uuid5(NAMESPACE_DNS, 'python.org')` vector β€” if the + * bundled SHA-1 or the v5 byte-twiddling were wrong, this fails. + */ +import { describe, it, expect } from 'vitest' +import { v4, v5, v7, isUUID, BRAINY_ID_NAMESPACE } from '../../../src/universal/uuid.js' + +describe('universal/uuid', () => { + it('v5 matches the known DNS-namespace vector (SHA-1 correctness)', () => { + // python: uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') + expect(v5('python.org', '6ba7b810-9dad-11d1-80b4-00c04fd430c8')).toBe( + '886313e1-3b8a-5372-9b90-0c9aee199e5d' + ) + }) + + it('v5 is deterministic, valid, version-5, and input-sensitive', () => { + const a = v5('user-123') + expect(v5('user-123')).toBe(a) // deterministic + expect(isUUID(a)).toBe(true) + expect(a[14]).toBe('5') // version nibble + expect(v5('user-124')).not.toBe(a) // different name β†’ different id + expect(BRAINY_ID_NAMESPACE).toBe('62726169-6e79-5000-8000-000000000000') + }) + + it('v7 is valid, version-7, unique, and time-sortable', () => { + const a = v7() + expect(isUUID(a)).toBe(true) + expect(a[14]).toBe('7') + const ids = Array.from({ length: 8 }, () => v7()) + expect(new Set(ids).size).toBe(8) // unique + // the timestamp prefix makes lexical order track creation order + const sorted = [...ids].sort() + expect(sorted[sorted.length - 1] >= sorted[0]).toBe(true) + }) + + it('v4 is valid', () => { + expect(isUUID(v4())).toBe(true) + }) + + it('isUUID rejects non-UUID strings', () => { + expect(isUUID('user-123')).toBe(false) + expect(isUUID('hello')).toBe(false) + expect(isUUID('')).toBe(false) + }) +}) diff --git a/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts b/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts index bbca5dbc..0b92f768 100644 --- a/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts +++ b/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts @@ -47,7 +47,7 @@ describe('EntityIdMapper U32 IdSpace ceiling (Brainy 8.0)', () => { expect(err.ceiling).toBe(0xffff_ffff) expect(err.attempted).toBe(0x1_0000_0000) expect(err.message).toMatch(/u32::MAX/) - expect(err.message).toMatch(/cortex/) + expect(err.message).toMatch(/@soulcraft\/cor/) expect(err.message).toMatch(/idSpace: 'u64'/) })