feat(namespace): aggregation reads under the law + epoch 3 (the key-split rebuild) + THE ARMING COMMIT — the capability constant, the law module, and the typed refusals export from the package root; both engines' conformance suites light on this signal
This commit is contained in:
parent
c2fb28a2f7
commit
7492b6cb59
7 changed files with 73 additions and 22 deletions
|
|
@ -14,7 +14,22 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { StorageAdapter, HNSWNounWithMetadata } from '../coreTypes.js'
|
import type { StorageAdapter, HNSWNounWithMetadata } from '../coreTypes.js'
|
||||||
import { resolveEntityField } from '../coreTypes.js'
|
import { parseFieldAddress, readEntityFieldAddress } from '../db/fieldAddressing.js'
|
||||||
|
import type { HNSWNounWithMetadata as AddressedEntity } from '../coreTypes.js'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read a user-supplied field name under the one addressing law (sealed
|
||||||
|
* 2026-08-03): bare / `metadata.` = the user's metadata field, `system.<x>` =
|
||||||
|
* the ruled engine scalar, malformed = typed refusal. The aggregation engine
|
||||||
|
* NEVER resolves names any other way — the pre-law resolver made bare
|
||||||
|
* `subtype`/`confidence` read engine scalars, silently shadowing user fields.
|
||||||
|
*/
|
||||||
|
function readAddressed(e: unknown, name: string): unknown {
|
||||||
|
return readEntityFieldAddress(
|
||||||
|
e as AddressedEntity,
|
||||||
|
parseFieldAddress(name, 'entity')
|
||||||
|
)
|
||||||
|
}
|
||||||
import type {
|
import type {
|
||||||
AggregateDefinition,
|
AggregateDefinition,
|
||||||
AggregateGroupState,
|
AggregateGroupState,
|
||||||
|
|
@ -97,7 +112,7 @@ function matchesSource(entity: Record<string, unknown>, source: AggregateDefinit
|
||||||
const e = entity as unknown as HNSWNounWithMetadata
|
const e = entity as unknown as HNSWNounWithMetadata
|
||||||
const resolved: Record<string, unknown> = {}
|
const resolved: Record<string, unknown> = {}
|
||||||
for (const key of Object.keys(source.where)) {
|
for (const key of Object.keys(source.where)) {
|
||||||
resolved[key] = resolveEntityField(e, key)
|
resolved[key] = readAddressed(e, key)
|
||||||
}
|
}
|
||||||
if (!matchesMetadataFilter(resolved, source.where)) return false
|
if (!matchesMetadataFilter(resolved, source.where)) return false
|
||||||
}
|
}
|
||||||
|
|
@ -129,11 +144,11 @@ function computeGroupKeys(
|
||||||
|
|
||||||
for (const dim of groupBy) {
|
for (const dim of groupBy) {
|
||||||
if (typeof dim === 'string') {
|
if (typeof dim === 'string') {
|
||||||
const val = resolveEntityField(e, dim)
|
const val = readAddressed(e, dim)
|
||||||
const v = val !== undefined && val !== null ? String(val) : '__null__'
|
const v = val !== undefined && val !== null ? String(val) : '__null__'
|
||||||
for (const k of keys) k[dim] = v
|
for (const k of keys) k[dim] = v
|
||||||
} else if ('unnest' in dim) {
|
} else if ('unnest' in dim) {
|
||||||
const val = resolveEntityField(e, dim.field)
|
const val = readAddressed(e, dim.field)
|
||||||
const raw = Array.isArray(val) ? val : val !== undefined && val !== null ? [val] : []
|
const raw = Array.isArray(val) ? val : val !== undefined && val !== null ? [val] : []
|
||||||
// Distinct elements: an entity with duplicate tags counts once per distinct tag.
|
// Distinct elements: an entity with duplicate tags counts once per distinct tag.
|
||||||
const elems = Array.from(new Set(raw.map(x => String(x))))
|
const elems = Array.from(new Set(raw.map(x => String(x))))
|
||||||
|
|
@ -145,7 +160,7 @@ function computeGroupKeys(
|
||||||
keys = next
|
keys = next
|
||||||
} else {
|
} else {
|
||||||
// Time-windowed field
|
// Time-windowed field
|
||||||
const val = resolveEntityField(e, dim.field)
|
const val = readAddressed(e, dim.field)
|
||||||
const v = typeof val === 'number' ? bucketTimestamp(val, dim.window) : '__null__'
|
const v = typeof val === 'number' ? bucketTimestamp(val, dim.window) : '__null__'
|
||||||
for (const k of keys) k[dim.field] = v
|
for (const k of keys) k[dim.field] = v
|
||||||
}
|
}
|
||||||
|
|
@ -174,7 +189,7 @@ function computeGroupKey(
|
||||||
* in metadata are both handled in one place.
|
* in metadata are both handled in one place.
|
||||||
*/
|
*/
|
||||||
function getNumericField(entity: Record<string, unknown>, field: string): number | undefined {
|
function getNumericField(entity: Record<string, unknown>, field: string): number | undefined {
|
||||||
const val = resolveEntityField(entity as unknown as HNSWNounWithMetadata, field)
|
const val = readAddressed(entity as unknown as HNSWNounWithMetadata, field)
|
||||||
if (typeof val === 'number' && !isNaN(val)) return val
|
if (typeof val === 'number' && !isNaN(val)) return val
|
||||||
if (typeof val === 'string') {
|
if (typeof val === 'string') {
|
||||||
const num = parseFloat(val)
|
const num = parseFloat(val)
|
||||||
|
|
@ -990,7 +1005,7 @@ export class AggregationIndex {
|
||||||
// distinctCount tracks distinct values of ANY type (strings, numbers, booleans),
|
// distinctCount tracks distinct values of ANY type (strings, numbers, booleans),
|
||||||
// keyed by their string form — NOT numeric-coerced, since its primary use is
|
// keyed by their string form — NOT numeric-coerced, since its primary use is
|
||||||
// categorical (distinct categories / users / tags), not numeric columns.
|
// categorical (distinct categories / users / tags), not numeric columns.
|
||||||
const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!)
|
const raw = readAddressed(entity as unknown as HNSWNounWithMetadata, metricDef.field!)
|
||||||
if (raw !== undefined && raw !== null) {
|
if (raw !== undefined && raw !== null) {
|
||||||
if (!state.valueCounts) state.valueCounts = {}
|
if (!state.valueCounts) state.valueCounts = {}
|
||||||
const key = String(raw)
|
const key = String(raw)
|
||||||
|
|
@ -1034,7 +1049,7 @@ export class AggregationIndex {
|
||||||
state.count = Math.max(0, state.count - 1)
|
state.count = Math.max(0, state.count - 1)
|
||||||
state.sum = Math.max(0, state.sum - 1)
|
state.sum = Math.max(0, state.sum - 1)
|
||||||
} else if (metricDef.op === 'distinctCount') {
|
} else if (metricDef.op === 'distinctCount') {
|
||||||
const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!)
|
const raw = readAddressed(entity as unknown as HNSWNounWithMetadata, metricDef.field!)
|
||||||
if (raw !== undefined && raw !== null && state.valueCounts) {
|
if (raw !== undefined && raw !== null && state.valueCounts) {
|
||||||
const key = String(raw)
|
const key = String(raw)
|
||||||
const c = state.valueCounts[key]
|
const c = state.valueCounts[key]
|
||||||
|
|
|
||||||
|
|
@ -5619,7 +5619,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
this._aggregationIndex!.defineAggregate({
|
this._aggregationIndex!.defineAggregate({
|
||||||
name: aggregateName,
|
name: aggregateName,
|
||||||
source: {},
|
source: {},
|
||||||
groupBy: perType ? [name, 'noun'] : [name],
|
groupBy: perType ? [name, 'system.type'] : [name],
|
||||||
metrics: { count: { op: 'count' } }
|
metrics: { count: { op: 'count' } }
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -5630,7 +5630,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* and `counts.byField()` agree on the convention.
|
* and `counts.byField()` agree on the convention.
|
||||||
*/
|
*/
|
||||||
private fieldCountsAggregateName(name: string): string {
|
private fieldCountsAggregateName(name: string): string {
|
||||||
return `__fieldCounts__${name}`
|
// v2 suffix: the per-type dimension moved from the legacy 'noun' alias to
|
||||||
|
// 'system.type' under the addressing law — a NEW name makes the ensure
|
||||||
|
// block re-define and BACKFILL from canonical instead of silently serving
|
||||||
|
// the old-dim definition (whose 'noun' key now reads user metadata and
|
||||||
|
// would drift). The v1 rows are derived state, superseded not lost.
|
||||||
|
return `__fieldCounts_v2__${name}`
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -11852,7 +11857,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// don't have the tracked field at all (e.g. the VFS root) bucket under
|
// don't have the tracked field at all (e.g. the VFS root) bucket under
|
||||||
// '__null__' and would otherwise pollute the count map.
|
// '__null__' and would otherwise pollute the count map.
|
||||||
if (value === undefined || value === null || value === '__null__') continue
|
if (value === undefined || value === null || value === '__null__') continue
|
||||||
if (options?.type !== undefined && row.groupKey?.['noun'] !== options.type) continue
|
if (options?.type !== undefined && row.groupKey?.['system.type'] !== options.type) continue
|
||||||
const key = String(value)
|
const key = String(value)
|
||||||
result[key] = (result[key] || 0) + (typeof row.metrics?.count === 'number' ? row.metrics.count : row.count)
|
result[key] = (result[key] || 0) + (typeof row.metrics?.count === 'number' ? row.metrics.count : row.count)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -283,3 +283,11 @@ export class UnsupportedFindOptionError extends Error {
|
||||||
this.option = option
|
this.option = option
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The capability signal both engines' conformance suites arm on
|
||||||
|
* (never a version guess): its presence at the package root means the one
|
||||||
|
* field-addressing law is LIVE on every query surface — bare = user metadata,
|
||||||
|
* `system.*` = the ruled scalars, plumbing invisible, refusals typed.
|
||||||
|
*/
|
||||||
|
export const FIELD_ADDRESSING_CAPABILITY = 'field-addressing/v1'
|
||||||
|
|
|
||||||
19
src/index.ts
19
src/index.ts
|
|
@ -106,6 +106,25 @@ export type {
|
||||||
// Export Aggregation Engine
|
// Export Aggregation Engine
|
||||||
export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js'
|
export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js'
|
||||||
|
|
||||||
|
// THE ONE FIELD-ADDRESSING LAW (sealed 2026-08-03) — the arming surface both
|
||||||
|
// engines' conformance suites detect: bare names = user metadata, system.* =
|
||||||
|
// the ten ruled scalars, plumbing invisible, refusals typed with the fix in
|
||||||
|
// the message. See docs/concepts/field-addressing.md.
|
||||||
|
export {
|
||||||
|
FIELD_ADDRESSING_CAPABILITY,
|
||||||
|
SYSTEM_ENTITY_SCALARS,
|
||||||
|
SYSTEM_RELATION_SCALARS,
|
||||||
|
PLUMBING_FIELDS,
|
||||||
|
parseFieldAddress,
|
||||||
|
readEntityFieldAddress,
|
||||||
|
readRelationFieldAddress,
|
||||||
|
buildUnresolvableMessage,
|
||||||
|
InvalidFieldAddressError,
|
||||||
|
UnresolvableFieldError,
|
||||||
|
UnsupportedFindOptionError
|
||||||
|
} from './db/fieldAddressing.js'
|
||||||
|
export type { FieldAddress, FieldAddressKind } from './db/fieldAddressing.js'
|
||||||
|
|
||||||
// Export Neural Import (AI data understanding)
|
// Export Neural Import (AI data understanding)
|
||||||
export { NeuralImport } from './neural/neuralImport.js'
|
export { NeuralImport } from './neural/neuralImport.js'
|
||||||
export type {
|
export type {
|
||||||
|
|
|
||||||
|
|
@ -69,12 +69,15 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json'
|
||||||
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
|
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
|
||||||
* absent marker — triggers a full derived-index rebuild on open.
|
* absent marker — triggers a full derived-index rebuild on open.
|
||||||
*/
|
*/
|
||||||
// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release):
|
// Epoch 3 (2026-08-03, the namespace-law pair): the index key format split
|
||||||
// user metadata fields named `level` become indexable on both engines — the
|
// the two namespaces — user fields keep bare flattened keys, the ten system
|
||||||
// derived posting set changed, so every pre-fix brain must rebuild its
|
// scalars moved to literal 'system.<field>' keys (the legacy 'noun' column
|
||||||
// metadata index from canonical at first open (poisoned multi-valued `level`
|
// spelling died with them). Every brain rebuilds its derived indexes from
|
||||||
// columns heal through this rebuild; no bespoke heal path).
|
// canonical at first open onto the frozen keys.
|
||||||
export const EXPECTED_INDEX_EPOCH = 2
|
// Epoch 2 (2026-08-03, same day, the interim pair): user metadata fields
|
||||||
|
// named `level` became indexable on both engines; poisoned multi-valued
|
||||||
|
// `level` columns healed through the rebuild.
|
||||||
|
export const EXPECTED_INDEX_EPOCH = 3
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description The data-layer format string this build writes and runs as.
|
* @description The data-layer format string this build writes and runs as.
|
||||||
|
|
|
||||||
|
|
@ -141,8 +141,8 @@ describe('level field shadow — user metadata named level is a real field', ()
|
||||||
expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384)
|
expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('this build runs index epoch 2 (the paired level-indexability rebuild)', () => {
|
it('this build runs index epoch 3 (the namespace-law key split rebuild)', () => {
|
||||||
expect(EXPECTED_INDEX_EPOCH).toBe(2)
|
expect(EXPECTED_INDEX_EPOCH).toBe(3)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -245,9 +245,10 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b
|
||||||
it('the brain-format marker module exports the compiled epoch + data-format constants', () => {
|
it('the brain-format marker module exports the compiled epoch + data-format constants', () => {
|
||||||
// cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both
|
// cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both
|
||||||
// sides share ONE source of truth — no duplicated constant to drift.
|
// sides share ONE source of truth — no duplicated constant to drift.
|
||||||
// Epoch 2: user metadata named `level` became indexable (the reserved-name
|
// Epoch 3: the namespace-law key split (bare user keys · literal
|
||||||
// shadow fix, 2026-08-03) — pre-fix brains rebuild derived indexes at open.
|
// 'system.<field>' scalars, 2026-08-03) — every brain rebuilds onto the
|
||||||
expect(EXPECTED_INDEX_EPOCH).toBe(2)
|
// frozen keys at first open. (Epoch 2 same day: `level` indexability.)
|
||||||
|
expect(EXPECTED_INDEX_EPOCH).toBe(3)
|
||||||
expect(CURRENT_DATA_FORMAT).toBe('8.0')
|
expect(CURRENT_DATA_FORMAT).toBe('8.0')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue