fix(vfs): a path-scoped search is a served range over the path, not a refused prefix match
Some checks failed
Delta Gate / Delta gate — candidate vs control (push) Waiting to run
CI / Node 22 (push) Failing after 1m29s
CI / Bun (latest) (push) Has been cancelled
CI / Node 24 (push) Failing after 1m29s
CI / Integration + conformance (Node 22) (push) Failing after 1m30s
Some checks failed
Delta Gate / Delta gate — candidate vs control (push) Waiting to run
CI / Node 22 (push) Failing after 1m29s
CI / Bun (latest) (push) Has been cancelled
CI / Node 24 (push) Failing after 1m29s
CI / Integration + conformance (Node 22) (push) Failing after 1m30s
`vfs.search({ path })` built its scope as `path: { $startsWith: path }`.
`$startsWith` is not in the filter vocabulary at all, and the `$`-less spelling
is REFUSED by the metadata index's served-operator law — an equality/range
posting index cannot evaluate a substring without reading every row, so it
refuses rather than answering an empty page. Every path-scoped VFS search threw
on this engine line; the two pins in tests/vfs/vfs.unit.test.ts that exercise it
have been red since the operator law landed.
The scope is now a half-open range over `metadata.path`:
`[dir + '/', dir + '0')`. Every descendant path begins with `dir + '/'`, and '0'
is the code point directly after '/', so membership in the range is EXACTLY
"carries that prefix" — and because the bounds differ at one ASCII position the
answer is identical under code-unit and code-point collation. Siblings fall out
correctly for the same reason: `/scope-sibling/x` sorts below the lower bound
and `/scope0` sits at the open upper bound. `recursive: false` narrows to the
directory's own identity instead — `parent`, an indexed equality. The root
adds no clause, because every VFS entity is under it.
`path` is the VFS's truth (write and rename maintain it; the `Contains` edges
are a projection of it), it is already indexed on every VFS entity, and
`explain()` reports the range as `column-store` — "O(log n) binary search +
roaring bitmap". So the scope narrows the search before it runs: no tree walk,
no migration, no backfill, and nothing fetched that the scope then discards.
Two other shapes were considered and rejected. A graph-scoped walk over
`Contains` reads the projection rather than the truth and costs O(subtree)
adjacency lookups per search, with the subtree's height as an unknown `depth`.
An indexed `ancestors: string[]` field cannot be implemented honestly today:
the index extractor skips arrays longer than ten elements, so a path more than
ten levels deep would silently drop out of every scoped search — and it needs
a backfill besides.
Pinned in tests/vfs/vfs-search-path-scope.test.ts (all eight red before this
change): descendants at three depths and never a sibling, including the
`/scope-sibling` and `/scope0` prefix traps; a trailing or doubled slash names
the same scope; the root scope equals the unscoped search; `recursive: false`
is the immediate children and refuses a missing directory by name; every
operator the search emits is ANSWERED by the index's own door rather than
refused; the id universe the index resolves for the search is already the
scope; and the range agrees with walking the tree.
This commit is contained in:
parent
dee46b35c8
commit
65493ba2de
2 changed files with 233 additions and 5 deletions
|
|
@ -1572,7 +1572,19 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
// ============= Semantic Operations =============
|
||||
|
||||
/**
|
||||
* Search files with natural language
|
||||
* Search files with natural language.
|
||||
*
|
||||
* `options.path` scopes the search to a directory: its whole subtree by
|
||||
* default, its immediate children when `recursive` is `false`. Both scopes
|
||||
* are metadata filters the index SERVES, so the scope narrows the search
|
||||
* before it runs — no tree walk, and never an over-fetch filtered afterwards.
|
||||
*
|
||||
* @param query - The natural-language query.
|
||||
* @param options - Scope, metadata filters and paging (see {@link SearchOptions}).
|
||||
* @returns The matching files, best first.
|
||||
* @throws {VFSError} ENOENT when `recursive: false` names a path that does
|
||||
* not exist (the non-recursive scope is the directory's own identity, so
|
||||
* the directory has to be there).
|
||||
*/
|
||||
async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -1588,11 +1600,26 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
}
|
||||
}
|
||||
|
||||
// Add path filter if specified
|
||||
// Scope to a directory, if asked. This used to emit
|
||||
// `path: { $startsWith }` — an operator that is not in the filter
|
||||
// vocabulary at all, and whose `$`-less spelling the metadata index
|
||||
// REFUSES by the served-operator law (an equality/range posting index
|
||||
// cannot evaluate a substring without reading every row). Every
|
||||
// path-scoped VFS search therefore threw, and none has ever worked on
|
||||
// this engine line. Both scopes below are served shapes.
|
||||
if (options?.path) {
|
||||
params.where = {
|
||||
...params.where,
|
||||
path: { $startsWith: options.path }
|
||||
if (options.recursive === false) {
|
||||
// Immediate children only: the directory's identity IS the scope, and
|
||||
// `parent` is an indexed equality on every VFS entity.
|
||||
params.where = {
|
||||
...params.where,
|
||||
parent: await this.pathResolver.resolve(options.path)
|
||||
}
|
||||
} else {
|
||||
const scope = this.descendantPathScope(options.path)
|
||||
if (scope) {
|
||||
params.where = { ...params.where, path: scope }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1754,6 +1781,42 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
return entity as VFSEntity
|
||||
}
|
||||
|
||||
/**
|
||||
* The SERVED metadata shape for "everything under this directory".
|
||||
*
|
||||
* `metadata.path` is the VFS's truth — write and rename maintain it, and the
|
||||
* `Contains` edges are a projection of it (see {@link repairContainment}) —
|
||||
* it is indexed on every VFS entity, and the metadata index serves ordered
|
||||
* range operators. So a subtree scope is a half-open range over the path
|
||||
* column: O(log n + matches), no tree walk, and nothing fetched that the
|
||||
* scope then discards.
|
||||
*
|
||||
* The range is `[dir + '/', dir + <successor of '/'>)`. Every descendant path
|
||||
* begins with `dir + '/'`, and '0' is the code point directly after '/', so a
|
||||
* string lies in the range EXACTLY when it carries that prefix. The two
|
||||
* bounds differ at a single ASCII position, so the answer is the same under
|
||||
* code-unit and code-point collation alike — no dependence on how the store
|
||||
* orders the rest of the string.
|
||||
*
|
||||
* Sibling exclusion falls out of the same fact and is worth stating, because
|
||||
* it is where a naive prefix test goes wrong: for `dir = '/scope'`,
|
||||
* `/scope-sibling/x` sorts BELOW the lower bound ('-' precedes '/') and
|
||||
* `/scope0` sits at the open upper bound — both outside, while
|
||||
* `/scope/sub/deep/c.txt` is inside at any depth.
|
||||
*
|
||||
* @param path - The directory to scope to.
|
||||
* @returns The `where` fragment for the `path` field, or `null` for the root
|
||||
* — every VFS entity is under it, so no clause narrows the search.
|
||||
*/
|
||||
private descendantPathScope(path: string): { gte: string; lt: string } | null {
|
||||
const dir = path.replace(/\/+/g, '/').replace(/\/$/, '') || '/'
|
||||
if (dir === '/') return null
|
||||
// Computed, so the bound carries its own reason: the first string that can
|
||||
// no longer share the `dir + '/'` prefix.
|
||||
const separatorSuccessor = String.fromCharCode('/'.charCodeAt(0) + 1)
|
||||
return { gte: `${dir}/`, lt: `${dir}${separatorSuccessor}` }
|
||||
}
|
||||
|
||||
private getParentPath(path: string): string {
|
||||
const normalized = path.replace(/\/+/g, '/').replace(/\/$/, '')
|
||||
const lastSlash = normalized.lastIndexOf('/')
|
||||
|
|
|
|||
165
tests/vfs/vfs-search-path-scope.test.ts
Normal file
165
tests/vfs/vfs-search-path-scope.test.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
/**
|
||||
* @module tests/vfs/vfs-search-path-scope
|
||||
* @description `vfs.search({ path })` scopes with a SERVED filter.
|
||||
*
|
||||
* The scope used to be emitted as `path: { $startsWith }` — an operator that is
|
||||
* not in the filter vocabulary at all, and whose `$`-less spelling the metadata
|
||||
* index refuses by the served-operator law (an equality/range posting index
|
||||
* cannot evaluate a substring without reading every row). Every path-scoped VFS
|
||||
* search threw; none has ever worked on this engine line.
|
||||
*
|
||||
* The scope is now a half-open range over `metadata.path`, which is the VFS's
|
||||
* truth, is indexed on every VFS entity, and is served by the ordered range
|
||||
* operators: `[dir + '/', dir + '0')` — '0' being the code point after '/', so
|
||||
* membership in the range is EXACTLY "carries the prefix `dir/`". The
|
||||
* non-recursive scope is the directory's own identity, `parent`, an equality.
|
||||
*
|
||||
* These pins hold the answer (descendants at every depth, siblings never — the
|
||||
* `/scope-sibling` trap included), the shape (the operators the search emits
|
||||
* are answered by the index's own door, never refused), and the law that the
|
||||
* scope narrows the search BEFORE it runs rather than filtering an over-fetch.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { VFSErrorCode } from '../../src/vfs/types.js'
|
||||
|
||||
/** A word every fixture file carries, so the text leg reaches all of them. */
|
||||
const TOKEN = 'quasar'
|
||||
|
||||
describe('vfs.search({ path }) scopes with a served filter', () => {
|
||||
let brain: Brainy
|
||||
let vfs: VirtualFileSystem
|
||||
|
||||
/** In scope for '/scope', at three depths. */
|
||||
const inScope = ['/scope/a.txt', '/scope/sub/b.txt', '/scope/sub/deep/c.txt']
|
||||
/** Out of scope — including the two prefix traps a naive test misses. */
|
||||
const outOfScope = ['/scope-sibling/d.txt', '/scope0/e.txt', '/elsewhere/f.txt', '/g.txt']
|
||||
|
||||
beforeAll(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true })
|
||||
await brain.init()
|
||||
vfs = brain.vfs
|
||||
await vfs.init()
|
||||
|
||||
await vfs.mkdir('/scope/sub/deep', { recursive: true })
|
||||
await vfs.mkdir('/scope-sibling', { recursive: true })
|
||||
await vfs.mkdir('/scope0', { recursive: true })
|
||||
await vfs.mkdir('/elsewhere', { recursive: true })
|
||||
|
||||
for (const path of [...inScope, ...outOfScope]) {
|
||||
await vfs.writeFile(path, `${TOKEN} content for ${path}`)
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await vfs?.close()
|
||||
await brain?.close()
|
||||
})
|
||||
|
||||
it('includes every descendant depth and excludes every sibling', async () => {
|
||||
const results = await vfs.search(TOKEN, { path: '/scope', limit: 50 })
|
||||
const paths = results.map((r) => r.path).sort()
|
||||
|
||||
expect(paths).toEqual([...inScope].sort())
|
||||
for (const path of outOfScope) expect(paths).not.toContain(path)
|
||||
})
|
||||
|
||||
it('a trailing slash and a doubled slash name the same scope', async () => {
|
||||
const plain = await vfs.search(TOKEN, { path: '/scope', limit: 50 })
|
||||
const trailing = await vfs.search(TOKEN, { path: '/scope/', limit: 50 })
|
||||
const doubled = await vfs.search(TOKEN, { path: '//scope//', limit: 50 })
|
||||
|
||||
const ids = (rs: Array<{ entityId: string }>) => rs.map((r) => r.entityId).sort()
|
||||
expect(ids(trailing)).toEqual(ids(plain))
|
||||
expect(ids(doubled)).toEqual(ids(plain))
|
||||
})
|
||||
|
||||
it('the root scope is every VFS file — it adds no clause to narrow with', async () => {
|
||||
const rooted = await vfs.search(TOKEN, { path: '/', limit: 50 })
|
||||
const unscoped = await vfs.search(TOKEN, { limit: 50 })
|
||||
|
||||
const paths = rooted.map((r) => r.path).sort()
|
||||
expect(paths).toEqual([...inScope, ...outOfScope].sort())
|
||||
expect(paths).toEqual(unscoped.map((r) => r.path).sort())
|
||||
})
|
||||
|
||||
it('recursive: false is the immediate children, not the subtree', async () => {
|
||||
const results = await vfs.search(TOKEN, { path: '/scope', recursive: false, limit: 50 })
|
||||
expect(results.map((r) => r.path)).toEqual(['/scope/a.txt'])
|
||||
})
|
||||
|
||||
it('recursive: false on a path that does not exist refuses by name', async () => {
|
||||
await expect(
|
||||
vfs.search(TOKEN, { path: '/no-such-dir', recursive: false, limit: 50 })
|
||||
).rejects.toMatchObject({ code: VFSErrorCode.ENOENT })
|
||||
})
|
||||
|
||||
it('every operator the search emits is ANSWERED by the index door, never refused', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const emitted: any[] = []
|
||||
const find = vi.spyOn(brain as any, 'find')
|
||||
try {
|
||||
await vfs.search(TOKEN, { path: '/scope', limit: 50 })
|
||||
await vfs.search(TOKEN, { path: '/scope/sub', where: { mimeType: 'text/plain' }, limit: 50 })
|
||||
await vfs.search(TOKEN, { path: '/scope', recursive: false, limit: 50 })
|
||||
await vfs.search(TOKEN, { path: '/', limit: 50 })
|
||||
for (const call of find.mock.calls) emitted.push((call[0] as any).where)
|
||||
} finally {
|
||||
find.mockRestore()
|
||||
}
|
||||
|
||||
expect(emitted).toHaveLength(4)
|
||||
for (const where of emitted) {
|
||||
// The door itself is the judge: an operator outside the served set is
|
||||
// REFUSED here (BrainyError INVALID_QUERY), never answered.
|
||||
await expect(index.getIdsForFilter(where)).resolves.toBeInstanceOf(Array)
|
||||
}
|
||||
|
||||
// And the scope really is a range on the path — the shape this fix chose.
|
||||
expect(emitted[0].path).toEqual({ gte: '/scope/', lt: '/scope0' })
|
||||
expect(emitted[3].path).toBeUndefined()
|
||||
})
|
||||
|
||||
it('the scope narrows the search before it runs — no over-fetch to filter', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const filter = vi.spyOn(index, 'getIdsForFilter')
|
||||
let universe: string[] = []
|
||||
try {
|
||||
await vfs.search(TOKEN, { path: '/scope', limit: 50 })
|
||||
// The search's own call — the one carrying the scope. (Path resolution
|
||||
// asks this same door for the root, before the search is built.)
|
||||
const scoped = filter.mock.calls.findIndex(
|
||||
(c) => (c[0] as any)?.path?.gte === '/scope/'
|
||||
)
|
||||
expect(scoped).toBeGreaterThanOrEqual(0)
|
||||
universe = (await filter.mock.results[scoped].value) as string[]
|
||||
} finally {
|
||||
filter.mockRestore()
|
||||
}
|
||||
|
||||
// The id universe the index resolved for the search is already the scope:
|
||||
// three files, and not one row from outside it.
|
||||
const rows = await brain.batchGet(universe)
|
||||
const paths = [...rows.values()].map((e: any) => e.metadata.path).sort()
|
||||
expect(paths).toEqual([...inScope].sort())
|
||||
})
|
||||
|
||||
it('the range answers the same ids as walking the tree', async () => {
|
||||
// The path is the truth and the Contains edges are its projection; a scope
|
||||
// read from the truth must agree with one walked over the projection.
|
||||
const walked: string[] = []
|
||||
const walk = async (dir: string): Promise<void> => {
|
||||
for (const name of await vfs.readdir(dir)) {
|
||||
const child = dir === '/' ? `/${name}` : `${dir}/${name}`
|
||||
const stat = await vfs.stat(child)
|
||||
if (stat.isDirectory()) await walk(child)
|
||||
else walked.push(child)
|
||||
}
|
||||
}
|
||||
await walk('/scope')
|
||||
|
||||
const searched = await vfs.search(TOKEN, { path: '/scope', limit: 50 })
|
||||
expect(searched.map((r) => r.path).sort()).toEqual(walked.sort())
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue