**feat(core, storage, tests): enhance verb construction with timestamps and metadata**

- **Core**: Improved verb creation logic by adding `createdAt`, `updatedAt`, and `createdBy` attributes. These fields include timestamped metadata (`seconds`, `nanoseconds`) and source augmentation/service information for better tracking.
- **Storage**: Refactored `BaseStorage` methods to utilize internal variants (e.g., `saveVerb_internal`, `getNoun_internal`). Added support for new verb attributes while maintaining backward compatibility with existing data structures.
- **Tests**:
  - Updated `s3-storage.test.ts` and `opfs-storage.test.ts` to validate changes in verb attributes such as timestamps and augmentation metadata.
  - Added assertions for `createdAt`, `updatedAt`, and `createdBy` fields in test cases.
- **Cleanup**: Replaced ambiguous type aliases like `Edge` and `HNSWNode` with clearer equivalents (`Verb` and `HNSWNoun_internal`) for consistency across storage adapters.

**Purpose**: Enhance metadata tracking and standardize attribute handling across storage and core modules to ensure accurate and consistent data throughout the system.
This commit is contained in:
David Snelling 2025-07-25 09:44:10 -07:00
parent ae3560aad3
commit 501244398e
10 changed files with 641 additions and 310 deletions

View file

@ -1656,16 +1656,34 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
} }
} }
// Get service name from options or current augmentation
const service = options.service || this.getCurrentAugmentation()
// Create timestamp for creation/update time
const now = new Date()
const timestamp = {
seconds: Math.floor(now.getTime() / 1000),
nanoseconds: (now.getTime() % 1000) * 1000000
}
// Create verb // Create verb
const verb: GraphVerb = { const verb: GraphVerb = {
id, id,
vector: verbVector, vector: verbVector,
connections: new Map(), connections: new Map(),
sourceId, sourceId: sourceId,
targetId, targetId: targetId,
type: verbType, source: sourceId,
target: targetId,
verb: verbType as VerbType,
weight: options.weight, weight: options.weight,
metadata: options.metadata metadata: options.metadata,
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
augmentation: service,
version: '1.0' // TODO: Get actual version from augmentation
}
} }
// Add to index // Add to index
@ -1687,8 +1705,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
await this.storage!.saveVerb(verb) await this.storage!.saveVerb(verb)
// Track verb statistics // Track verb statistics
const service = options.service || 'default' const serviceForStats = options.service || 'default'
await this.storage!.incrementStatistic('verb', service) await this.storage!.incrementStatistic('verb', serviceForStats)
// Update HNSW index size (excluding verbs) // Update HNSW index size (excluding verbs)
await this.storage!.updateHnswIndexSize(await this.getNounCount()) await this.storage!.updateHnswIndexSize(await this.getNounCount())

View file

@ -82,6 +82,11 @@ export interface GraphVerb extends HNSWNoun {
verb?: string // Alias for type verb?: string // Alias for type
data?: Record<string, any> // Additional flexible data storage data?: Record<string, any> // Additional flexible data storage
embedding?: Vector // Vector representation of the relationship embedding?: Vector // Vector representation of the relationship
// Timestamp and creator properties
createdAt?: { seconds: number, nanoseconds: number } // When the verb was created
updatedAt?: { seconds: number, nanoseconds: number } // When the verb was last updated
createdBy?: { augmentation: string, version: string } // Information about what created this verb
} }
/** /**

View file

@ -7,8 +7,8 @@ import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js' import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY } from '../baseStorage.js'
// Type aliases for better readability // Type aliases for better readability
type HNSWNode = HNSWNoun type HNSWNoun_internal = HNSWNoun
type Edge = GraphVerb type Verb = GraphVerb
// Node.js modules - dynamically imported to avoid issues in browser environments // Node.js modules - dynamically imported to avoid issues in browser environments
let fs: any let fs: any
@ -110,77 +110,77 @@ export class FileSystemStorage extends BaseStorage {
} }
/** /**
* Save a node to storage * Save a noun to storage
*/ */
protected async saveNode(node: HNSWNode): Promise<void> { protected async saveNoun_internal(noun: HNSWNoun_internal): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
// Convert connections Map to a serializable format // Convert connections Map to a serializable format
const serializableNode = { const serializableNoun = {
...node, ...noun,
connections: this.mapToObject(node.connections, (set) => connections: this.mapToObject(noun.connections, (set) =>
Array.from(set as Set<string>) Array.from(set as Set<string>)
) )
} }
const filePath = path.join(this.nounsDir, `${node.id}.json`) const filePath = path.join(this.nounsDir, `${noun.id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2)) await fs.promises.writeFile(filePath, JSON.stringify(serializableNoun, null, 2))
} }
/** /**
* Get a node from storage * Get a noun from storage
*/ */
protected async getNode(id: string): Promise<HNSWNode | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun_internal | null> {
await this.ensureInitialized() await this.ensureInitialized()
const filePath = path.join(this.nounsDir, `${id}.json`) const filePath = path.join(this.nounsDir, `${id}.json`)
try { try {
const data = await fs.promises.readFile(filePath, 'utf-8') const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedNode = JSON.parse(data) const parsedNoun = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { for (const [level, nounIds] of Object.entries(parsedNoun.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nounIds as string[]))
} }
return { return {
id: parsedNode.id, id: parsedNoun.id,
vector: parsedNode.vector, vector: parsedNoun.vector,
connections connections
} }
} catch (error: any) { } catch (error: any) {
if (error.code !== 'ENOENT') { if (error.code !== 'ENOENT') {
console.error(`Error reading node ${id}:`, error) console.error(`Error reading noun ${id}:`, error)
} }
return null return null
} }
} }
/** /**
* Get all nodes from storage * Get all nouns from storage
*/ */
protected async getAllNodes(): Promise<HNSWNode[]> { protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
await this.ensureInitialized() await this.ensureInitialized()
const allNodes: HNSWNode[] = [] const allNouns: HNSWNoun_internal[] = []
try { try {
const files = await fs.promises.readdir(this.nounsDir) const files = await fs.promises.readdir(this.nounsDir)
for (const file of files) { for (const file of files) {
if (file.endsWith('.json')) { if (file.endsWith('.json')) {
const filePath = path.join(this.nounsDir, file) const filePath = path.join(this.nounsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8') const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedNode = JSON.parse(data) const parsedNoun = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { for (const [level, nounIds] of Object.entries(parsedNoun.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nounIds as string[]))
} }
allNodes.push({ allNouns.push({
id: parsedNode.id, id: parsedNoun.id,
vector: parsedNode.vector, vector: parsedNoun.vector,
connections connections
}) })
} }
@ -190,39 +190,39 @@ export class FileSystemStorage extends BaseStorage {
console.error(`Error reading directory ${this.nounsDir}:`, error) console.error(`Error reading directory ${this.nounsDir}:`, error)
} }
} }
return allNodes return allNouns
} }
/** /**
* Get nodes by noun type * Get nouns by noun type
* @param nounType The noun type to filter by * @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type * @returns Promise that resolves to an array of nouns of the specified noun type
*/ */
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> { protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun_internal[]> {
await this.ensureInitialized() await this.ensureInitialized()
const nouns: HNSWNode[] = [] const nouns: HNSWNoun_internal[] = []
try { try {
const files = await fs.promises.readdir(this.nounsDir) const files = await fs.promises.readdir(this.nounsDir)
for (const file of files) { for (const file of files) {
if (file.endsWith('.json')) { if (file.endsWith('.json')) {
const filePath = path.join(this.nounsDir, file) const filePath = path.join(this.nounsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8') const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedNode = JSON.parse(data) const parsedNoun = JSON.parse(data)
// Filter by noun type using metadata // Filter by noun type using metadata
const nodeId = parsedNode.id const nounId = parsedNoun.id
const metadata = await this.getMetadata(nodeId) const metadata = await this.getMetadata(nounId)
if (metadata && metadata.noun === nounType) { if (metadata && metadata.noun === nounType) {
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) { for (const [level, nounIds] of Object.entries(parsedNoun.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nounIds as string[]))
} }
nouns.push({ nouns.push({
id: parsedNode.id, id: parsedNoun.id,
vector: parsedNode.vector, vector: parsedNoun.vector,
connections connections
}) })
} }
@ -238,9 +238,9 @@ export class FileSystemStorage extends BaseStorage {
} }
/** /**
* Delete a node from storage * Delete a noun from storage
*/ */
protected async deleteNode(id: string): Promise<void> { protected async deleteNoun_internal(id: string): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
const filePath = path.join(this.nounsDir, `${id}.json`) const filePath = path.join(this.nounsDir, `${id}.json`)
@ -248,95 +248,112 @@ export class FileSystemStorage extends BaseStorage {
await fs.promises.unlink(filePath) await fs.promises.unlink(filePath)
} catch (error: any) { } catch (error: any) {
if (error.code !== 'ENOENT') { if (error.code !== 'ENOENT') {
console.error(`Error deleting node file ${filePath}:`, error) console.error(`Error deleting noun ${id}:`, error)
throw error throw error
} }
} }
} }
/** /**
* Save an edge to storage * Save a verb to storage
*/ */
protected async saveEdge(edge: Edge): Promise<void> { protected async saveVerb_internal(verb: Verb): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
// Convert connections Map to a serializable format // Convert connections Map to a serializable format
const serializableEdge = { const serializableVerb = {
...edge, ...verb,
connections: this.mapToObject(edge.connections, (set) => connections: this.mapToObject(verb.connections, (set) =>
Array.from(set as Set<string>) Array.from(set as Set<string>)
) )
} }
const filePath = path.join(this.verbsDir, `${edge.id}.json`) const filePath = path.join(this.verbsDir, `${verb.id}.json`)
await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2)) await fs.promises.writeFile(filePath, JSON.stringify(serializableVerb, null, 2))
} }
/** /**
* Get an edge from storage * Get a verb from storage
*/ */
protected async getEdge(id: string): Promise<Edge | null> { protected async getVerb_internal(id: string): Promise<Verb | null> {
await this.ensureInitialized() await this.ensureInitialized()
const filePath = path.join(this.verbsDir, `${id}.json`) const filePath = path.join(this.verbsDir, `${id}.json`)
try { try {
const data = await fs.promises.readFile(filePath, 'utf-8') const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedEdge = JSON.parse(data) const parsedVerb = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { for (const [level, nodeIds] of Object.entries(parsedVerb.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
return { return {
id: parsedEdge.id, id: parsedVerb.id,
vector: parsedEdge.vector, vector: parsedVerb.vector,
connections, connections,
sourceId: parsedEdge.sourceId, sourceId: parsedVerb.sourceId || parsedVerb.source,
targetId: parsedEdge.targetId, targetId: parsedVerb.targetId || parsedVerb.target,
type: parsedEdge.type, source: parsedVerb.sourceId || parsedVerb.source,
weight: parsedEdge.weight, target: parsedVerb.targetId || parsedVerb.target,
metadata: parsedEdge.metadata verb: parsedVerb.type || parsedVerb.verb,
weight: parsedVerb.weight,
metadata: parsedVerb.metadata,
createdAt: parsedVerb.createdAt || defaultTimestamp,
updatedAt: parsedVerb.updatedAt || defaultTimestamp,
createdBy: parsedVerb.createdBy || defaultCreatedBy
} }
} catch (error: any) { } catch (error: any) {
if (error.code !== 'ENOENT') { if (error.code !== 'ENOENT') {
console.error(`Error reading edge ${id}:`, error) console.error(`Error reading verb ${id}:`, error)
} }
return null return null
} }
} }
/** /**
* Get all edges from storage * Get all verbs from storage
*/ */
protected async getAllEdges(): Promise<Edge[]> { protected async getAllVerbs_internal(): Promise<Verb[]> {
await this.ensureInitialized() await this.ensureInitialized()
const allEdges: Edge[] = [] const allVerbs: Verb[] = []
try { try {
const files = await fs.promises.readdir(this.verbsDir) const files = await fs.promises.readdir(this.verbsDir)
for (const file of files) { for (const file of files) {
if (file.endsWith('.json')) { if (file.endsWith('.json')) {
const filePath = path.join(this.verbsDir, file) const filePath = path.join(this.verbsDir, file)
const data = await fs.promises.readFile(filePath, 'utf-8') const data = await fs.promises.readFile(filePath, 'utf-8')
const parsedEdge = JSON.parse(data) const parsedVerb = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) { for (const [level, nodeIds] of Object.entries(parsedVerb.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
allEdges.push({ allVerbs.push({
id: parsedEdge.id, id: parsedVerb.id,
vector: parsedEdge.vector, vector: parsedVerb.vector,
connections, connections,
sourceId: parsedEdge.sourceId, sourceId: parsedVerb.sourceId,
targetId: parsedEdge.targetId, targetId: parsedVerb.targetId,
type: parsedEdge.type, type: parsedVerb.type,
weight: parsedEdge.weight, weight: parsedVerb.weight,
metadata: parsedEdge.metadata metadata: parsedVerb.metadata
}) })
} }
} }
@ -345,37 +362,37 @@ export class FileSystemStorage extends BaseStorage {
console.error(`Error reading directory ${this.verbsDir}:`, error) console.error(`Error reading directory ${this.verbsDir}:`, error)
} }
} }
return allEdges return allVerbs
} }
/** /**
* Get edges by source * Get verbs by source
*/ */
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> { protected async getVerbsBySource_internal(sourceId: string): Promise<Verb[]> {
const edges = await this.getAllEdges() const verbs = await this.getAllVerbs_internal()
return edges.filter((edge) => edge.sourceId === sourceId) return verbs.filter((verb) => (verb.sourceId || verb.source) === sourceId)
} }
/** /**
* Get edges by target * Get verbs by target
*/ */
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> { protected async getVerbsByTarget_internal(targetId: string): Promise<Verb[]> {
const edges = await this.getAllEdges() const verbs = await this.getAllVerbs_internal()
return edges.filter((edge) => edge.targetId === targetId) return verbs.filter((verb) => (verb.targetId || verb.target) === targetId)
} }
/** /**
* Get edges by type * Get verbs by type
*/ */
protected async getEdgesByType(type: string): Promise<Edge[]> { protected async getVerbsByType_internal(type: string): Promise<Verb[]> {
const edges = await this.getAllEdges() const verbs = await this.getAllVerbs_internal()
return edges.filter((edge) => edge.type === type) return verbs.filter((verb) => (verb.type || verb.verb) === type)
} }
/** /**
* Delete an edge from storage * Delete a verb from storage
*/ */
protected async deleteEdge(id: string): Promise<void> { protected async deleteVerb_internal(id: string): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
const filePath = path.join(this.verbsDir, `${id}.json`) const filePath = path.join(this.verbsDir, `${id}.json`)
@ -383,7 +400,7 @@ export class FileSystemStorage extends BaseStorage {
await fs.promises.unlink(filePath) await fs.promises.unlink(filePath)
} catch (error: any) { } catch (error: any) {
if (error.code !== 'ENOENT') { if (error.code !== 'ENOENT') {
console.error(`Error deleting edge file ${filePath}:`, error) console.error(`Error deleting verb file ${filePath}:`, error)
throw error throw error
} }
} }

View file

@ -9,12 +9,12 @@ import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
/** /**
* Type alias for HNSWNoun to make the code more readable * Type alias for HNSWNoun to make the code more readable
*/ */
type HNSWNode = HNSWNoun type HNSWNoun_internal = HNSWNoun
/** /**
* Type alias for GraphVerb to make the code more readable * Type alias for GraphVerb to make the code more readable
*/ */
type Edge = GraphVerb type Verb = GraphVerb
/** /**
* In-memory storage adapter * In-memory storage adapter
@ -22,8 +22,8 @@ type Edge = GraphVerb
*/ */
export class MemoryStorage extends BaseStorage { export class MemoryStorage extends BaseStorage {
// Single map of noun ID to noun // Single map of noun ID to noun
private nouns: Map<string, HNSWNode> = new Map() private nouns: Map<string, HNSWNoun_internal> = new Map()
private verbs: Map<string, Edge> = new Map() private verbs: Map<string, Verb> = new Map()
private metadata: Map<string, any> = new Map() private metadata: Map<string, any> = new Map()
private statistics: StatisticsData | null = null private statistics: StatisticsData | null = null
@ -40,237 +40,270 @@ export class MemoryStorage extends BaseStorage {
} }
/** /**
* Save a node to storage * Save a noun to storage
*/ */
protected async saveNode(node: HNSWNode): Promise<void> { protected async saveNoun_internal(noun: HNSWNoun_internal): Promise<void> {
// Create a deep copy to avoid reference issues // Create a deep copy to avoid reference issues
const nodeCopy: HNSWNode = { const nounCopy: HNSWNoun_internal = {
id: node.id, id: noun.id,
vector: [...node.vector], vector: [...noun.vector],
connections: new Map() connections: new Map()
} }
// Copy connections // Copy connections
for (const [level, connections] of node.connections.entries()) { for (const [level, connections] of noun.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections)) nounCopy.connections.set(level, new Set(connections))
} }
// Save the node directly in the nouns map // Save the noun directly in the nouns map
this.nouns.set(node.id, nodeCopy) this.nouns.set(noun.id, nounCopy)
} }
/** /**
* Get a node from storage * Get a noun from storage
*/ */
protected async getNode(id: string): Promise<HNSWNode | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun_internal | null> {
// Get the node directly from the nouns map // Get the noun directly from the nouns map
const node = this.nouns.get(id) const noun = this.nouns.get(id)
// If not found, return null // If not found, return null
if (!node) { if (!noun) {
return null return null
} }
// Return a deep copy to avoid reference issues // Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = { const nounCopy: HNSWNoun_internal = {
id: node.id, id: noun.id,
vector: [...node.vector], vector: [...noun.vector],
connections: new Map() connections: new Map()
} }
// Copy connections // Copy connections
for (const [level, connections] of node.connections.entries()) { for (const [level, connections] of noun.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections)) nounCopy.connections.set(level, new Set(connections))
} }
return nodeCopy return nounCopy
} }
/** /**
* Get all nodes from storage * Get all nouns from storage
*/ */
protected async getAllNodes(): Promise<HNSWNode[]> { protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
const allNodes: HNSWNode[] = [] const allNouns: HNSWNoun_internal[] = []
// Iterate through all nodes in the nouns map // Iterate through all nouns in the nouns map
for (const [nodeId, node] of this.nouns.entries()) { for (const [nounId, noun] of this.nouns.entries()) {
// Return a deep copy to avoid reference issues // Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = { const nounCopy: HNSWNoun_internal = {
id: node.id, id: noun.id,
vector: [...node.vector], vector: [...noun.vector],
connections: new Map() connections: new Map()
} }
// Copy connections // Copy connections
for (const [level, connections] of node.connections.entries()) { for (const [level, connections] of noun.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections)) nounCopy.connections.set(level, new Set(connections))
} }
allNodes.push(nodeCopy) allNouns.push(nounCopy)
} }
return allNodes return allNouns
} }
/** /**
* Get nodes by noun type * Get nouns by noun type
* @param nounType The noun type to filter by * @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type * @returns Promise that resolves to an array of nouns of the specified noun type
*/ */
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> { protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun_internal[]> {
const nodes: HNSWNode[] = [] const nouns: HNSWNoun_internal[] = []
// Iterate through all nodes and filter by noun type using metadata // Iterate through all nouns and filter by noun type using metadata
for (const [nodeId, node] of this.nouns.entries()) { for (const [nounId, noun] of this.nouns.entries()) {
// Get the metadata to check the noun type // Get the metadata to check the noun type
const metadata = await this.getMetadata(nodeId) const metadata = await this.getMetadata(nounId)
// Include the node if its noun type matches the requested type // Include the noun if its noun type matches the requested type
if (metadata && metadata.noun === nounType) { if (metadata && metadata.noun === nounType) {
// Return a deep copy to avoid reference issues // Return a deep copy to avoid reference issues
const nodeCopy: HNSWNode = { const nounCopy: HNSWNoun_internal = {
id: node.id, id: noun.id,
vector: [...node.vector], vector: [...noun.vector],
connections: new Map() connections: new Map()
} }
// Copy connections // Copy connections
for (const [level, connections] of node.connections.entries()) { for (const [level, connections] of noun.connections.entries()) {
nodeCopy.connections.set(level, new Set(connections)) nounCopy.connections.set(level, new Set(connections))
} }
nodes.push(nodeCopy) nouns.push(nounCopy)
} }
} }
return nodes return nouns
} }
/** /**
* Delete a node from storage * Delete a noun from storage
*/ */
protected async deleteNode(id: string): Promise<void> { protected async deleteNoun_internal(id: string): Promise<void> {
// Delete the node directly from the nouns map
this.nouns.delete(id) this.nouns.delete(id)
} }
/** /**
* Save an edge to storage * Save a verb to storage
*/ */
protected async saveEdge(edge: Edge): Promise<void> { protected async saveVerb_internal(verb: Verb): Promise<void> {
// Create a deep copy to avoid reference issues // Create a deep copy to avoid reference issues
const edgeCopy: Edge = { const verbCopy: Verb = {
id: edge.id, id: verb.id,
vector: [...edge.vector], vector: [...verb.vector],
connections: new Map(), connections: new Map(),
sourceId: edge.sourceId, sourceId: verb.sourceId,
targetId: edge.targetId, targetId: verb.targetId,
type: edge.type, type: verb.type,
weight: edge.weight, weight: verb.weight,
metadata: edge.metadata metadata: verb.metadata
} }
// Copy connections // Copy connections
for (const [level, connections] of edge.connections.entries()) { for (const [level, connections] of verb.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections)) verbCopy.connections.set(level, new Set(connections))
} }
// Save the edge directly in the verbs map // Save the verb directly in the verbs map
this.verbs.set(edge.id, edgeCopy) this.verbs.set(verb.id, verbCopy)
} }
/** /**
* Get an edge from storage * Get a verb from storage
*/ */
protected async getEdge(id: string): Promise<Edge | null> { protected async getVerb_internal(id: string): Promise<Verb | null> {
// Get the edge directly from the verbs map // Get the verb directly from the verbs map
const edge = this.verbs.get(id) const verb = this.verbs.get(id)
// If not found, return null // If not found, return null
if (!edge) { if (!verb) {
return null return null
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
// Return a deep copy to avoid reference issues // Return a deep copy to avoid reference issues
const edgeCopy: Edge = { const verbCopy: Verb = {
id: edge.id, id: verb.id,
vector: [...edge.vector], vector: [...verb.vector],
connections: new Map(), connections: new Map(),
sourceId: edge.sourceId, sourceId: (verb.sourceId || verb.source || ""),
targetId: edge.targetId, targetId: (verb.targetId || verb.target || ""),
type: edge.type, source: (verb.sourceId || verb.source || ""),
weight: edge.weight, target: (verb.targetId || verb.target || ""),
metadata: edge.metadata verb: verb.type || verb.verb,
weight: verb.weight,
metadata: verb.metadata,
createdAt: verb.createdAt || defaultTimestamp,
updatedAt: verb.updatedAt || defaultTimestamp,
createdBy: verb.createdBy || defaultCreatedBy
} }
// Copy connections // Copy connections
for (const [level, connections] of edge.connections.entries()) { for (const [level, connections] of verb.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections)) verbCopy.connections.set(level, new Set(connections))
} }
return edgeCopy return verbCopy
} }
/** /**
* Get all edges from storage * Get all verbs from storage
*/ */
protected async getAllEdges(): Promise<Edge[]> { protected async getAllVerbs_internal(): Promise<Verb[]> {
const allEdges: Edge[] = [] const allVerbs: Verb[] = []
// Iterate through all verbs in the verbs map
for (const [verbId, verb] of this.verbs.entries()) {
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
// Iterate through all edges in the verbs map
for (const [edgeId, edge] of this.verbs.entries()) {
// Return a deep copy to avoid reference issues // Return a deep copy to avoid reference issues
const edgeCopy: Edge = { const verbCopy: Verb = {
id: edge.id, id: verb.id,
vector: [...edge.vector], vector: [...verb.vector],
connections: new Map(), connections: new Map(),
sourceId: edge.sourceId, sourceId: (verb.sourceId || verb.source || ""),
targetId: edge.targetId, targetId: (verb.targetId || verb.target || ""),
type: edge.type, source: (verb.sourceId || verb.source || ""),
weight: edge.weight, target: (verb.targetId || verb.target || ""),
metadata: edge.metadata verb: verb.type || verb.verb,
weight: verb.weight,
metadata: verb.metadata,
createdAt: verb.createdAt || defaultTimestamp,
updatedAt: verb.updatedAt || defaultTimestamp,
createdBy: verb.createdBy || defaultCreatedBy
} }
// Copy connections // Copy connections
for (const [level, connections] of edge.connections.entries()) { for (const [level, connections] of verb.connections.entries()) {
edgeCopy.connections.set(level, new Set(connections)) verbCopy.connections.set(level, new Set(connections))
} }
allEdges.push(edgeCopy) allVerbs.push(verbCopy)
} }
return allEdges return allVerbs
} }
/** /**
* Get edges by source * Get verbs by source
*/ */
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> { protected async getVerbsBySource_internal(sourceId: string): Promise<Verb[]> {
const edges = await this.getAllEdges() const allVerbs = await this.getAllVerbs_internal()
return edges.filter((edge) => edge.sourceId === sourceId) return allVerbs.filter((verb: Verb) => (verb.sourceId || verb.source) === sourceId)
} }
/** /**
* Get edges by target * Get verbs by target
*/ */
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> { protected async getVerbsByTarget_internal(targetId: string): Promise<Verb[]> {
const edges = await this.getAllEdges() const allVerbs = await this.getAllVerbs_internal()
return edges.filter((edge) => edge.targetId === targetId) return allVerbs.filter((verb: Verb) => (verb.targetId || verb.target) === targetId)
} }
/** /**
* Get edges by type * Get verbs by type
*/ */
protected async getEdgesByType(type: string): Promise<Edge[]> { protected async getVerbsByType_internal(type: string): Promise<Verb[]> {
const edges = await this.getAllEdges() const allVerbs = await this.getAllVerbs_internal()
return edges.filter((edge) => edge.type === type) return allVerbs.filter((verb: Verb) => (verb.type || verb.verb) === type)
} }
/** /**
* Delete an edge from storage * Delete a verb from storage
*/ */
protected async deleteEdge(id: string): Promise<void> { protected async deleteVerb_internal(id: string): Promise<void> {
// Delete the edge directly from the verbs map // Delete the verb directly from the verbs map
this.verbs.delete(id) this.verbs.delete(id)
} }

View file

@ -7,6 +7,14 @@ import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js' import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
import '../../types/fileSystemTypes.js' import '../../types/fileSystemTypes.js'
// Type alias for HNSWNode
type HNSWNode = HNSWNoun
/**
* Type alias for GraphVerb to make the code more readable
*/
type Edge = GraphVerb
/** /**
* Helper function to safely get a file from a FileSystemHandle * Helper function to safely get a file from a FileSystemHandle
* This is needed because TypeScript doesn't recognize that a FileSystemHandle * This is needed because TypeScript doesn't recognize that a FileSystemHandle
@ -18,8 +26,8 @@ async function safeGetFile(handle: FileSystemHandle): Promise<File> {
} }
// Type aliases for better readability // Type aliases for better readability
type HNSWNode = HNSWNoun type HNSWNoun_internal = HNSWNoun
type Edge = GraphVerb type Verb = GraphVerb
// Root directory name for OPFS storage // Root directory name for OPFS storage
const ROOT_DIR = 'opfs-vector-db' const ROOT_DIR = 'opfs-vector-db'
@ -149,54 +157,54 @@ export class OPFSStorage extends BaseStorage {
} }
/** /**
* Save a node to storage * Save a noun to storage
*/ */
protected async saveNode(node: HNSWNode): Promise<void> { protected async saveNoun_internal(noun: HNSWNoun_internal): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
try { try {
// Convert connections Map to a serializable format // Convert connections Map to a serializable format
const serializableNode = { const serializableNoun = {
...node, ...noun,
connections: this.mapToObject(node.connections, (set) => connections: this.mapToObject(noun.connections, (set) =>
Array.from(set as Set<string>) Array.from(set as Set<string>)
) )
} }
// Create or get the file for this noun // Create or get the file for this noun
const fileHandle = await this.nounsDir!.getFileHandle(node.id, { const fileHandle = await this.nounsDir!.getFileHandle(noun.id, {
create: true create: true
}) })
// Write the noun data to the file // Write the noun data to the file
const writable = await fileHandle.createWritable() const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableNode)) await writable.write(JSON.stringify(serializableNoun))
await writable.close() await writable.close()
} catch (error) { } catch (error) {
console.error(`Failed to save node ${node.id}:`, error) console.error(`Failed to save noun ${noun.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`) throw new Error(`Failed to save noun ${noun.id}: ${error}`)
} }
} }
/** /**
* Get a node from storage * Get a noun from storage
*/ */
protected async getNode(id: string): Promise<HNSWNode | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun_internal | null> {
await this.ensureInitialized() await this.ensureInitialized()
try { try {
// Get the file handle for this node // Get the file handle for this noun
const fileHandle = await this.nounsDir!.getFileHandle(id) const fileHandle = await this.nounsDir!.getFileHandle(id)
// Read the node data from the file // Read the noun data from the file
const file = await fileHandle.getFile() const file = await fileHandle.getFile()
const text = await file.text() const text = await file.text()
const data = JSON.parse(text) const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) { for (const [level, nounIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nounIds as string[]))
} }
return { return {
@ -205,41 +213,41 @@ export class OPFSStorage extends BaseStorage {
connections connections
} }
} catch (error) { } catch (error) {
// Node not found or other error // Noun not found or other error
return null return null
} }
} }
/** /**
* Get all nodes from storage * Get all nouns from storage
*/ */
protected async getAllNodes(): Promise<HNSWNode[]> { protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
await this.ensureInitialized() await this.ensureInitialized()
const allNodes: HNSWNode[] = [] const allNouns: HNSWNoun_internal[] = []
try { try {
// Iterate through all files in the nouns directory // Iterate through all files in the nouns directory
for await (const [name, handle] of this.nounsDir!.entries()) { for await (const [name, handle] of this.nounsDir!.entries()) {
if (handle.kind === 'file') { if (handle.kind === 'file') {
try { try {
// Read the node data from the file // Read the noun data from the file
const file = await safeGetFile(handle) const file = await safeGetFile(handle)
const text = await file.text() const text = await file.text()
const data = JSON.parse(text) const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>> // Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>() const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) { for (const [level, nounIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nounIds as string[]))
} }
allNodes.push({ allNouns.push({
id: data.id, id: data.id,
vector: data.vector, vector: data.vector,
connections connections
}) })
} catch (error) { } catch (error) {
console.error(`Error reading node file ${name}:`, error) console.error(`Error reading noun file ${name}:`, error)
} }
} }
} }
@ -247,7 +255,16 @@ export class OPFSStorage extends BaseStorage {
console.error('Error reading nouns directory:', error) console.error('Error reading nouns directory:', error)
} }
return allNodes return allNouns
}
/**
* Get nouns by noun type (internal implementation)
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nouns of the specified noun type
*/
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
return this.getNodesByNounType(nounType)
} }
/** /**
@ -299,6 +316,13 @@ export class OPFSStorage extends BaseStorage {
return nodes return nodes
} }
/**
* Delete a noun from storage (internal implementation)
*/
protected async deleteNoun_internal(id: string): Promise<void> {
return this.deleteNode(id)
}
/** /**
* Delete a node from storage * Delete a node from storage
*/ */
@ -316,6 +340,13 @@ export class OPFSStorage extends BaseStorage {
} }
} }
/**
* Save a verb to storage (internal implementation)
*/
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
return this.saveEdge(verb)
}
/** /**
* Save an edge to storage * Save an edge to storage
*/ */
@ -346,6 +377,13 @@ export class OPFSStorage extends BaseStorage {
} }
} }
/**
* Get a verb from storage (internal implementation)
*/
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
return this.getEdge(id)
}
/** /**
* Get an edge from storage * Get an edge from storage
*/ */
@ -367,15 +405,32 @@ export class OPFSStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
return { return {
id: data.id, id: data.id,
vector: data.vector, vector: data.vector,
connections, connections,
sourceId: data.sourceId, sourceId: data.sourceId || data.source,
targetId: data.targetId, targetId: data.targetId || data.target,
type: data.type, source: data.sourceId || data.source,
target: data.targetId || data.target,
verb: data.type || data.verb,
weight: data.weight, weight: data.weight,
metadata: data.metadata metadata: data.metadata,
createdAt: data.createdAt || defaultTimestamp,
updatedAt: data.updatedAt || defaultTimestamp,
createdBy: data.createdBy || defaultCreatedBy
} }
} catch (error) { } catch (error) {
// Edge not found or other error // Edge not found or other error
@ -383,6 +438,13 @@ export class OPFSStorage extends BaseStorage {
} }
} }
/**
* Get all verbs from storage (internal implementation)
*/
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
return this.getAllEdges()
}
/** /**
* Get all edges from storage * Get all edges from storage
*/ */
@ -406,15 +468,32 @@ export class OPFSStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
allEdges.push({ allEdges.push({
id: data.id, id: data.id,
vector: data.vector, vector: data.vector,
connections, connections,
sourceId: data.sourceId, sourceId: data.sourceId || data.source,
targetId: data.targetId, targetId: data.targetId || data.target,
type: data.type, source: data.sourceId || data.source,
target: data.targetId || data.target,
verb: data.type || data.verb,
weight: data.weight, weight: data.weight,
metadata: data.metadata metadata: data.metadata,
createdAt: data.createdAt || defaultTimestamp,
updatedAt: data.updatedAt || defaultTimestamp,
createdBy: data.createdBy || defaultCreatedBy
}) })
} catch (error) { } catch (error) {
console.error(`Error reading edge file ${name}:`, error) console.error(`Error reading edge file ${name}:`, error)
@ -428,12 +507,26 @@ export class OPFSStorage extends BaseStorage {
return allEdges return allEdges
} }
/**
* Get verbs by source (internal implementation)
*/
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId)
}
/** /**
* Get edges by source * Get edges by source
*/ */
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> { protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges() const edges = await this.getAllEdges()
return edges.filter((edge) => edge.sourceId === sourceId) return edges.filter((edge) => (edge.sourceId || edge.source) === sourceId)
}
/**
* Get verbs by target (internal implementation)
*/
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId)
} }
/** /**
@ -441,7 +534,14 @@ export class OPFSStorage extends BaseStorage {
*/ */
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> { protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges() const edges = await this.getAllEdges()
return edges.filter((edge) => edge.targetId === targetId) return edges.filter((edge) => (edge.targetId || edge.target) === targetId)
}
/**
* Get verbs by type (internal implementation)
*/
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type)
} }
/** /**
@ -449,7 +549,14 @@ export class OPFSStorage extends BaseStorage {
*/ */
protected async getEdgesByType(type: string): Promise<Edge[]> { protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges() const edges = await this.getAllEdges()
return edges.filter((edge) => edge.type === type) return edges.filter((edge) => (edge.type || edge.verb) === type)
}
/**
* Delete a verb from storage (internal implementation)
*/
protected async deleteVerb_internal(id: string): Promise<void> {
return this.deleteEdge(id)
} }
/** /**

View file

@ -147,6 +147,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Save a noun to storage (internal implementation)
*/
protected async saveNoun_internal(noun: HNSWNoun): Promise<void> {
return this.saveNode(noun)
}
/** /**
* Save a node to storage * Save a node to storage
*/ */
@ -209,6 +216,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Get a noun from storage (internal implementation)
*/
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
return this.getNode(id)
}
/** /**
* Get a node from storage * Get a node from storage
*/ */
@ -277,6 +291,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Get all nouns from storage (internal implementation)
*/
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
return this.getAllNodes()
}
/** /**
* Get all nodes from storage * Get all nodes from storage
*/ */
@ -401,6 +422,15 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Get nouns by noun type (internal implementation)
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nouns of the specified noun type
*/
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
return this.getNodesByNounType(nounType)
}
/** /**
* Get nodes by noun type * Get nodes by noun type
* @param nounType The noun type to filter by * @param nounType The noun type to filter by
@ -429,6 +459,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Delete a noun from storage (internal implementation)
*/
protected async deleteNoun_internal(id: string): Promise<void> {
return this.deleteNode(id)
}
/** /**
* Delete a node from storage * Delete a node from storage
*/ */
@ -452,6 +489,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Save a verb to storage (internal implementation)
*/
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
return this.saveEdge(verb)
}
/** /**
* Save an edge to storage * Save an edge to storage
*/ */
@ -485,6 +529,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Get a verb from storage (internal implementation)
*/
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
return this.getEdge(id)
}
/** /**
* Get an edge from storage * Get an edge from storage
*/ */
@ -524,7 +575,9 @@ export class S3CompatibleStorage extends BaseStorage {
// Ensure the parsed edge has the expected properties // Ensure the parsed edge has the expected properties
if (!parsedEdge || !parsedEdge.id || !parsedEdge.vector || !parsedEdge.connections || if (!parsedEdge || !parsedEdge.id || !parsedEdge.vector || !parsedEdge.connections ||
!parsedEdge.sourceId || !parsedEdge.targetId || !parsedEdge.type) { !(parsedEdge.sourceId || parsedEdge.source) ||
!(parsedEdge.targetId || parsedEdge.target) ||
!(parsedEdge.type || parsedEdge.verb)) {
console.error(`Invalid edge data for ${id}:`, parsedEdge) console.error(`Invalid edge data for ${id}:`, parsedEdge)
return null return null
} }
@ -535,15 +588,33 @@ export class S3CompatibleStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
const edge = { const edge = {
id: parsedEdge.id, id: parsedEdge.id,
vector: parsedEdge.vector, vector: parsedEdge.vector,
connections, connections,
sourceId: parsedEdge.sourceId, sourceId: parsedEdge.sourceId || parsedEdge.source,
targetId: parsedEdge.targetId, targetId: parsedEdge.targetId || parsedEdge.target,
type: parsedEdge.type, source: parsedEdge.sourceId || parsedEdge.source,
target: parsedEdge.targetId || parsedEdge.target,
verb: parsedEdge.type || parsedEdge.verb,
type: parsedEdge.type || parsedEdge.verb,
weight: parsedEdge.weight || 1.0, // Default weight if not provided weight: parsedEdge.weight || 1.0, // Default weight if not provided
metadata: parsedEdge.metadata || {} metadata: parsedEdge.metadata || {},
createdAt: parsedEdge.createdAt || defaultTimestamp,
updatedAt: parsedEdge.updatedAt || defaultTimestamp,
createdBy: parsedEdge.createdBy || defaultCreatedBy
} }
console.log(`Successfully retrieved edge ${id}:`, edge) console.log(`Successfully retrieved edge ${id}:`, edge)
@ -559,6 +630,13 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Get all verbs from storage (internal implementation)
*/
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
return this.getAllEdges()
}
/** /**
* Get all edges from storage * Get all edges from storage
*/ */
@ -611,15 +689,33 @@ export class S3CompatibleStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[])) connections.set(Number(level), new Set(nodeIds as string[]))
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
return { return {
id: parsedEdge.id, id: parsedEdge.id,
vector: parsedEdge.vector, vector: parsedEdge.vector,
connections, connections,
sourceId: parsedEdge.sourceId, sourceId: parsedEdge.sourceId || parsedEdge.source,
targetId: parsedEdge.targetId, targetId: parsedEdge.targetId || parsedEdge.target,
type: parsedEdge.type, source: parsedEdge.sourceId || parsedEdge.source,
weight: parsedEdge.weight, target: parsedEdge.targetId || parsedEdge.target,
metadata: parsedEdge.metadata verb: parsedEdge.type || parsedEdge.verb,
type: parsedEdge.type || parsedEdge.verb,
weight: parsedEdge.weight || 1.0,
metadata: parsedEdge.metadata || {},
createdAt: parsedEdge.createdAt || defaultTimestamp,
updatedAt: parsedEdge.updatedAt || defaultTimestamp,
createdBy: parsedEdge.createdBy || defaultCreatedBy
} }
} catch (error) { } catch (error) {
console.error(`Error getting edge from ${object.Key}:`, error) console.error(`Error getting edge from ${object.Key}:`, error)
@ -637,12 +733,26 @@ export class S3CompatibleStorage extends BaseStorage {
} }
} }
/**
* Get verbs by source (internal implementation)
*/
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId)
}
/** /**
* Get edges by source * Get edges by source
*/ */
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> { protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges() const edges = await this.getAllEdges()
return edges.filter((edge) => edge.sourceId === sourceId) return edges.filter((edge) => (edge.sourceId || edge.source) === sourceId)
}
/**
* Get verbs by target (internal implementation)
*/
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId)
} }
/** /**
@ -650,7 +760,14 @@ export class S3CompatibleStorage extends BaseStorage {
*/ */
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> { protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges() const edges = await this.getAllEdges()
return edges.filter((edge) => edge.targetId === targetId) return edges.filter((edge) => (edge.targetId || edge.target) === targetId)
}
/**
* Get verbs by type (internal implementation)
*/
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type)
} }
/** /**
@ -658,7 +775,14 @@ export class S3CompatibleStorage extends BaseStorage {
*/ */
protected async getEdgesByType(type: string): Promise<Edge[]> { protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges() const edges = await this.getAllEdges()
return edges.filter((edge) => edge.type === type) return edges.filter((edge) => (edge.type || edge.verb) === type)
}
/**
* Delete a verb from storage (internal implementation)
*/
protected async deleteVerb_internal(id: string): Promise<void> {
return this.deleteEdge(id)
} }
/** /**

View file

@ -40,7 +40,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async saveNoun(noun: HNSWNoun): Promise<void> { public async saveNoun(noun: HNSWNoun): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
return this.saveNode(noun) return this.saveNoun_internal(noun)
} }
/** /**
@ -48,7 +48,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getNoun(id: string): Promise<HNSWNoun | null> { public async getNoun(id: string): Promise<HNSWNoun | null> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getNode(id) return this.getNoun_internal(id)
} }
/** /**
@ -56,7 +56,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getAllNouns(): Promise<HNSWNoun[]> { public async getAllNouns(): Promise<HNSWNoun[]> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getAllNodes() return this.getAllNouns_internal()
} }
/** /**
@ -66,7 +66,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> { public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getNodesByNounType(nounType) return this.getNounsByNounType_internal(nounType)
} }
/** /**
@ -74,7 +74,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async deleteNoun(id: string): Promise<void> { public async deleteNoun(id: string): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
return this.deleteNode(id) return this.deleteNoun_internal(id)
} }
/** /**
@ -82,7 +82,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async saveVerb(verb: GraphVerb): Promise<void> { public async saveVerb(verb: GraphVerb): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
return this.saveEdge(verb) return this.saveVerb_internal(verb)
} }
/** /**
@ -90,7 +90,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getVerb(id: string): Promise<GraphVerb | null> { public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getEdge(id) return this.getVerb_internal(id)
} }
/** /**
@ -98,7 +98,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getAllVerbs(): Promise<GraphVerb[]> { public async getAllVerbs(): Promise<GraphVerb[]> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getAllEdges() return this.getAllVerbs_internal()
} }
/** /**
@ -106,7 +106,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> { public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getEdgesBySource(sourceId) return this.getVerbsBySource_internal(sourceId)
} }
/** /**
@ -114,7 +114,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> { public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getEdgesByTarget(targetId) return this.getVerbsByTarget_internal(targetId)
} }
/** /**
@ -122,7 +122,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async getVerbsByType(type: string): Promise<GraphVerb[]> { public async getVerbsByType(type: string): Promise<GraphVerb[]> {
await this.ensureInitialized() await this.ensureInitialized()
return this.getEdgesByType(type) return this.getVerbsByType_internal(type)
} }
/** /**
@ -130,7 +130,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*/ */
public async deleteVerb(id: string): Promise<void> { public async deleteVerb(id: string): Promise<void> {
await this.ensureInitialized() await this.ensureInitialized()
return this.deleteEdge(id) return this.deleteVerb_internal(id)
} }
/** /**
@ -163,76 +163,76 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public abstract getMetadata(id: string): Promise<any | null> public abstract getMetadata(id: string): Promise<any | null>
/** /**
* Save a node to storage * Save a noun to storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract saveNode(node: HNSWNoun): Promise<void> protected abstract saveNoun_internal(noun: HNSWNoun): Promise<void>
/** /**
* Get a node from storage * Get a noun from storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getNode(id: string): Promise<HNSWNoun | null> protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>
/** /**
* Get all nodes from storage * Get all nouns from storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getAllNodes(): Promise<HNSWNoun[]> protected abstract getAllNouns_internal(): Promise<HNSWNoun[]>
/** /**
* Get nodes by noun type * Get nouns by noun type
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getNodesByNounType(nounType: string): Promise<HNSWNoun[]> protected abstract getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>
/** /**
* Delete a node from storage * Delete a noun from storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract deleteNode(id: string): Promise<void> protected abstract deleteNoun_internal(id: string): Promise<void>
/** /**
* Save an edge to storage * Save a verb to storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract saveEdge(edge: GraphVerb): Promise<void> protected abstract saveVerb_internal(verb: GraphVerb): Promise<void>
/** /**
* Get an edge from storage * Get a verb from storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getEdge(id: string): Promise<GraphVerb | null> protected abstract getVerb_internal(id: string): Promise<GraphVerb | null>
/** /**
* Get all edges from storage * Get all verbs from storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getAllEdges(): Promise<GraphVerb[]> protected abstract getAllVerbs_internal(): Promise<GraphVerb[]>
/** /**
* Get edges by source * Get verbs by source
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getEdgesBySource(sourceId: string): Promise<GraphVerb[]> protected abstract getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>
/** /**
* Get edges by target * Get verbs by target
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getEdgesByTarget(targetId: string): Promise<GraphVerb[]> protected abstract getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>
/** /**
* Get edges by type * Get verbs by type
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract getEdgesByType(type: string): Promise<GraphVerb[]> protected abstract getVerbsByType_internal(type: string): Promise<GraphVerb[]>
/** /**
* Delete an edge from storage * Delete a verb from storage
* This method should be implemented by each specific adapter * This method should be implemented by each specific adapter
*/ */
protected abstract deleteEdge(id: string): Promise<void> protected abstract deleteVerb_internal(id: string): Promise<void>
/** /**
* Helper method to convert a Map to a plain object for serialization * Helper method to convert a Map to a plain object for serialization

View file

@ -34,7 +34,7 @@ export interface GraphNoun {
} }
/** /**
* Base interface for edges (verbs) in the graph * Base interface for verbs in the graph
* Represents relationships between nouns * Represents relationships between nouns
*/ */
export interface GraphVerb { export interface GraphVerb {
@ -45,6 +45,7 @@ export interface GraphVerb {
verb: VerbType // Type of relationship verb: VerbType // Type of relationship
createdAt: Timestamp // When the verb was created createdAt: Timestamp // When the verb was created
updatedAt: Timestamp // When the verb was last updated updatedAt: Timestamp // When the verb was last updated
createdBy: CreatorMetadata // Information about what created this verb
data?: Record<string, any> // Additional flexible data storage data?: Record<string, any> // Additional flexible data storage
embedding?: number[] // Vector representation of the relationship embedding?: number[] // Vector representation of the relationship
confidence?: number // Confidence score (0-1) confidence?: number // Confidence score (0-1)

View file

@ -120,15 +120,25 @@ describe('OPFSStorage', () => {
// Create test verb // Create test verb
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5] const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const timestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
const testVerb = { const testVerb = {
id: 'test-verb-1', id: 'test-verb-1',
vector: testVector, vector: testVector,
connections: new Map(), connections: new Map(),
sourceId: 'source-noun-1', source: 'source-noun-1',
targetId: 'target-noun-1', target: 'target-noun-1',
type: 'test-relation', verb: 'test-relation',
weight: 0.75, weight: 0.75,
metadata: { description: 'Test relation' } metadata: { description: 'Test relation' },
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
augmentation: 'test-service',
version: '1.0'
}
} }
// Save the verb // Save the verb
@ -141,11 +151,17 @@ describe('OPFSStorage', () => {
expect(retrievedVerb).toBeDefined() expect(retrievedVerb).toBeDefined()
expect(retrievedVerb?.id).toBe('test-verb-1') expect(retrievedVerb?.id).toBe('test-verb-1')
expect(retrievedVerb?.vector).toEqual(testVector) expect(retrievedVerb?.vector).toEqual(testVector)
expect(retrievedVerb?.sourceId).toBe('source-noun-1') expect(retrievedVerb?.source).toBe('source-noun-1')
expect(retrievedVerb?.targetId).toBe('target-noun-1') expect(retrievedVerb?.target).toBe('target-noun-1')
expect(retrievedVerb?.type).toBe('test-relation') expect(retrievedVerb?.verb).toBe('test-relation')
expect(retrievedVerb?.weight).toBe(0.75) expect(retrievedVerb?.weight).toBe(0.75)
expect(retrievedVerb?.metadata).toEqual({ description: 'Test relation' }) expect(retrievedVerb?.metadata).toEqual({ description: 'Test relation' })
expect(retrievedVerb?.createdAt).toEqual(timestamp)
expect(retrievedVerb?.updatedAt).toEqual(timestamp)
expect(retrievedVerb?.createdBy).toEqual({
augmentation: 'test-service',
version: '1.0'
})
// Test getAllVerbs // Test getAllVerbs
const allVerbs = await opfsStorage.getAllVerbs() const allVerbs = await opfsStorage.getAllVerbs()

View file

@ -222,15 +222,25 @@ describe('S3CompatibleStorage', () => {
// Create test verb // Create test verb
const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5] const testVector: Vector = [0.1, 0.2, 0.3, 0.4, 0.5]
const timestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
const testVerb = { const testVerb = {
id: 'test-verb-1', id: 'test-verb-1',
vector: testVector, vector: testVector,
connections: new Map(), connections: new Map(),
sourceId: 'source-noun-1', source: 'source-noun-1',
targetId: 'target-noun-1', target: 'target-noun-1',
type: 'test-relation', verb: 'test-relation',
weight: 0.75, weight: 0.75,
metadata: { description: 'Test relation' } metadata: { description: 'Test relation' },
createdAt: timestamp,
updatedAt: timestamp,
createdBy: {
augmentation: 'test-service',
version: '1.0'
}
} }
// Save the verb // Save the verb