David Snelling
f8c45f2d8d
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
548 lines
No EOL
19 KiB
JavaScript
548 lines
No EOL
19 KiB
JavaScript
/**
|
|
* Memory Storage Adapter
|
|
* In-memory storage adapter for environments where persistent storage is not available or needed
|
|
*/
|
|
import { BaseStorage } from '../baseStorage.js';
|
|
// No type aliases needed - using the original types directly
|
|
/**
|
|
* In-memory storage adapter
|
|
* Uses Maps to store data in memory
|
|
*/
|
|
export class MemoryStorage extends BaseStorage {
|
|
constructor() {
|
|
super();
|
|
// Single map of noun ID to noun
|
|
this.nouns = new Map();
|
|
this.verbs = new Map();
|
|
this.metadata = new Map();
|
|
this.nounMetadata = new Map();
|
|
this.verbMetadata = new Map();
|
|
this.statistics = null;
|
|
}
|
|
/**
|
|
* Initialize the storage adapter
|
|
* Nothing to initialize for in-memory storage
|
|
*/
|
|
async init() {
|
|
this.isInitialized = true;
|
|
}
|
|
/**
|
|
* Save a noun to storage
|
|
*/
|
|
async saveNoun_internal(noun) {
|
|
// Create a deep copy to avoid reference issues
|
|
const nounCopy = {
|
|
id: noun.id,
|
|
vector: [...noun.vector],
|
|
connections: new Map(),
|
|
level: noun.level || 0
|
|
};
|
|
// Copy connections
|
|
for (const [level, connections] of noun.connections.entries()) {
|
|
nounCopy.connections.set(level, new Set(connections));
|
|
}
|
|
// Save the noun directly in the nouns map
|
|
this.nouns.set(noun.id, nounCopy);
|
|
}
|
|
/**
|
|
* Get a noun from storage
|
|
*/
|
|
async getNoun_internal(id) {
|
|
// Get the noun directly from the nouns map
|
|
const noun = this.nouns.get(id);
|
|
// If not found, return null
|
|
if (!noun) {
|
|
return null;
|
|
}
|
|
// Return a deep copy to avoid reference issues
|
|
const nounCopy = {
|
|
id: noun.id,
|
|
vector: [...noun.vector],
|
|
connections: new Map(),
|
|
level: noun.level || 0
|
|
};
|
|
// Copy connections
|
|
for (const [level, connections] of noun.connections.entries()) {
|
|
nounCopy.connections.set(level, new Set(connections));
|
|
}
|
|
return nounCopy;
|
|
}
|
|
/**
|
|
* Get nouns with pagination and filtering
|
|
* @param options Pagination and filtering options
|
|
* @returns Promise that resolves to a paginated result of nouns
|
|
*/
|
|
async getNouns(options = {}) {
|
|
const pagination = options.pagination || {};
|
|
const filter = options.filter || {};
|
|
// Default values
|
|
const offset = pagination.offset || 0;
|
|
const limit = pagination.limit || 100;
|
|
// Convert string types to arrays for consistent handling
|
|
const nounTypes = filter.nounType
|
|
? Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
|
|
: undefined;
|
|
const services = filter.service
|
|
? Array.isArray(filter.service) ? filter.service : [filter.service]
|
|
: undefined;
|
|
// First, collect all noun IDs that match the filter criteria
|
|
const matchingIds = [];
|
|
// Iterate through all nouns to find matches
|
|
for (const [nounId, noun] of this.nouns.entries()) {
|
|
// Get the metadata to check filters
|
|
const metadata = await this.getMetadata(nounId);
|
|
if (!metadata)
|
|
continue;
|
|
// Filter by noun type if specified
|
|
if (nounTypes && !nounTypes.includes(metadata.noun)) {
|
|
continue;
|
|
}
|
|
// Filter by service if specified
|
|
if (services && metadata.service && !services.includes(metadata.service)) {
|
|
continue;
|
|
}
|
|
// Filter by metadata fields if specified
|
|
if (filter.metadata) {
|
|
let metadataMatch = true;
|
|
for (const [key, value] of Object.entries(filter.metadata)) {
|
|
if (metadata[key] !== value) {
|
|
metadataMatch = false;
|
|
break;
|
|
}
|
|
}
|
|
if (!metadataMatch)
|
|
continue;
|
|
}
|
|
// If we got here, the noun matches all filters
|
|
matchingIds.push(nounId);
|
|
}
|
|
// Calculate pagination
|
|
const totalCount = matchingIds.length;
|
|
const paginatedIds = matchingIds.slice(offset, offset + limit);
|
|
const hasMore = offset + limit < totalCount;
|
|
// Create cursor for next page if there are more results
|
|
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
|
// Fetch the actual nouns for the current page
|
|
const items = [];
|
|
for (const id of paginatedIds) {
|
|
const noun = this.nouns.get(id);
|
|
if (!noun)
|
|
continue;
|
|
// Create a deep copy to avoid reference issues
|
|
const nounCopy = {
|
|
id: noun.id,
|
|
vector: [...noun.vector],
|
|
connections: new Map(),
|
|
level: noun.level || 0
|
|
};
|
|
// Copy connections
|
|
for (const [level, connections] of noun.connections.entries()) {
|
|
nounCopy.connections.set(level, new Set(connections));
|
|
}
|
|
items.push(nounCopy);
|
|
}
|
|
return {
|
|
items,
|
|
totalCount,
|
|
hasMore,
|
|
nextCursor
|
|
};
|
|
}
|
|
/**
|
|
* Get nouns with pagination - simplified interface for compatibility
|
|
*/
|
|
async getNounsWithPagination(options = {}) {
|
|
// Convert to the getNouns format
|
|
const result = await this.getNouns({
|
|
pagination: {
|
|
offset: options.cursor ? parseInt(options.cursor) : 0,
|
|
limit: options.limit || 100
|
|
},
|
|
filter: options.filter
|
|
});
|
|
return {
|
|
items: result.items,
|
|
totalCount: result.totalCount || 0,
|
|
hasMore: result.hasMore,
|
|
nextCursor: result.nextCursor
|
|
};
|
|
}
|
|
/**
|
|
* Get nouns by noun type
|
|
* @param nounType The noun type to filter by
|
|
* @returns Promise that resolves to an array of nouns of the specified noun type
|
|
* @deprecated Use getNouns() with filter.nounType instead
|
|
*/
|
|
async getNounsByNounType_internal(nounType) {
|
|
const result = await this.getNouns({
|
|
filter: {
|
|
nounType
|
|
}
|
|
});
|
|
return result.items;
|
|
}
|
|
/**
|
|
* Delete a noun from storage
|
|
*/
|
|
async deleteNoun_internal(id) {
|
|
this.nouns.delete(id);
|
|
}
|
|
/**
|
|
* Save a verb to storage
|
|
*/
|
|
async saveVerb_internal(verb) {
|
|
// Create a deep copy to avoid reference issues
|
|
const verbCopy = {
|
|
id: verb.id,
|
|
vector: [...verb.vector],
|
|
connections: new Map()
|
|
};
|
|
// Copy connections
|
|
for (const [level, connections] of verb.connections.entries()) {
|
|
verbCopy.connections.set(level, new Set(connections));
|
|
}
|
|
// Save the verb directly in the verbs map
|
|
this.verbs.set(verb.id, verbCopy);
|
|
}
|
|
/**
|
|
* Get a verb from storage
|
|
*/
|
|
async getVerb_internal(id) {
|
|
// Get the verb directly from the verbs map
|
|
const verb = this.verbs.get(id);
|
|
// If not found, return null
|
|
if (!verb) {
|
|
return null;
|
|
}
|
|
// Create default timestamp if not present
|
|
const defaultTimestamp = {
|
|
seconds: Math.floor(Date.now() / 1000),
|
|
nanoseconds: (Date.now() % 1000) * 1000000
|
|
};
|
|
// Create default createdBy if not present
|
|
const defaultCreatedBy = {
|
|
augmentation: 'unknown',
|
|
version: '1.0'
|
|
};
|
|
// Return a deep copy of the HNSWVerb
|
|
const verbCopy = {
|
|
id: verb.id,
|
|
vector: [...verb.vector],
|
|
connections: new Map()
|
|
};
|
|
// Copy connections
|
|
for (const [level, connections] of verb.connections.entries()) {
|
|
verbCopy.connections.set(level, new Set(connections));
|
|
}
|
|
return verbCopy;
|
|
}
|
|
/**
|
|
* Get verbs with pagination and filtering
|
|
* @param options Pagination and filtering options
|
|
* @returns Promise that resolves to a paginated result of verbs
|
|
*/
|
|
async getVerbs(options = {}) {
|
|
const pagination = options.pagination || {};
|
|
const filter = options.filter || {};
|
|
// Default values
|
|
const offset = pagination.offset || 0;
|
|
const limit = pagination.limit || 100;
|
|
// Convert string types to arrays for consistent handling
|
|
const verbTypes = filter.verbType
|
|
? Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType]
|
|
: undefined;
|
|
const sourceIds = filter.sourceId
|
|
? Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId]
|
|
: undefined;
|
|
const targetIds = filter.targetId
|
|
? Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]
|
|
: undefined;
|
|
const services = filter.service
|
|
? Array.isArray(filter.service) ? filter.service : [filter.service]
|
|
: undefined;
|
|
// First, collect all verb IDs that match the filter criteria
|
|
const matchingIds = [];
|
|
// Iterate through all verbs to find matches
|
|
for (const [verbId, hnswVerb] of this.verbs.entries()) {
|
|
// Get the metadata for this verb to do filtering
|
|
const metadata = this.verbMetadata.get(verbId);
|
|
// Filter by verb type if specified
|
|
if (verbTypes && metadata && !verbTypes.includes(metadata.type || metadata.verb || '')) {
|
|
continue;
|
|
}
|
|
// Filter by source ID if specified
|
|
if (sourceIds && metadata && !sourceIds.includes(metadata.sourceId || metadata.source || '')) {
|
|
continue;
|
|
}
|
|
// Filter by target ID if specified
|
|
if (targetIds && metadata && !targetIds.includes(metadata.targetId || metadata.target || '')) {
|
|
continue;
|
|
}
|
|
// Filter by metadata fields if specified
|
|
if (filter.metadata && metadata && metadata.data) {
|
|
let metadataMatch = true;
|
|
for (const [key, value] of Object.entries(filter.metadata)) {
|
|
if (metadata.data[key] !== value) {
|
|
metadataMatch = false;
|
|
break;
|
|
}
|
|
}
|
|
if (!metadataMatch)
|
|
continue;
|
|
}
|
|
// Filter by service if specified
|
|
if (services && metadata && metadata.createdBy && metadata.createdBy.augmentation &&
|
|
!services.includes(metadata.createdBy.augmentation)) {
|
|
continue;
|
|
}
|
|
// If we got here, the verb matches all filters
|
|
matchingIds.push(verbId);
|
|
}
|
|
// Calculate pagination
|
|
const totalCount = matchingIds.length;
|
|
const paginatedIds = matchingIds.slice(offset, offset + limit);
|
|
const hasMore = offset + limit < totalCount;
|
|
// Create cursor for next page if there are more results
|
|
const nextCursor = hasMore ? `${offset + limit}` : undefined;
|
|
// Fetch the actual verbs for the current page
|
|
const items = [];
|
|
for (const id of paginatedIds) {
|
|
const hnswVerb = this.verbs.get(id);
|
|
const metadata = this.verbMetadata.get(id);
|
|
if (!hnswVerb)
|
|
continue;
|
|
if (!metadata) {
|
|
console.warn(`Verb ${id} found but no metadata - creating minimal GraphVerb`);
|
|
// Return minimal GraphVerb if metadata is missing
|
|
items.push({
|
|
id: hnswVerb.id,
|
|
vector: hnswVerb.vector,
|
|
sourceId: '',
|
|
targetId: ''
|
|
});
|
|
continue;
|
|
}
|
|
// Create a complete GraphVerb by combining HNSWVerb with metadata
|
|
const graphVerb = {
|
|
id: hnswVerb.id,
|
|
vector: [...hnswVerb.vector],
|
|
sourceId: metadata.sourceId,
|
|
targetId: metadata.targetId,
|
|
source: metadata.source,
|
|
target: metadata.target,
|
|
verb: metadata.verb,
|
|
type: metadata.type,
|
|
weight: metadata.weight,
|
|
createdAt: metadata.createdAt,
|
|
updatedAt: metadata.updatedAt,
|
|
createdBy: metadata.createdBy,
|
|
data: metadata.data,
|
|
metadata: metadata.data // Alias for backward compatibility
|
|
};
|
|
items.push(graphVerb);
|
|
}
|
|
return {
|
|
items,
|
|
totalCount,
|
|
hasMore,
|
|
nextCursor
|
|
};
|
|
}
|
|
/**
|
|
* Get verbs by source
|
|
* @deprecated Use getVerbs() with filter.sourceId instead
|
|
*/
|
|
async getVerbsBySource_internal(sourceId) {
|
|
const result = await this.getVerbs({
|
|
filter: {
|
|
sourceId
|
|
}
|
|
});
|
|
return result.items;
|
|
}
|
|
/**
|
|
* Get verbs by target
|
|
* @deprecated Use getVerbs() with filter.targetId instead
|
|
*/
|
|
async getVerbsByTarget_internal(targetId) {
|
|
const result = await this.getVerbs({
|
|
filter: {
|
|
targetId
|
|
}
|
|
});
|
|
return result.items;
|
|
}
|
|
/**
|
|
* Get verbs by type
|
|
* @deprecated Use getVerbs() with filter.verbType instead
|
|
*/
|
|
async getVerbsByType_internal(type) {
|
|
const result = await this.getVerbs({
|
|
filter: {
|
|
verbType: type
|
|
}
|
|
});
|
|
return result.items;
|
|
}
|
|
/**
|
|
* Delete a verb from storage
|
|
*/
|
|
async deleteVerb_internal(id) {
|
|
// Delete the verb directly from the verbs map
|
|
this.verbs.delete(id);
|
|
}
|
|
/**
|
|
* Save metadata to storage
|
|
*/
|
|
async saveMetadata(id, metadata) {
|
|
this.metadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
|
}
|
|
/**
|
|
* Get metadata from storage
|
|
*/
|
|
async getMetadata(id) {
|
|
const metadata = this.metadata.get(id);
|
|
if (!metadata) {
|
|
return null;
|
|
}
|
|
return JSON.parse(JSON.stringify(metadata));
|
|
}
|
|
/**
|
|
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
|
* Memory storage implementation is simple since all data is already in memory
|
|
*/
|
|
async getMetadataBatch(ids) {
|
|
const results = new Map();
|
|
// Memory storage can handle all IDs at once since it's in-memory
|
|
for (const id of ids) {
|
|
const metadata = this.metadata.get(id);
|
|
if (metadata) {
|
|
// Deep clone to prevent mutation
|
|
results.set(id, JSON.parse(JSON.stringify(metadata)));
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
/**
|
|
* Save noun metadata to storage
|
|
*/
|
|
async saveNounMetadata(id, metadata) {
|
|
this.nounMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
|
}
|
|
/**
|
|
* Get noun metadata from storage
|
|
*/
|
|
async getNounMetadata(id) {
|
|
const metadata = this.nounMetadata.get(id);
|
|
if (!metadata) {
|
|
return null;
|
|
}
|
|
return JSON.parse(JSON.stringify(metadata));
|
|
}
|
|
/**
|
|
* Save verb metadata to storage
|
|
*/
|
|
async saveVerbMetadata(id, metadata) {
|
|
this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
|
|
}
|
|
/**
|
|
* Get verb metadata from storage
|
|
*/
|
|
async getVerbMetadata(id) {
|
|
const metadata = this.verbMetadata.get(id);
|
|
if (!metadata) {
|
|
return null;
|
|
}
|
|
return JSON.parse(JSON.stringify(metadata));
|
|
}
|
|
/**
|
|
* Clear all data from storage
|
|
*/
|
|
async clear() {
|
|
this.nouns.clear();
|
|
this.verbs.clear();
|
|
this.metadata.clear();
|
|
this.nounMetadata.clear();
|
|
this.verbMetadata.clear();
|
|
this.statistics = null;
|
|
// Clear the statistics cache
|
|
this.statisticsCache = null;
|
|
this.statisticsModified = false;
|
|
}
|
|
/**
|
|
* Get information about storage usage and capacity
|
|
*/
|
|
async getStorageStatus() {
|
|
return {
|
|
type: 'memory',
|
|
used: 0, // In-memory storage doesn't have a meaningful size
|
|
quota: null, // In-memory storage doesn't have a quota
|
|
details: {
|
|
nodeCount: this.nouns.size,
|
|
edgeCount: this.verbs.size,
|
|
metadataCount: this.metadata.size
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* Save statistics data to storage
|
|
* @param statistics The statistics data to save
|
|
*/
|
|
async saveStatisticsData(statistics) {
|
|
// For memory storage, we just need to store the statistics in memory
|
|
// Create a deep copy to avoid reference issues
|
|
this.statistics = {
|
|
nounCount: { ...statistics.nounCount },
|
|
verbCount: { ...statistics.verbCount },
|
|
metadataCount: { ...statistics.metadataCount },
|
|
hnswIndexSize: statistics.hnswIndexSize,
|
|
lastUpdated: statistics.lastUpdated,
|
|
// Include serviceActivity if present
|
|
...(statistics.serviceActivity && {
|
|
serviceActivity: Object.fromEntries(Object.entries(statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
|
}),
|
|
// Include services if present
|
|
...(statistics.services && {
|
|
services: statistics.services.map(s => ({ ...s }))
|
|
}),
|
|
// Include distributedConfig if present
|
|
...(statistics.distributedConfig && {
|
|
distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
|
|
})
|
|
};
|
|
// Since this is in-memory, there's no need for time-based partitioning
|
|
// or legacy file handling
|
|
}
|
|
/**
|
|
* Get statistics data from storage
|
|
* @returns Promise that resolves to the statistics data or null if not found
|
|
*/
|
|
async getStatisticsData() {
|
|
if (!this.statistics) {
|
|
return null;
|
|
}
|
|
// Return a deep copy to avoid reference issues
|
|
return {
|
|
nounCount: { ...this.statistics.nounCount },
|
|
verbCount: { ...this.statistics.verbCount },
|
|
metadataCount: { ...this.statistics.metadataCount },
|
|
hnswIndexSize: this.statistics.hnswIndexSize,
|
|
lastUpdated: this.statistics.lastUpdated,
|
|
// Include serviceActivity if present
|
|
...(this.statistics.serviceActivity && {
|
|
serviceActivity: Object.fromEntries(Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, { ...v }]))
|
|
}),
|
|
// Include services if present
|
|
...(this.statistics.services && {
|
|
services: this.statistics.services.map(s => ({ ...s }))
|
|
}),
|
|
// Include distributedConfig if present
|
|
...(this.statistics.distributedConfig && {
|
|
distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
|
|
})
|
|
};
|
|
// Since this is in-memory, there's no need for fallback mechanisms
|
|
// to check multiple storage locations
|
|
}
|
|
}
|
|
//# sourceMappingURL=memoryStorage.js.map
|