David Snelling
2ebed2a466
- **Removed Files**: - Deleted outdated statistics documentation files (`statistics.md`, `statistics-flush-solution.md`, `statistics-summary.md`) to clean up the repository and avoid confusion. - **Added Standards**: - Introduced `DOCUMENTATION_STANDARDS.md` to outline naming conventions and troubleshooting practices for more consistent and maintainable project documentation. - **Tests**: - Added a new test file `edge-cases.test.ts` to verify handling of edge cases, ensuring robust behavior against boundary values and invalid inputs. **Purpose**: Cleans up deprecated documentation while introducing concrete standards for maintaining and updating documentation. Enhances test coverage for unusual or boundary inputs, improving overall system resilience.
691 lines
19 KiB
TypeScript
691 lines
19 KiB
TypeScript
/**
|
|
* File System Storage Adapter
|
|
* File system storage adapter for Node.js environments
|
|
*/
|
|
|
|
import { GraphVerb, HNSWNoun, StatisticsData } from '../../coreTypes.js'
|
|
import {
|
|
BaseStorage,
|
|
NOUNS_DIR,
|
|
VERBS_DIR,
|
|
METADATA_DIR,
|
|
INDEX_DIR,
|
|
STATISTICS_KEY
|
|
} from '../baseStorage.js'
|
|
|
|
// Type aliases for better readability
|
|
type HNSWNode = HNSWNoun
|
|
type Edge = GraphVerb
|
|
|
|
// Node.js modules - dynamically imported to avoid issues in browser environments
|
|
let fs: any
|
|
let path: any
|
|
|
|
// Try to load Node.js modules
|
|
try {
|
|
// Using dynamic imports to avoid issues in browser environments
|
|
const fsPromise = import('fs')
|
|
const pathPromise = import('path')
|
|
|
|
Promise.all([fsPromise, pathPromise])
|
|
.then(([fsModule, pathModule]) => {
|
|
fs = fsModule
|
|
path = pathModule.default
|
|
})
|
|
.catch((error) => {
|
|
console.error('Failed to load Node.js modules:', error)
|
|
})
|
|
} catch (error) {
|
|
console.error(
|
|
'FileSystemStorage: Failed to load Node.js modules. This adapter is not supported in this environment.',
|
|
error
|
|
)
|
|
}
|
|
|
|
/**
|
|
* File system storage adapter for Node.js environments
|
|
* Uses the file system to store data in the specified directory structure
|
|
*/
|
|
export class FileSystemStorage extends BaseStorage {
|
|
private rootDir: string
|
|
private nounsDir!: string
|
|
private verbsDir!: string
|
|
private metadataDir!: string
|
|
private indexDir!: string
|
|
|
|
/**
|
|
* Initialize the storage adapter
|
|
* @param rootDirectory The root directory for storage
|
|
*/
|
|
constructor(rootDirectory: string) {
|
|
super()
|
|
this.rootDir = rootDirectory
|
|
// Defer path operations until init() when path module is guaranteed to be loaded
|
|
}
|
|
|
|
/**
|
|
* Initialize the storage adapter
|
|
*/
|
|
public async init(): Promise<void> {
|
|
if (this.isInitialized) {
|
|
return
|
|
}
|
|
|
|
// Check if Node.js modules are available
|
|
if (!fs || !path) {
|
|
throw new Error(
|
|
'FileSystemStorage requires a Node.js environment, but `fs` and `path` modules could not be loaded.'
|
|
)
|
|
}
|
|
|
|
try {
|
|
// Initialize directory paths now that path module is loaded
|
|
this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
|
|
this.verbsDir = path.join(this.rootDir, VERBS_DIR)
|
|
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
|
|
this.indexDir = path.join(this.rootDir, INDEX_DIR)
|
|
|
|
// Create the root directory if it doesn't exist
|
|
await this.ensureDirectoryExists(this.rootDir)
|
|
|
|
// Create the nouns directory if it doesn't exist
|
|
await this.ensureDirectoryExists(this.nounsDir)
|
|
|
|
// Create the verbs directory if it doesn't exist
|
|
await this.ensureDirectoryExists(this.verbsDir)
|
|
|
|
// Create the metadata directory if it doesn't exist
|
|
await this.ensureDirectoryExists(this.metadataDir)
|
|
|
|
// Create the index directory if it doesn't exist
|
|
await this.ensureDirectoryExists(this.indexDir)
|
|
|
|
this.isInitialized = true
|
|
} catch (error) {
|
|
console.error('Error initializing FileSystemStorage:', error)
|
|
throw error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ensure a directory exists, creating it if necessary
|
|
*/
|
|
private async ensureDirectoryExists(dirPath: string): Promise<void> {
|
|
try {
|
|
await fs.promises.mkdir(dirPath, { recursive: true })
|
|
} catch (error: any) {
|
|
// Ignore EEXIST error, which means the directory already exists
|
|
if (error.code !== 'EEXIST') {
|
|
throw error
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save a node to storage
|
|
*/
|
|
protected async saveNode(node: HNSWNode): Promise<void> {
|
|
await this.ensureInitialized()
|
|
|
|
// Convert connections Map to a serializable format
|
|
const serializableNode = {
|
|
...node,
|
|
connections: this.mapToObject(node.connections, (set) =>
|
|
Array.from(set as Set<string>)
|
|
)
|
|
}
|
|
|
|
const filePath = path.join(this.nounsDir, `${node.id}.json`)
|
|
await fs.promises.writeFile(
|
|
filePath,
|
|
JSON.stringify(serializableNode, null, 2)
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Get a node from storage
|
|
*/
|
|
protected async getNode(id: string): Promise<HNSWNode | null> {
|
|
await this.ensureInitialized()
|
|
|
|
const filePath = path.join(this.nounsDir, `${id}.json`)
|
|
try {
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
const parsedNode = JSON.parse(data)
|
|
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
|
const connections = new Map<number, Set<string>>()
|
|
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
|
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
|
}
|
|
|
|
return {
|
|
id: parsedNode.id,
|
|
vector: parsedNode.vector,
|
|
connections
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error reading node ${id}:`, error)
|
|
}
|
|
return null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get all nodes from storage
|
|
*/
|
|
protected async getAllNodes(): Promise<HNSWNode[]> {
|
|
await this.ensureInitialized()
|
|
|
|
const allNodes: HNSWNode[] = []
|
|
try {
|
|
const files = await fs.promises.readdir(this.nounsDir)
|
|
for (const file of files) {
|
|
if (file.endsWith('.json')) {
|
|
const filePath = path.join(this.nounsDir, file)
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
const parsedNode = JSON.parse(data)
|
|
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
|
const connections = new Map<number, Set<string>>()
|
|
for (const [level, nodeIds] of Object.entries(
|
|
parsedNode.connections
|
|
)) {
|
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
|
}
|
|
|
|
allNodes.push({
|
|
id: parsedNode.id,
|
|
vector: parsedNode.vector,
|
|
connections
|
|
})
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
|
}
|
|
}
|
|
return allNodes
|
|
}
|
|
|
|
/**
|
|
* Get nodes by noun type
|
|
* @param nounType The noun type to filter by
|
|
* @returns Promise that resolves to an array of nodes of the specified noun type
|
|
*/
|
|
protected async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
|
|
await this.ensureInitialized()
|
|
|
|
const nouns: HNSWNode[] = []
|
|
try {
|
|
const files = await fs.promises.readdir(this.nounsDir)
|
|
for (const file of files) {
|
|
if (file.endsWith('.json')) {
|
|
const filePath = path.join(this.nounsDir, file)
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
const parsedNode = JSON.parse(data)
|
|
|
|
// Filter by noun type using metadata
|
|
const nodeId = parsedNode.id
|
|
const metadata = await this.getMetadata(nodeId)
|
|
if (metadata && metadata.noun === nounType) {
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
|
const connections = new Map<number, Set<string>>()
|
|
for (const [level, nodeIds] of Object.entries(
|
|
parsedNode.connections
|
|
)) {
|
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
|
}
|
|
|
|
nouns.push({
|
|
id: parsedNode.id,
|
|
vector: parsedNode.vector,
|
|
connections
|
|
})
|
|
}
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error reading directory ${this.nounsDir}:`, error)
|
|
}
|
|
}
|
|
|
|
return nouns
|
|
}
|
|
|
|
/**
|
|
* Delete a node from storage
|
|
*/
|
|
protected async deleteNode(id: string): Promise<void> {
|
|
await this.ensureInitialized()
|
|
|
|
const filePath = path.join(this.nounsDir, `${id}.json`)
|
|
try {
|
|
await fs.promises.unlink(filePath)
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error deleting node file ${filePath}:`, error)
|
|
throw error
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save an edge to storage
|
|
*/
|
|
protected async saveEdge(edge: Edge): Promise<void> {
|
|
await this.ensureInitialized()
|
|
|
|
// Convert connections Map to a serializable format
|
|
const serializableEdge = {
|
|
...edge,
|
|
connections: this.mapToObject(edge.connections, (set) =>
|
|
Array.from(set as Set<string>)
|
|
)
|
|
}
|
|
|
|
const filePath = path.join(this.verbsDir, `${edge.id}.json`)
|
|
await fs.promises.writeFile(
|
|
filePath,
|
|
JSON.stringify(serializableEdge, null, 2)
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Get an edge from storage
|
|
*/
|
|
protected async getEdge(id: string): Promise<Edge | null> {
|
|
await this.ensureInitialized()
|
|
|
|
const filePath = path.join(this.verbsDir, `${id}.json`)
|
|
try {
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
const parsedEdge = JSON.parse(data)
|
|
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
|
const connections = new Map<number, Set<string>>()
|
|
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
|
}
|
|
|
|
return {
|
|
id: parsedEdge.id,
|
|
vector: parsedEdge.vector,
|
|
connections,
|
|
sourceId: parsedEdge.sourceId,
|
|
targetId: parsedEdge.targetId,
|
|
type: parsedEdge.type,
|
|
weight: parsedEdge.weight,
|
|
metadata: parsedEdge.metadata
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error reading edge ${id}:`, error)
|
|
}
|
|
return null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get all edges from storage
|
|
*/
|
|
protected async getAllEdges(): Promise<Edge[]> {
|
|
await this.ensureInitialized()
|
|
|
|
const allEdges: Edge[] = []
|
|
try {
|
|
const files = await fs.promises.readdir(this.verbsDir)
|
|
for (const file of files) {
|
|
if (file.endsWith('.json')) {
|
|
const filePath = path.join(this.verbsDir, file)
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
const parsedEdge = JSON.parse(data)
|
|
|
|
// Convert serialized connections back to Map<number, Set<string>>
|
|
const connections = new Map<number, Set<string>>()
|
|
for (const [level, nodeIds] of Object.entries(
|
|
parsedEdge.connections
|
|
)) {
|
|
connections.set(Number(level), new Set(nodeIds as string[]))
|
|
}
|
|
|
|
allEdges.push({
|
|
id: parsedEdge.id,
|
|
vector: parsedEdge.vector,
|
|
connections,
|
|
sourceId: parsedEdge.sourceId,
|
|
targetId: parsedEdge.targetId,
|
|
type: parsedEdge.type,
|
|
weight: parsedEdge.weight,
|
|
metadata: parsedEdge.metadata
|
|
})
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error reading directory ${this.verbsDir}:`, error)
|
|
}
|
|
}
|
|
return allEdges
|
|
}
|
|
|
|
/**
|
|
* Get edges by source
|
|
*/
|
|
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
|
|
const edges = await this.getAllEdges()
|
|
return edges.filter((edge) => edge.sourceId === sourceId)
|
|
}
|
|
|
|
/**
|
|
* Get edges by target
|
|
*/
|
|
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
|
|
const edges = await this.getAllEdges()
|
|
return edges.filter((edge) => edge.targetId === targetId)
|
|
}
|
|
|
|
/**
|
|
* Get edges by type
|
|
*/
|
|
protected async getEdgesByType(type: string): Promise<Edge[]> {
|
|
const edges = await this.getAllEdges()
|
|
return edges.filter((edge) => edge.type === type)
|
|
}
|
|
|
|
/**
|
|
* Delete an edge from storage
|
|
*/
|
|
protected async deleteEdge(id: string): Promise<void> {
|
|
await this.ensureInitialized()
|
|
|
|
const filePath = path.join(this.verbsDir, `${id}.json`)
|
|
try {
|
|
await fs.promises.unlink(filePath)
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error deleting edge file ${filePath}:`, error)
|
|
throw error
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save metadata to storage
|
|
*/
|
|
public async saveMetadata(id: string, metadata: any): Promise<void> {
|
|
await this.ensureInitialized()
|
|
|
|
const filePath = path.join(this.metadataDir, `${id}.json`)
|
|
await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2))
|
|
}
|
|
|
|
/**
|
|
* Get metadata from storage
|
|
*/
|
|
public async getMetadata(id: string): Promise<any | null> {
|
|
await this.ensureInitialized()
|
|
|
|
const filePath = path.join(this.metadataDir, `${id}.json`)
|
|
try {
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
return JSON.parse(data)
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error reading metadata ${id}:`, error)
|
|
}
|
|
return null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clear all data from storage
|
|
*/
|
|
public async clear(): Promise<void> {
|
|
await this.ensureInitialized()
|
|
|
|
// Helper function to remove all files in a directory
|
|
const removeDirectoryContents = async (dirPath: string): Promise<void> => {
|
|
try {
|
|
const files = await fs.promises.readdir(dirPath)
|
|
for (const file of files) {
|
|
const filePath = path.join(dirPath, file)
|
|
const stats = await fs.promises.stat(filePath)
|
|
if (stats.isDirectory()) {
|
|
await removeDirectoryContents(filePath)
|
|
await fs.promises.rmdir(filePath)
|
|
} else {
|
|
await fs.promises.unlink(filePath)
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(`Error removing directory contents ${dirPath}:`, error)
|
|
throw error
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove all files in the nouns directory
|
|
await removeDirectoryContents(this.nounsDir)
|
|
|
|
// Remove all files in the verbs directory
|
|
await removeDirectoryContents(this.verbsDir)
|
|
|
|
// Remove all files in the metadata directory
|
|
await removeDirectoryContents(this.metadataDir)
|
|
|
|
// Remove all files in the index directory
|
|
await removeDirectoryContents(this.indexDir)
|
|
}
|
|
|
|
/**
|
|
* Get information about storage usage and capacity
|
|
*/
|
|
public async getStorageStatus(): Promise<{
|
|
type: string
|
|
used: number
|
|
quota: number | null
|
|
details?: Record<string, any>
|
|
}> {
|
|
await this.ensureInitialized()
|
|
|
|
try {
|
|
// Calculate the total size of all files in the storage directories
|
|
let totalSize = 0
|
|
|
|
// Helper function to calculate directory size
|
|
const calculateSize = async (dirPath: string): Promise<number> => {
|
|
let size = 0
|
|
try {
|
|
const files = await fs.promises.readdir(dirPath)
|
|
for (const file of files) {
|
|
const filePath = path.join(dirPath, file)
|
|
const stats = await fs.promises.stat(filePath)
|
|
if (stats.isDirectory()) {
|
|
size += await calculateSize(filePath)
|
|
} else {
|
|
size += stats.size
|
|
}
|
|
}
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') {
|
|
console.error(
|
|
`Error calculating size for directory ${dirPath}:`,
|
|
error
|
|
)
|
|
}
|
|
}
|
|
return size
|
|
}
|
|
|
|
// Calculate size for each directory
|
|
const nounsDirSize = await calculateSize(this.nounsDir)
|
|
const verbsDirSize = await calculateSize(this.verbsDir)
|
|
const metadataDirSize = await calculateSize(this.metadataDir)
|
|
const indexDirSize = await calculateSize(this.indexDir)
|
|
|
|
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
|
|
|
|
// Count files in each directory
|
|
const nounsCount = (await fs.promises.readdir(this.nounsDir)).filter(
|
|
(file: string) => file.endsWith('.json')
|
|
).length
|
|
const verbsCount = (await fs.promises.readdir(this.verbsDir)).filter(
|
|
(file: string) => file.endsWith('.json')
|
|
).length
|
|
const metadataCount = (
|
|
await fs.promises.readdir(this.metadataDir)
|
|
).filter((file: string) => file.endsWith('.json')).length
|
|
|
|
// Count nouns by type using metadata
|
|
const nounTypeCounts: Record<string, number> = {}
|
|
const metadataFiles = await fs.promises.readdir(this.metadataDir)
|
|
for (const file of metadataFiles) {
|
|
if (file.endsWith('.json')) {
|
|
try {
|
|
const filePath = path.join(this.metadataDir, file)
|
|
const data = await fs.promises.readFile(filePath, 'utf-8')
|
|
const metadata = JSON.parse(data)
|
|
if (metadata.noun) {
|
|
nounTypeCounts[metadata.noun] =
|
|
(nounTypeCounts[metadata.noun] || 0) + 1
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error reading metadata file ${file}:`, error)
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
type: 'filesystem',
|
|
used: totalSize,
|
|
quota: null, // File system doesn't provide quota information
|
|
details: {
|
|
rootDirectory: this.rootDir,
|
|
nounsCount,
|
|
verbsCount,
|
|
metadataCount,
|
|
nounsDirSize,
|
|
verbsDirSize,
|
|
metadataDirSize,
|
|
indexDirSize,
|
|
nounTypes: nounTypeCounts
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('Failed to get storage status:', error)
|
|
return {
|
|
type: 'filesystem',
|
|
used: 0,
|
|
quota: null,
|
|
details: { error: String(error) }
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Implementation of abstract methods from BaseStorage
|
|
*/
|
|
|
|
/**
|
|
* Save a noun to storage
|
|
*/
|
|
protected async saveNoun_internal(noun: HNSWNoun): Promise<void> {
|
|
return this.saveNode(noun)
|
|
}
|
|
|
|
/**
|
|
* Get a noun from storage
|
|
*/
|
|
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
|
return this.getNode(id)
|
|
}
|
|
|
|
/**
|
|
* Get all nouns from storage
|
|
*/
|
|
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
|
return this.getAllNodes()
|
|
}
|
|
|
|
/**
|
|
* Get nouns by noun type
|
|
*/
|
|
protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
|
|
return this.getNodesByNounType(nounType)
|
|
}
|
|
|
|
/**
|
|
* Delete a noun from storage
|
|
*/
|
|
protected async deleteNoun_internal(id: string): Promise<void> {
|
|
return this.deleteNode(id)
|
|
}
|
|
|
|
/**
|
|
* Save a verb to storage
|
|
*/
|
|
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
|
|
return this.saveEdge(verb)
|
|
}
|
|
|
|
/**
|
|
* Get a verb from storage
|
|
*/
|
|
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
|
|
return this.getEdge(id)
|
|
}
|
|
|
|
/**
|
|
* Get all verbs from storage
|
|
*/
|
|
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
|
|
return this.getAllEdges()
|
|
}
|
|
|
|
/**
|
|
* Get verbs by source
|
|
*/
|
|
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
|
|
return this.getEdgesBySource(sourceId)
|
|
}
|
|
|
|
/**
|
|
* Get verbs by target
|
|
*/
|
|
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
|
|
return this.getEdgesByTarget(targetId)
|
|
}
|
|
|
|
/**
|
|
* Get verbs by type
|
|
*/
|
|
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
|
|
return this.getEdgesByType(type)
|
|
}
|
|
|
|
/**
|
|
* Delete a verb from storage
|
|
*/
|
|
protected async deleteVerb_internal(id: string): Promise<void> {
|
|
return this.deleteEdge(id)
|
|
}
|
|
|
|
/**
|
|
* Save statistics data to storage
|
|
*/
|
|
protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
|
|
await this.saveMetadata(STATISTICS_KEY, statistics)
|
|
}
|
|
|
|
/**
|
|
* Get statistics data from storage
|
|
*/
|
|
protected async getStatisticsData(): Promise<StatisticsData | null> {
|
|
return this.getMetadata(STATISTICS_KEY)
|
|
}
|
|
}
|