From 79df44351cf4530bc678ad06add5bc40fc7065c0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 30 Jul 2025 11:51:39 -0700 Subject: [PATCH] **chore: remove outdated changelog and summary documents** - Deleted `CHANGES.md`, `CHANGES_SUMMARY.md`, `CONCURRENCY_ANALYSIS.md`, `CONCURRENCY_IMPLEMENTATION_SUMMARY.md`, and related developer documentation files. - Removed redundant or legacy content no longer aligned with the current codebase and workflows. - Updated repository to reflect streamlined documentation approach, reducing clutter and improving maintainability. **Purpose**: Simplify and declutter repository by removing obsolete documentation files, ensuring it remains focused and relevant. --- CHANGELOG.md | 76 ++++++++++ CHANGES.md => archive/CHANGES.md | 0 .../CHANGES_SUMMARY.md | 0 demo.md => archive/demo.md | 0 .../fix-documentation.md | 0 .../test-issue-summary.md | 0 docs/DOCUMENTATION_ORGANIZATION.md | 124 ++++++++++++++++ .../development/DEVELOPERS.md | 0 .../development/DOCUMENTATION_STANDARDS.md | 0 .../development/EXPECTED_TEST_MESSAGES.md | 0 .../development/MARKDOWN_CONVENTIONS.md | 0 .../development/PRETTY_TEST_REPORTER.md | 0 .../guides/production-migration-guide.md | 0 .../technical/CONCURRENCY_ANALYSIS.md | 0 .../CONCURRENCY_IMPLEMENTATION_SUMMARY.md | 0 .../technical/DIMENSION_MISMATCH_SUMMARY.md | 0 .../technical/METADATA_HANDLING.md | 0 .../technical/REALTIME_UPDATES.md | 0 STATISTICS.md => docs/technical/STATISTICS.md | 0 .../technical/STORAGE_CONCURRENCY_ANALYSIS.md | 0 .../technical/STORAGE_TESTING.md | 0 .../technical/TECHNICAL_GUIDES.md | 0 TESTING.md => docs/technical/TESTING.md | 0 THREADING.md => docs/technical/THREADING.md | 0 .../USE_MODEL_LOADING_EXPLANATION.md | 0 .../VECTOR_DIMENSION_STANDARDIZATION.md | 0 .../technical/VITEST_IMPROVEMENTS.md | 0 .../technical/scalingStrategy.md | 0 package.json | 8 +- scripts/update-changelog.js | 138 ++++++++++++++++++ 30 files changed, 343 insertions(+), 3 deletions(-) create mode 100644 CHANGELOG.md rename CHANGES.md => archive/CHANGES.md (100%) rename CHANGES_SUMMARY.md => archive/CHANGES_SUMMARY.md (100%) rename demo.md => archive/demo.md (100%) rename fix-documentation.md => archive/fix-documentation.md (100%) rename test-issue-summary.md => archive/test-issue-summary.md (100%) create mode 100644 docs/DOCUMENTATION_ORGANIZATION.md rename DEVELOPERS.md => docs/development/DEVELOPERS.md (100%) rename DOCUMENTATION_STANDARDS.md => docs/development/DOCUMENTATION_STANDARDS.md (100%) rename EXPECTED_TEST_MESSAGES.md => docs/development/EXPECTED_TEST_MESSAGES.md (100%) rename MARKDOWN_CONVENTIONS.md => docs/development/MARKDOWN_CONVENTIONS.md (100%) rename PRETTY_TEST_REPORTER.md => docs/development/PRETTY_TEST_REPORTER.md (100%) rename production-migration-guide.md => docs/guides/production-migration-guide.md (100%) rename CONCURRENCY_ANALYSIS.md => docs/technical/CONCURRENCY_ANALYSIS.md (100%) rename CONCURRENCY_IMPLEMENTATION_SUMMARY.md => docs/technical/CONCURRENCY_IMPLEMENTATION_SUMMARY.md (100%) rename DIMENSION_MISMATCH_SUMMARY.md => docs/technical/DIMENSION_MISMATCH_SUMMARY.md (100%) rename METADATA_HANDLING.md => docs/technical/METADATA_HANDLING.md (100%) rename REALTIME_UPDATES.md => docs/technical/REALTIME_UPDATES.md (100%) rename STATISTICS.md => docs/technical/STATISTICS.md (100%) rename STORAGE_CONCURRENCY_ANALYSIS.md => docs/technical/STORAGE_CONCURRENCY_ANALYSIS.md (100%) rename STORAGE_TESTING.md => docs/technical/STORAGE_TESTING.md (100%) rename TECHNICAL_GUIDES.md => docs/technical/TECHNICAL_GUIDES.md (100%) rename TESTING.md => docs/technical/TESTING.md (100%) rename THREADING.md => docs/technical/THREADING.md (100%) rename USE_MODEL_LOADING_EXPLANATION.md => docs/technical/USE_MODEL_LOADING_EXPLANATION.md (100%) rename VECTOR_DIMENSION_STANDARDIZATION.md => docs/technical/VECTOR_DIMENSION_STANDARDIZATION.md (100%) rename VITEST_IMPROVEMENTS.md => docs/technical/VITEST_IMPROVEMENTS.md (100%) rename scalingStrategy.md => docs/technical/scalingStrategy.md (100%) create mode 100644 scripts/update-changelog.js diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..cfbc50c2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,76 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +### Changed + +### Fixed + +## [0.26.0] - 2025-07-30 + +### Added +- Organized documentation structure with docs/ directory +- Proper CHANGELOG.md for release management +- Statistics optimizations implemented across all storage adapters +- In-memory caching of statistics data +- Batched updates with adaptive flush timing +- Time-based partitioning for statistics files +- Error handling and retry mechanisms for statistics operations + +### Changed +- Moved technical documentation to docs/technical/ +- Moved development documentation to docs/development/ +- Moved guides to docs/guides/ +- Archived temporary documentation files +- Refactored BaseStorageAdapter to include shared optimizations +- Updated FileSystemStorage, MemoryStorage, and OPFSStorage with new statistics handling +- Improved performance through reduced storage operations +- Enhanced scalability with time-based partitioning + +### Fixed +- Fixed FileSystemStorage constructor path operations issue where path module was used before being fully loaded +- Deferred path operations to init() method when path module is guaranteed to be available +- Resolved "Cannot read properties of undefined (reading 'join')" error + +### Technical Details +- Added `scheduleBatchUpdate()` and `flushStatistics()` methods to BaseStorageAdapter +- Updated core statistics methods: `saveStatistics()`, `getStatistics()`, `incrementStatistic()`, `decrementStatistic()`, and `updateHnswIndexSize()` +- Maintained backward compatibility with legacy statistics files +- Added fallback mechanisms for multiple storage locations + +## [Previous Versions] + +For detailed implementation notes and technical summaries of previous versions, see: +- `docs/technical/` - Technical documentation and analysis +- `archive/` - Archived change logs and summaries + +--- + +## How to Update This Changelog + +When making changes to the project: + +1. Add new entries under `[Unreleased]` section +2. Use the following categories: + - `Added` for new features + - `Changed` for changes in existing functionality + - `Deprecated` for soon-to-be removed features + - `Removed` for now removed features + - `Fixed` for any bug fixes + - `Security` for vulnerability fixes + +3. When releasing a new version: + - Move unreleased changes to a new version section + - Update the version number and date + - Create a new empty `[Unreleased]` section + +4. Link to GitHub releases: `[0.26.0] - 2025-07-28` format +5. Keep entries concise but informative for users +6. Include technical details in a separate subsection if needed diff --git a/CHANGES.md b/archive/CHANGES.md similarity index 100% rename from CHANGES.md rename to archive/CHANGES.md diff --git a/CHANGES_SUMMARY.md b/archive/CHANGES_SUMMARY.md similarity index 100% rename from CHANGES_SUMMARY.md rename to archive/CHANGES_SUMMARY.md diff --git a/demo.md b/archive/demo.md similarity index 100% rename from demo.md rename to archive/demo.md diff --git a/fix-documentation.md b/archive/fix-documentation.md similarity index 100% rename from fix-documentation.md rename to archive/fix-documentation.md diff --git a/test-issue-summary.md b/archive/test-issue-summary.md similarity index 100% rename from test-issue-summary.md rename to archive/test-issue-summary.md diff --git a/docs/DOCUMENTATION_ORGANIZATION.md b/docs/DOCUMENTATION_ORGANIZATION.md new file mode 100644 index 00000000..325415ed --- /dev/null +++ b/docs/DOCUMENTATION_ORGANIZATION.md @@ -0,0 +1,124 @@ +# Documentation Organization + +This document explains the new documentation structure and workflow for managing markdown files in the Brainy project. + +## Overview + +The documentation has been reorganized from 29 scattered markdown files at the root level to a clean, organized structure with only essential files at the root and categorized documentation in the `docs/` directory. + +## New Structure + +### Root Level Files (GitHub/NPM Standards) +- `README.md` - Main project documentation +- `CONTRIBUTING.md` - Contribution guidelines +- `CODE_OF_CONDUCT.md` - Community standards +- `CHANGELOG.md` - Version history and release notes + +### Documentation Directory Structure +``` +docs/ +├── technical/ # Technical documentation and analysis +│ ├── concurrency-analysis.md +│ ├── storage-concurrency-analysis.md +│ ├── threading.md +│ ├── statistics.md +│ ├── testing.md +│ ├── vitest-improvements.md +│ ├── realtime-updates.md +│ ├── metadata-handling.md +│ ├── vector-dimension-standardization.md +│ ├── use-model-loading-explanation.md +│ ├── dimension-mismatch-summary.md +│ ├── storage-testing.md +│ ├── technical-guides.md +│ └── concurrency-implementation-summary.md +├── development/ # Development and contributor documentation +│ ├── developers.md +│ ├── documentation-standards.md +│ ├── markdown-conventions.md +│ ├── expected-test-messages.md +│ └── pretty-test-reporter.md +└── guides/ # User guides and migration documentation + └── production-migration-guide.md +``` + +### Archive Directory +``` +archive/ # Archived and temporary files +├── changes.md # Old detailed changelog +├── changes-summary.md # Old changelog summary +├── demo.md # Demo documentation +├── fix-documentation.md # Temporary fix notes +└── test-issue-summary.md # Test issue summary +``` + +## CHANGELOG.md Management + +### Automated Workflow + +The project now uses automated changelog management: + +1. **Adding Changes**: Add entries to the `[Unreleased]` section in `CHANGELOG.md` +2. **Version Bumping**: Use npm scripts that automatically update the changelog: + - `npm run version:patch` - Patch version bump + changelog update + - `npm run version:minor` - Minor version bump + changelog update + - `npm run version:major` - Major version bump + changelog update + +3. **Manual Updates**: Use `npm run changelog:update` to manually update the changelog + +### Changelog Format + +The changelog follows the [Keep a Changelog](https://keepachangelog.com/) standard: + +- **Added** - New features +- **Changed** - Changes in existing functionality +- **Deprecated** - Soon-to-be removed features +- **Removed** - Now removed features +- **Fixed** - Bug fixes +- **Security** - Vulnerability fixes + +### GitHub Integration + +The CHANGELOG.md is automatically used for: +- GitHub releases (via `scripts/create-github-release.js`) +- NPM package release notes +- Version history tracking + +## Benefits of New Structure + +1. **Clean Root Directory**: Reduced from 29 to 4 essential markdown files +2. **Better Organization**: Logical categorization of documentation +3. **GitHub Compliance**: Follows GitHub and NPM best practices +4. **Automated Maintenance**: Changelog updates are automated +5. **Easy Navigation**: Clear directory structure for different doc types +6. **Historical Preservation**: Old documentation archived, not lost + +## Workflow for Contributors + +### Adding Documentation +1. **Technical docs** → `docs/technical/` +2. **Development docs** → `docs/development/` +3. **User guides** → `docs/guides/` +4. **Temporary files** → `archive/` (if needed) + +### Making Changes +1. Add changes to `[Unreleased]` section in `CHANGELOG.md` +2. Use appropriate category (Added, Changed, Fixed, etc.) +3. When ready to release, use `npm run version:patch/minor/major` +4. The changelog will be automatically updated with version and date + +### Release Process +1. Ensure `[Unreleased]` section has all changes +2. Run `npm run version:patch/minor/major` +3. Run `npm run deploy` to publish and create GitHub release +4. GitHub release will use CHANGELOG.md content + +## Migration Notes + +- All technical documentation moved to `docs/technical/` +- Development documentation moved to `docs/development/` +- Old changelog files archived in `archive/` +- Links in existing documentation may need updates +- New automation ensures changelog stays current + +This reorganization provides a sustainable, scalable approach to documentation management that follows industry best practices and integrates seamlessly with GitHub and NPM workflows. diff --git a/DEVELOPERS.md b/docs/development/DEVELOPERS.md similarity index 100% rename from DEVELOPERS.md rename to docs/development/DEVELOPERS.md diff --git a/DOCUMENTATION_STANDARDS.md b/docs/development/DOCUMENTATION_STANDARDS.md similarity index 100% rename from DOCUMENTATION_STANDARDS.md rename to docs/development/DOCUMENTATION_STANDARDS.md diff --git a/EXPECTED_TEST_MESSAGES.md b/docs/development/EXPECTED_TEST_MESSAGES.md similarity index 100% rename from EXPECTED_TEST_MESSAGES.md rename to docs/development/EXPECTED_TEST_MESSAGES.md diff --git a/MARKDOWN_CONVENTIONS.md b/docs/development/MARKDOWN_CONVENTIONS.md similarity index 100% rename from MARKDOWN_CONVENTIONS.md rename to docs/development/MARKDOWN_CONVENTIONS.md diff --git a/PRETTY_TEST_REPORTER.md b/docs/development/PRETTY_TEST_REPORTER.md similarity index 100% rename from PRETTY_TEST_REPORTER.md rename to docs/development/PRETTY_TEST_REPORTER.md diff --git a/production-migration-guide.md b/docs/guides/production-migration-guide.md similarity index 100% rename from production-migration-guide.md rename to docs/guides/production-migration-guide.md diff --git a/CONCURRENCY_ANALYSIS.md b/docs/technical/CONCURRENCY_ANALYSIS.md similarity index 100% rename from CONCURRENCY_ANALYSIS.md rename to docs/technical/CONCURRENCY_ANALYSIS.md diff --git a/CONCURRENCY_IMPLEMENTATION_SUMMARY.md b/docs/technical/CONCURRENCY_IMPLEMENTATION_SUMMARY.md similarity index 100% rename from CONCURRENCY_IMPLEMENTATION_SUMMARY.md rename to docs/technical/CONCURRENCY_IMPLEMENTATION_SUMMARY.md diff --git a/DIMENSION_MISMATCH_SUMMARY.md b/docs/technical/DIMENSION_MISMATCH_SUMMARY.md similarity index 100% rename from DIMENSION_MISMATCH_SUMMARY.md rename to docs/technical/DIMENSION_MISMATCH_SUMMARY.md diff --git a/METADATA_HANDLING.md b/docs/technical/METADATA_HANDLING.md similarity index 100% rename from METADATA_HANDLING.md rename to docs/technical/METADATA_HANDLING.md diff --git a/REALTIME_UPDATES.md b/docs/technical/REALTIME_UPDATES.md similarity index 100% rename from REALTIME_UPDATES.md rename to docs/technical/REALTIME_UPDATES.md diff --git a/STATISTICS.md b/docs/technical/STATISTICS.md similarity index 100% rename from STATISTICS.md rename to docs/technical/STATISTICS.md diff --git a/STORAGE_CONCURRENCY_ANALYSIS.md b/docs/technical/STORAGE_CONCURRENCY_ANALYSIS.md similarity index 100% rename from STORAGE_CONCURRENCY_ANALYSIS.md rename to docs/technical/STORAGE_CONCURRENCY_ANALYSIS.md diff --git a/STORAGE_TESTING.md b/docs/technical/STORAGE_TESTING.md similarity index 100% rename from STORAGE_TESTING.md rename to docs/technical/STORAGE_TESTING.md diff --git a/TECHNICAL_GUIDES.md b/docs/technical/TECHNICAL_GUIDES.md similarity index 100% rename from TECHNICAL_GUIDES.md rename to docs/technical/TECHNICAL_GUIDES.md diff --git a/TESTING.md b/docs/technical/TESTING.md similarity index 100% rename from TESTING.md rename to docs/technical/TESTING.md diff --git a/THREADING.md b/docs/technical/THREADING.md similarity index 100% rename from THREADING.md rename to docs/technical/THREADING.md diff --git a/USE_MODEL_LOADING_EXPLANATION.md b/docs/technical/USE_MODEL_LOADING_EXPLANATION.md similarity index 100% rename from USE_MODEL_LOADING_EXPLANATION.md rename to docs/technical/USE_MODEL_LOADING_EXPLANATION.md diff --git a/VECTOR_DIMENSION_STANDARDIZATION.md b/docs/technical/VECTOR_DIMENSION_STANDARDIZATION.md similarity index 100% rename from VECTOR_DIMENSION_STANDARDIZATION.md rename to docs/technical/VECTOR_DIMENSION_STANDARDIZATION.md diff --git a/VITEST_IMPROVEMENTS.md b/docs/technical/VITEST_IMPROVEMENTS.md similarity index 100% rename from VITEST_IMPROVEMENTS.md rename to docs/technical/VITEST_IMPROVEMENTS.md diff --git a/scalingStrategy.md b/docs/technical/scalingStrategy.md similarity index 100% rename from scalingStrategy.md rename to docs/technical/scalingStrategy.md diff --git a/package.json b/package.json index 9957f353..2c102568 100644 --- a/package.json +++ b/package.json @@ -55,9 +55,11 @@ "start": "node dist/unified.js", "demo": "npm run build && npm run build:browser && npx http-server -o /demo/index.html", "version": "echo 'Version updated in package.json'", - "version:patch": "npm version patch", - "version:minor": "npm version minor", - "version:major": "npm version major", + "version:patch": "npm version patch && node scripts/update-changelog.js", + "version:minor": "npm version minor && node scripts/update-changelog.js", + "version:major": "npm version major && node scripts/update-changelog.js", + "changelog:update": "node scripts/update-changelog.js", + "changelog:check": "echo 'Please ensure CHANGELOG.md [Unreleased] section has your changes before releasing'", "lint": "eslint --ext .ts,.js src/", "lint:fix": "eslint --ext .ts,.js src/ --fix", "format": "prettier --write \"src/**/*.{ts,js}\"", diff --git a/scripts/update-changelog.js b/scripts/update-changelog.js new file mode 100644 index 00000000..576314c1 --- /dev/null +++ b/scripts/update-changelog.js @@ -0,0 +1,138 @@ +#!/usr/bin/env node + +/* eslint-env node */ +/* eslint no-console: "off" */ + +/** + * Update Changelog Script + * + * This script helps maintain the CHANGELOG.md file by: + * 1. Moving unreleased changes to a new version section + * 2. Adding the current date + * 3. Creating a new empty unreleased section + * + * Usage: + * - npm run changelog:update - Interactive mode to update changelog + * - node scripts/update-changelog.js - Direct version update + */ + +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +// Get the directory of the current module +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +// Path to the root directory and CHANGELOG.md +const rootDir = path.join(__dirname, '..') +const changelogPath = path.join(rootDir, 'CHANGELOG.md') + +// Get version from command line argument or package.json +function getVersion() { + const args = process.argv.slice(2) + if (args.length > 0) { + return args[0] + } + + // Read from package.json + const packageJsonPath = path.join(rootDir, 'package.json') + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) + return packageJson.version +} + +// Get current date in YYYY-MM-DD format +function getCurrentDate() { + return new Date().toISOString().split('T')[0] +} + +// Update the changelog +function updateChangelog(version) { + if (!fs.existsSync(changelogPath)) { + console.error('CHANGELOG.md not found!') + process.exit(1) + } + + const changelog = fs.readFileSync(changelogPath, 'utf8') + const lines = changelog.split('\n') + + // Find the unreleased section + const unreleasedIndex = lines.findIndex(line => line.includes('## [Unreleased]')) + if (unreleasedIndex === -1) { + console.error('Could not find [Unreleased] section in CHANGELOG.md') + process.exit(1) + } + + // Find the next version section or end of unreleased content + let nextSectionIndex = lines.findIndex((line, index) => + index > unreleasedIndex && line.startsWith('## [') && !line.includes('[Unreleased]') + ) + + if (nextSectionIndex === -1) { + nextSectionIndex = lines.length + } + + // Extract unreleased content + const unreleasedContent = lines.slice(unreleasedIndex + 1, nextSectionIndex) + + // Check if there's actual content in unreleased section + const hasContent = unreleasedContent.some(line => + line.trim() && !line.startsWith('#') && line.includes('-') + ) + + if (!hasContent) { + console.log('No unreleased changes found. Please add changes to the [Unreleased] section first.') + console.log('Example:') + console.log('## [Unreleased]') + console.log('') + console.log('### Added') + console.log('- New feature description') + console.log('') + console.log('### Fixed') + console.log('- Bug fix description') + return + } + + // Create new version section + const currentDate = getCurrentDate() + const newVersionSection = [ + `## [${version}] - ${currentDate}`, + ...unreleasedContent + ] + + // Create new unreleased section + const newUnreleasedSection = [ + '## [Unreleased]', + '', + '### Added', + '', + '### Changed', + '', + '### Fixed', + '', + ] + + // Reconstruct the changelog + const newLines = [ + ...lines.slice(0, unreleasedIndex), + ...newUnreleasedSection, + ...newVersionSection, + ...lines.slice(nextSectionIndex) + ] + + // Write the updated changelog + fs.writeFileSync(changelogPath, newLines.join('\n')) + console.log(`✅ CHANGELOG.md updated for version ${version}`) + console.log(`📅 Release date: ${currentDate}`) + console.log('🔄 New [Unreleased] section created') +} + +// Main execution +try { + const version = getVersion() + console.log(`Updating CHANGELOG.md for version ${version}...`) + updateChangelog(version) +} catch (error) { + console.error('Error updating changelog:', error.message) + process.exit(1) +}