diff --git a/.github/workflows/changelog_aggregation.yml b/.github/workflows/changelog_aggregation.yml new file mode 100644 index 0000000..23c5ecc --- /dev/null +++ b/.github/workflows/changelog_aggregation.yml @@ -0,0 +1,57 @@ +name: Aggregate Changelogs + +on: + schedule: + # Run 3 times per day: 00:00, 08:00, 16:00 UTC + - cron: '0 0,8,16 * * *' + workflow_dispatch: # Allow manual trigger + +permissions: + contents: write # For committing generated files + +jobs: + aggregate: + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v4 + with: + token: ${{ secrets.APPPACK_DOCS_TOKEN }} + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Install dependencies + run: uv sync + + - name: Aggregate changelogs + env: + GITHUB_TOKEN: ${{ secrets.APPPACK_DOCS_TOKEN }} + run: | + uv run python scripts/aggregate_changelogs.py --verbose + + - name: Check for changes + id: verify_diff + run: | + git diff --quiet src/changelog/ || echo "changed=true" >> $GITHUB_OUTPUT + + - name: Commit and push if changed + if: steps.verify_diff.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add src/changelog/ + git commit -m "Update aggregated changelogs + + πŸ€– Generated with [Claude Code](https://claude.com/claude-code) + + Co-Authored-By: Claude " + git push diff --git a/_specs/CHANGELOG_AGGREGATOR_SPEC.md b/_specs/CHANGELOG_AGGREGATOR_SPEC.md new file mode 100644 index 0000000..b3ecfa9 --- /dev/null +++ b/_specs/CHANGELOG_AGGREGATOR_SPEC.md @@ -0,0 +1,560 @@ +# Changelog Aggregation Feature Specification + +## Overview + +This feature will aggregate CHANGELOG.md files from multiple AppPack repositories, parse them using the `keepachangelog` library, merge them chronologically, and generate individual MkDocs pages for each version entry with a combined index page. + +## Objectives + +1. **Fetch** CHANGELOG.md from multiple repositories (public and private) +2. **Parse** changelogs using the keepachangelog format +3. **Merge** entries in date descending order +4. **Generate** individual pages for each version with repository tags +5. **Create** a list page showing all versions and their changelog entries +6. **Support** local execution and scheduled GitHub Actions runs (3x daily) +7. **Plan** for extensibility to add more repositories + +## Architecture + +### Components + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Changelog Aggregator β”‚ +β”‚ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Fetcher β”‚ β”‚ Parser β”‚ β”‚ Generator β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ β”‚ β€’ Git clone │─▢│ keepachange │─▢│ MkDocs pages β”‚ β”‚ +β”‚ β”‚ β€’ GH API β”‚ β”‚ log library β”‚ β”‚ generation β”‚ β”‚ +β”‚ β”‚ β€’ Auth β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β–Ό β–Ό + Local Execution GitHub Actions + (CLI script) (scheduled workflow) +``` + +### Directory Structure + +``` +/workspace/ +β”œβ”€β”€ scripts/ +β”‚ └── aggregate_changelogs.py # Main aggregation script +β”œβ”€β”€ changelog_config.yml # Repository configuration +β”œβ”€β”€ src/ +β”‚ └── changelog/ +β”‚ β”œβ”€β”€ index.md # Combined changelog list page +β”‚ └── versions/ +β”‚ β”œβ”€β”€ cli-v1.2.3.md # Individual version pages +β”‚ β”œβ”€β”€ stacks-v2.0.1.md +β”‚ └── ci-builder-v0.5.0.md +└── .github/ + └── workflows/ + └── changelog_aggregation.yml # GHA workflow (3x daily) +``` + +## Data Structures + +### Repository Configuration (`changelog_config.yml`) + +```yaml +repositories: + - name: apppack + alias: cli + url: https://github.com/apppackio/apppack + changelog_path: CHANGELOG.md + public: true + + - name: apppack-codebuild-image + alias: ci-builder + url: https://github.com/apppackio/apppack-codebuild-image + changelog_path: CHANGELOG.md + public: false + + - name: apppack-backend + alias: stacks + url: https://github.com/apppackio/apppack-backend + changelog_path: formations/CHANGELOG.md # Monorepo subdirectory + public: false + +# Future repositories can be added here +# - name: future-repo +# alias: feature-x +# url: https://github.com/apppackio/future-repo +# changelog_path: CHANGELOG.md +# public: false +``` + +### Parsed Changelog Entry Structure + +```python +@dataclass +class ChangelogEntry: + """Represents a single version entry from a changelog""" + version: str # e.g., "1.2.3" + date: datetime # Release date + repository: str # Repository name + alias: str # Short alias (cli, stacks, ci-builder) + sections: dict[str, list[str]] # {Added: [...], Fixed: [...], ...} + + @property + def version_id(self) -> str: + """Unique identifier: 'cli-v1.2.3'""" + return f"{self.alias}-v{self.version}" + + @property + def page_path(self) -> str: + """Path to generated page: 'changelog/versions/cli-v1.2.3.md'""" + return f"changelog/versions/{self.version_id}.md" +``` + +## Component Details + +### 1. Fetcher (`scripts/aggregate_changelogs.py` - FetcherModule) + +**Responsibilities:** +- Read configuration from `changelog_config.yml` +- Clone/fetch repositories to temporary directory +- Handle authentication for private repositories +- Extract CHANGELOG.md from correct path (including monorepo subdirs) + +**Authentication Strategy:** + +**Local execution:** +- Automatically uses `gh auth token` if GitHub CLI is installed and authenticated +- Falls back to `GITHUB_TOKEN` environment variable +- Can be explicitly set with `--token` flag + +**GitHub Actions:** +- Use `GITHUB_TOKEN` with appropriate permissions +- Requires token with `repo` scope for private repositories +- Token passed as environment variable + +**Implementation approach:** +```python +def fetch_changelog(repo_config: dict, auth_token: Optional[str] = None) -> str: + """Fetch CHANGELOG.md content from a repository""" + if repo_config['public']: + # Use GitHub raw content API (no auth needed) + url = f"https://raw.githubusercontent.com/{owner}/{repo}/main/{changelog_path}" + return requests.get(url).text + else: + # Use authenticated API + headers = {"Authorization": f"token {auth_token}"} + # GitHub Contents API + url = f"https://api.github.com/repos/{owner}/{repo}/contents/{changelog_path}" + response = requests.get(url, headers=headers) + # Decode base64 content + return base64.b64decode(response.json()['content']).decode('utf-8') +``` + +### 2. Parser (`scripts/aggregate_changelogs.py` - ParserModule) + +**Responsibilities:** +- Parse CHANGELOG.md using `keepachangelog` library +- Extract version, date, and change sections +- Handle malformed or missing dates gracefully +- Normalize data into `ChangelogEntry` objects + +**Implementation:** +```python +from keepachangelog import to_dict + +def parse_changelog(content: str, repo_name: str, alias: str) -> list[ChangelogEntry]: + """Parse changelog content into structured entries""" + # Parse with show_unreleased=False to exclude unreleased versions + changelog_dict = to_dict(content, show_unreleased=False) + entries = [] + + for version, data in changelog_dict.items(): + if version == "metadata": + continue + + entry = ChangelogEntry( + version=version, + date=parse_date(data.get('release_date')), + repository=repo_name, + alias=alias, + sections={ + 'Added': data.get('added', []), + 'Changed': data.get('changed', []), + 'Deprecated': data.get('deprecated', []), + 'Removed': data.get('removed', []), + 'Fixed': data.get('fixed', []), + 'Security': data.get('security', []), + } + ) + entries.append(entry) + + return entries +``` + +### 3. Generator (`scripts/aggregate_changelogs.py` - GeneratorModule) + +**Responsibilities:** +- Sort all entries by date (descending) +- Generate individual markdown pages for each version +- Generate combined index page with all versions +- Format content with proper MkDocs metadata + +**Individual Version Page Template:** + +```markdown +--- +title: "{alias} {version}" +--- + +# {alias} {version} + +**Released:** {date} +**Repository:** {repository} + +{for section, items in sections} +## {section} + +{for item in items} +- {item} +{endfor} +{endfor} + +--- + +[← Back to Changelog](/changelog/) +``` + +**Index Page Template:** + +```markdown +--- +title: "AppPack Changelog" +--- + +# AppPack Changelog + +This page aggregates changelogs from all AppPack repositories, showing the most recent changes first. + +{for entry in sorted_entries} +## [{entry.alias} {entry.version}](/changelog/versions/{entry.version_id}/) {:.changelog-entry} + +**{entry.date}** β€’ **{entry.repository}** + +{for section, items in entry.sections if items} +### {section} +{for item in items} +- {item} +{endfor} +{endfor} + +--- + +{endfor} +``` + +## Script Interface + +### CLI Usage (Local Execution) + +```bash +# Install dependencies +uv add keepachangelog pyyaml requests + +# Run aggregator (automatically uses gh CLI if authenticated) +uv run python scripts/aggregate_changelogs.py + +# Or with explicit token via environment variable +GITHUB_TOKEN=ghp_xxx uv run python scripts/aggregate_changelogs.py + +# Or with explicit token via flag +uv run python scripts/aggregate_changelogs.py --token ghp_xxx +``` + +### Script Arguments + +```python +parser = argparse.ArgumentParser(description='Aggregate AppPack changelogs') +parser.add_argument('--config', default='changelog_config.yml', + help='Path to repository configuration file') +parser.add_argument('--output', default='merged_changelog.md', + help='Output file for merged changelog') +parser.add_argument('--token', help='GitHub personal access token (auto-detects from gh CLI)') +parser.add_argument('--verbose', '-v', action='store_true', + help='Enable verbose logging') +``` + +## GitHub Actions Integration + +### Workflow File (`.github/workflows/changelog_aggregation.yml`) + +```yaml +name: Aggregate Changelogs + +on: + schedule: + # Run 3 times per day: 00:00, 08:00, 16:00 UTC + - cron: '0 0,8,16 * * *' + workflow_dispatch: # Allow manual trigger + +permissions: + contents: write # For committing generated files + +jobs: + aggregate: + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v4 + with: + token: ${{ secrets.APPPACK_DOCS_TOKEN }} # Personal access token + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: uv sync + + - name: Aggregate changelogs + env: + GITHUB_TOKEN: ${{ secrets.APPPACK_DOCS_TOKEN }} + run: | + uv run python scripts/aggregate_changelogs.py --verbose + + - name: Check for changes + id: verify_diff + run: | + git diff --quiet src/changelog/ || echo "changed=true" >> $GITHUB_OUTPUT + + - name: Commit and push if changed + if: steps.verify_diff.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add src/changelog/ + git commit -m "Update aggregated changelogs" + git push +``` + +### Authentication for Private Repos + +**GitHub Token Requirements:** +- Needs `repo` scope to access private repositories +- Created as a Personal Access Token (Classic) or Fine-Grained Token +- Stored as repository secret: `APPPACK_DOCS_TOKEN` + +**Token Setup:** +1. Create PAT at: https://github.com/settings/tokens +2. Grant scopes: `repo` (Full control of private repositories) +3. Add to repository secrets: Settings β†’ Secrets β†’ Actions β†’ New secret +4. Name: `APPPACK_DOCS_TOKEN` + +## Navigation Integration + +### Update `src/_navigation.md` + +```markdown +* [Home](index.md) +* [Why AppPack?](why-apppack.md) +* ...existing sections... +* [Changelog](changelog/index.md) + * [All Versions](changelog/index.md) +* [Under the Hood](under-the-hood/index.md) +``` + +### Update `mkdocs.yml` (if needed) + +No changes needed - MkDocs will automatically discover pages in `src/changelog/`. + +## Error Handling + +### Fail Fast Strategy + +The script will fail immediately if any repository is unavailable or has errors. This ensures we don't generate partial/incomplete changelog data. + +1. **Missing changelog:** Fail with error message indicating which repo failed +2. **Parse error:** Fail with error showing malformed changelog +3. **Auth failure:** Fail with clear error message about authentication +4. **Network timeout:** Retry with exponential backoff (3 attempts), then fail +5. **Malformed dates:** Fail with error showing which version has invalid date + +All errors should be logged with clear messages to help debugging. + +### Logging + +```python +import logging + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) + +logger = logging.getLogger('changelog_aggregator') +``` + +## Testing Strategy + +### Unit Tests + +```python +# tests/test_parser.py +def test_parse_keepachangelog_format(): + """Test parsing valid keepachangelog format""" + +def test_handle_missing_dates(): + """Test graceful handling of missing dates""" + +def test_merge_multiple_changelogs(): + """Test correct chronological ordering""" + +# tests/test_fetcher.py +def test_fetch_public_repo(): + """Test fetching from public repository""" + +def test_fetch_private_repo_with_auth(): + """Test fetching from private repository""" + +def test_monorepo_subdirectory(): + """Test fetching from monorepo subdirectory""" +``` + +### Integration Tests + +```python +# tests/test_integration.py +def test_full_aggregation_pipeline(): + """Test complete aggregation from fetch to generation""" +``` + +## CSS Styling + +### Custom Styles (`src/stylesheets/extra.css`) + +```css +/* Changelog entry styling */ +.changelog-entry { + color: var(--md-primary-fg-color); + border-left: 4px solid var(--md-primary-fg-color); + padding-left: 1rem; + margin-left: -1rem; +} + +/* Repository tags */ +.repo-tag { + display: inline-block; + padding: 0.2em 0.6em; + font-size: 0.875em; + font-weight: 600; + border-radius: 0.25rem; + background-color: var(--md-primary-fg-color--light); + color: white; +} + +.repo-tag.cli { background-color: #7c3aed; } +.repo-tag.stacks { background-color: #2563eb; } +.repo-tag.ci-builder { background-color: #059669; } +``` + +## Implementation Phases + +### Phase 1: Core Functionality +- [ ] Create `scripts/aggregate_changelogs.py` +- [ ] Implement fetcher for public repos +- [ ] Implement parser using keepachangelog +- [ ] Implement basic generator +- [ ] Create `changelog_config.yml` +- [ ] Test locally with CLI repo + +### Phase 2: Authentication & Private Repos +- [ ] Add authentication support (GitHub token) +- [ ] Test with private repos (ci-builder, stacks) +- [ ] Handle monorepo subdirectories +- [ ] Add error handling and logging + +### Phase 3: Page Generation +- [ ] Design page templates +- [ ] Generate individual version pages +- [ ] Generate combined index page +- [ ] Add CSS styling +- [ ] Update navigation + +### Phase 4: GitHub Actions +- [ ] Create workflow file +- [ ] Set up secrets (APPPACK_DOCS_TOKEN) +- [ ] Test manual workflow dispatch +- [ ] Enable scheduled runs (3x daily) +- [ ] Add commit automation + +### Phase 5: Polish & Documentation +- [ ] Add unit tests +- [ ] Add integration tests +- [ ] Write README for the script +- [ ] Add error notifications (optional) +- [ ] Performance optimization (caching) + +## Future Enhancements + +1. **Caching:** Cache fetched changelogs to avoid rate limits +2. **Diff detection:** Only regenerate pages if changelog changed +3. **RSS feed:** Generate RSS feed for changelog updates +4. **Search integration:** Ensure changelog entries are searchable +5. **Version filtering:** Add UI to filter by repository or date range +6. **Breaking changes:** Highlight breaking changes prominently + +## Dependencies + +Add to `pyproject.toml`: + +```toml +[project] +dependencies = [ + # ...existing dependencies... + "keepachangelog>=2.0.0", + "pyyaml>=6.0", + "requests>=2.31.0", +] +``` + +## Security Considerations + +1. **Token scope:** Use minimal required permissions +2. **Token rotation:** Rotate PAT regularly +3. **Secret management:** Never commit tokens to repository +4. **Rate limiting:** Respect GitHub API rate limits (5000 req/hour with auth) +5. **Input validation:** Validate repository configuration +6. **Output sanitization:** Escape markdown content to prevent XSS + +## Success Criteria + +- βœ… Changelog pages are generated successfully +- βœ… All 3 repositories are aggregated correctly +- βœ… Pages are sorted by date (newest first) +- βœ… Each version has its own page with correct repository tag +- βœ… Index page displays all versions with summaries +- βœ… Script runs successfully in local environment +- βœ… GitHub Actions runs 3x daily without errors +- βœ… Authentication works for private repositories +- βœ… Monorepo subdirectory (formations/) is handled correctly +- βœ… New repositories can be added easily via config file + +## Questions & Decisions + +### Resolved: +- **Format:** keepachangelog (specified by user) +- **Frequency:** 3x daily (specified by user) +- **Repos:** 3 initial repos with specific aliases + +### Confirmed Decisions: +- βœ… **No unreleased changes** - Skip any "Unreleased" entries +- βœ… **Fail fast** - If any repo is unavailable, fail the entire script (don't generate partial data) +- βœ… **No notifications** - No email/Slack alerts +- βœ… **No external links** - Don't link to GitHub releases, just show changelog content diff --git a/mkdocs.yml b/mkdocs.yml index 4e81b2f..6460723 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,6 +21,7 @@ plugins: - macros - literate-nav: nav_file: _navigation.md + - tags repo_url: https://github.com/apppackio/apppack-docs/ edit_uri: edit/main/src/ extra_css: diff --git a/pyproject.toml b/pyproject.toml index 71d8589..b79011b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,49 @@ description = "Documentation for AppPack.io" authors = [ {name = "Peter Baumgartner", email = "pete@lincolnloop.com"}, ] -dependencies = ["mkdocs", "mkdocs-literate-nav", "mkdocs-material", "mkdocs-macros-plugin", "setuptools", "pymdown-extensions"] +dependencies = ["mkdocs", "mkdocs-literate-nav", "mkdocs-material", "mkdocs-macros-plugin", "setuptools", "pymdown-extensions", "keepachangelog>=2.0.0", "requests>=2.31.0"] requires-python = ">=3.14" dynamic = ["classifiers"] license = {text = "MIT"} [project.urls] homepage = "https://github.com/apppackio/apppack-docs" + +[tool.ruff] +extend-exclude = ["migrations"] +target-version = "py314" + +[tool.ruff.lint] +ignore = [ + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `**kwargs`" + "ARG001", # Unused function argument (request, ...) + "ARG002", # Unused method argument (*args, **kwargs) + "D", # Missing or badly formatted docstrings + "E501", # Let the formatter handle long lines + "FBT", # Flake Boolean Trap (don't use arg=True in functions) + "RUF012", # Mutable class attributes https://github.com/astral-sh/ruff/issues/5243 + "COM812", # (ruff format) Checks for the absence of trailing commas + "ISC001", # (ruff format) Checks for implicitly concatenated strings on a single line +] +select = ["ALL"] + +[[tool.changelog.repositories]] +name = "apppack" +alias = "cli" +url = "https://github.com/apppackio/apppack" +changelog_path = "CHANGELOG.md" +public = true + +[[tool.changelog.repositories]] +name = "apppack-codebuild-image" +alias = "ci-builder" +url = "https://github.com/apppackio/apppack-codebuild-image" +changelog_path = "CHANGELOG.md" +public = false + +[[tool.changelog.repositories]] +name = "apppack-backend" +alias = "stacks" +url = "https://github.com/apppackio/apppack-backend" +changelog_path = "formations/CHANGELOG.md" +public = false diff --git a/scripts/aggregate_changelogs.py b/scripts/aggregate_changelogs.py new file mode 100755 index 0000000..fddb320 --- /dev/null +++ b/scripts/aggregate_changelogs.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +""" +Aggregate changelogs from multiple AppPack repositories. + +This script fetches CHANGELOG.md files from multiple repositories, +parses them using the keepachangelog format, and merges them +chronologically. +""" + +import argparse +import base64 +import logging +import os +import subprocess +import tomllib +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import requests +from keepachangelog import to_dict + +# Configure logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger("changelog_aggregator") + +# Constants +REQUEST_TIMEOUT = 30 # seconds +MIN_URL_PATH_PARTS = 2 + + +@dataclass +class ChangelogEntry: + """Represents a single version entry from a changelog""" + + version: str + date: datetime + repository: str + alias: str + sections: dict[str, list[str]] + + @property + def version_id(self) -> str: + """Unique identifier: 'cli-v1.2.3'""" + return f"{self.alias}-v{self.version}" + + +class ChangelogFetcher: + """Fetches changelog content from GitHub repositories""" + + def __init__(self, auth_token: str | None = None) -> None: + self.auth_token = auth_token + + def fetch_changelog(self, repo_config: dict) -> str: + """ + Fetch CHANGELOG.md content from a repository. + + Args: + repo_config: Repository configuration dict with url, changelog_path, etc. + + Returns: + String content of the changelog + + Raises: + ValueError: If URL is invalid or auth is missing for private repos + requests.HTTPError: If the request fails + """ + url = repo_config["url"] + changelog_path = repo_config["changelog_path"] + is_public = repo_config["public"] + + # Parse owner and repo from URL using urlparse + # Format: https://github.com/owner/repo + parsed_url = urlparse(url) + path_parts = parsed_url.path.strip("/").split("/") + if len(path_parts) < MIN_URL_PATH_PARTS: + msg = f"Invalid GitHub URL format: {url}" + raise ValueError(msg) + owner = path_parts[0] + repo_name = path_parts[1] + + logger.info( + "Fetching changelog from %s/%s at %s", owner, repo_name, changelog_path + ) + + if is_public: + # Use raw content API (no auth needed) + raw_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/main/{changelog_path}" + logger.debug("Fetching from raw URL: %s", raw_url) + + response = requests.get(raw_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + return response.text + + # Private repo - need authentication + if not self.auth_token: + msg = ( + f"Repository {owner}/{repo_name} is private but no auth token provided. " + "Set GITHUB_TOKEN environment variable or use --token flag." + ) + raise ValueError(msg) + + # Use GitHub Contents API + api_url = f"https://api.github.com/repos/{owner}/{repo_name}/contents/{changelog_path}" + headers = { + "Authorization": f"token {self.auth_token}", + "Accept": "application/vnd.github.v3+json", + } + + logger.debug("Fetching from API: %s", api_url) + response = requests.get(api_url, headers=headers, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + + # GitHub API returns base64 encoded content + content_b64 = response.json()["content"] + return base64.b64decode(content_b64).decode("utf-8") + + +class ChangelogParser: + """Parses changelog content using keepachangelog format""" + + def parse_changelog( + self, content: str, repo_name: str, alias: str + ) -> list[ChangelogEntry]: + """ + Parse changelog content into structured entries. + + Args: + content: Raw changelog markdown content + repo_name: Repository name + alias: Short alias for the repository + + Returns: + List of ChangelogEntry objects + """ + logger.info("Parsing changelog for %s (%s)", repo_name, alias) + + # Parse with show_unreleased=False to exclude unreleased versions + # to_dict expects an iterable of lines or a file path + changelog_dict = to_dict(content.splitlines(), show_unreleased=False) + + entries = [] + + for version, data in changelog_dict.items(): + if version == "metadata": + continue + + # Release date is in the metadata sub-dict + metadata = data.get("metadata", {}) + release_date = metadata.get("release_date") + + date = datetime.fromisoformat(release_date) + + entry = ChangelogEntry( + version=version, + date=date, + repository=repo_name, + alias=alias, + sections={ + "Added": data.get("added", []), + "Changed": data.get("changed", []), + "Deprecated": data.get("deprecated", []), + "Removed": data.get("removed", []), + "Fixed": data.get("fixed", []), + "Security": data.get("security", []), + }, + ) + entries.append(entry) + logger.debug(" Parsed %s released on %s", entry.version_id, date.date()) + + logger.info("Parsed %d versions from %s", len(entries), repo_name) + return entries + + +class ChangelogMerger: + """Merges multiple changelogs chronologically""" + + def merge_changelogs( + self, all_entries: list[list[ChangelogEntry]] + ) -> list[ChangelogEntry]: + """ + Merge multiple changelog entry lists and sort by date (descending). + + Args: + all_entries: List of lists of ChangelogEntry objects + + Returns: + Single sorted list of all entries (newest first) + """ + # Flatten the list of lists + merged = [] + for entries in all_entries: + merged.extend(entries) + + # Sort by date, newest first + merged.sort(key=lambda e: e.date, reverse=True) + + logger.info("Merged %d total versions", len(merged)) + return merged + + +def load_config(config_path: str) -> dict: + """Load repository configuration from pyproject.toml""" + logger.info("Loading config from %s", config_path) + + config_file = Path(config_path) + with config_file.open("rb") as f: + pyproject = tomllib.load(f) + + if "tool" not in pyproject or "changelog" not in pyproject["tool"]: + msg = "Config file must contain [tool.changelog] section" + raise ValueError(msg) + + config = pyproject["tool"]["changelog"] + logger.info("Loaded %d repositories", len(config["repositories"])) + return config + + +def generate_version_page(entry: ChangelogEntry, output_path: Path) -> None: + """Generate an individual version page""" + with output_path.open("w") as f: + # Front matter + f.write("---\n") + f.write(f'title: "{entry.alias} {entry.version}"\n') + f.write(f"tags: [{entry.alias}]\n") + f.write("---\n\n") + + # Page content + f.write(f"# {entry.alias} {entry.version}\n\n") + f.write(f"**Released:** {entry.date.strftime('%Y-%m-%d')}\n") + f.write(f"**Repository:** {entry.repository}\n\n") + + for section_name, items in entry.sections.items(): + if items: + f.write(f"## {section_name}\n\n") + f.writelines(f"- {item}\n" for item in items) + f.write("\n") + + f.write("---\n\n") + f.write("[← Back to Changelog](../index.md)\n") + + +def generate_index_page(entries: list[ChangelogEntry], output_path: Path) -> None: + """Generate the combined changelog index page""" + with output_path.open("w") as f: + # Front matter + f.write("---\n") + f.write('title: "AppPack Changelog"\n') + f.write("---\n\n") + + # Page content + f.write("# AppPack Changelog\n\n") + f.write( + "This page aggregates changelogs from all AppPack repositories, " + "showing the most recent changes first.\n\n" + ) + + for entry in entries: + f.write(f"## [{entry.alias} {entry.version}](versions/{entry.version_id}.md)\n\n") + f.write(f"**{entry.date.strftime('%Y-%m-%d')}** β€’ **{entry.repository}**\n\n") + + for section_name, items in entry.sections.items(): + if items: + f.write(f"### {section_name}\n\n") + f.writelines(f"- {item}\n" for item in items) + f.write("\n") + + f.write("---\n\n") + + +def get_auth_token(args: argparse.Namespace) -> str | None: + """Get GitHub authentication token from various sources""" + # Priority: CLI arg > env var > gh CLI (automatic) + if args.token: + logger.info("Using token from --token argument") + return args.token + + if "GITHUB_TOKEN" in os.environ: + logger.info("Using token from GITHUB_TOKEN environment variable") + return os.environ["GITHUB_TOKEN"] + + # Automatically try gh CLI if available + logger.debug("Attempting to use gh CLI authentication") + try: + result = subprocess.run( + ["gh", "auth", "token"], # noqa: S607 + capture_output=True, + text=True, + check=True, + timeout=5, + ) + token = result.stdout.strip() + if token: + logger.info("Using token from gh CLI") + return token + except FileNotFoundError: + logger.debug("gh CLI not found") + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.debug("Failed to get token from gh CLI: %s", e) + + return None + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Aggregate changelogs from multiple AppPack repositories" + ) + parser.add_argument( + "--config", + default="pyproject.toml", + help="Path to pyproject.toml file", + ) + parser.add_argument( + "--output-dir", + default="src/changelog", + help="Output directory for generated changelog pages", + ) + parser.add_argument( + "--token", + help="GitHub personal access token (auto-detects from gh CLI if not provided)", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + + args = parser.parse_args() + + if args.verbose: + logger.setLevel(logging.DEBUG) + + # Load configuration + config = load_config(args.config) + + # Get auth token + auth_token = get_auth_token(args) + if not auth_token: + logger.warning( + "No authentication token found - only public repos will be accessible. " + "Authenticate with 'gh auth login' or set GITHUB_TOKEN environment variable." + ) + + # Initialize components + fetcher = ChangelogFetcher(auth_token) + parser_obj = ChangelogParser() + merger = ChangelogMerger() + + # Fetch and parse all changelogs + all_entries = [] + for repo_config in config["repositories"]: + # Fetch + content = fetcher.fetch_changelog(repo_config) + + # Parse + entries = parser_obj.parse_changelog( + content, repo_config["name"], repo_config["alias"] + ) + all_entries.append(entries) + + # Merge all entries + merged = merger.merge_changelogs(all_entries) + + # Create output directories + output_dir = Path(args.output_dir) + versions_dir = output_dir / "versions" + versions_dir.mkdir(parents=True, exist_ok=True) + logger.info("Creating changelog pages in %s", output_dir) + + # Generate individual version pages + for entry in merged: + version_file = versions_dir / f"{entry.version_id}.md" + generate_version_page(entry, version_file) + logger.debug("Generated %s", version_file) + + # Generate index page + index_file = output_dir / "index.md" + generate_index_page(merged, index_file) + + logger.info("βœ“ Successfully aggregated changelogs!") + logger.info(" Total versions: %d", len(merged)) + logger.info(" Index page: %s", index_file) + logger.info(" Version pages: %s", versions_dir) + + +if __name__ == "__main__": + main() diff --git a/src/_navigation.md b/src/_navigation.md index 162c565..ba33458 100644 --- a/src/_navigation.md +++ b/src/_navigation.md @@ -5,4 +5,5 @@ * [How-to guides](how-to/) * [Under the hood](under-the-hood/) * [Command line reference](command-line-reference/) +* [Changelog](changelog/index.md) * [Why AppPack?](why-apppack.md) diff --git a/src/stylesheets/extra.css b/src/stylesheets/extra.css index 3c29251..99e7429 100644 --- a/src/stylesheets/extra.css +++ b/src/stylesheets/extra.css @@ -23,3 +23,34 @@ .md-source__facts { display:none } + +/* Changelog styling */ +.changelog-entry { + border-left: 4px solid var(--md-primary-fg-color); + padding-left: 1rem; + margin-left: -1rem; + margin-bottom: 2rem; +} + +.repo-tag { + display: inline-block; + padding: 0.2em 0.6em; + font-size: 0.875em; + font-weight: 600; + border-radius: 0.25rem; + background-color: var(--md-primary-fg-color--light); + color: white; + margin-left: 0.5em; +} + +.repo-tag.cli { + background-color: #7c3aed; +} + +.repo-tag.stacks { + background-color: #2563eb; +} + +.repo-tag.ci-builder { + background-color: #059669; +} diff --git a/uv.lock b/uv.lock index 9e754fd..9e8ce22 100644 --- a/uv.lock +++ b/uv.lock @@ -7,21 +7,25 @@ name = "apppack-docs" version = "0.9" source = { virtual = "." } dependencies = [ + { name = "keepachangelog" }, { name = "mkdocs" }, { name = "mkdocs-literate-nav" }, { name = "mkdocs-macros-plugin" }, { name = "mkdocs-material" }, { name = "pymdown-extensions" }, + { name = "requests" }, { name = "setuptools" }, ] [package.metadata] requires-dist = [ + { name = "keepachangelog", specifier = ">=2.0.0" }, { name = "mkdocs" }, { name = "mkdocs-literate-nav" }, { name = "mkdocs-macros-plugin" }, { name = "mkdocs-material" }, { name = "pymdown-extensions" }, + { name = "requests", specifier = ">=2.31.0" }, { name = "setuptools" }, ] @@ -145,6 +149,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "keepachangelog" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/2e/951923e56c149b6772dbffee0dcac00738405d7d7a0fdd30537047cf6727/keepachangelog-2.0.0.tar.gz", hash = "sha256:6839f0646690ae514aca6dedbc7b8aa652c56db3be43081a1940c55c1c807709", size = 24488, upload-time = "2024-06-14T12:56:28.442Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/06/b4f06ca7afb5d9e942c642980c308ffcfa1fa0e8b0a3ddbec78483ef1614/keepachangelog-2.0.0-py3-none-any.whl", hash = "sha256:0b2259fd73b55f2b99e61035ff83f3d86d0f2764841a7e2de86e31e675e42fee", size = 13349, upload-time = "2024-06-14T12:56:26.884Z" }, +] + [[package]] name = "markdown" version = "3.10"