diff --git a/.github/RELEASE_NOTES.template.md b/.github/RELEASE_NOTES.template.md index b99e3a30..30292845 100644 --- a/.github/RELEASE_NOTES.template.md +++ b/.github/RELEASE_NOTES.template.md @@ -13,7 +13,7 @@ All upgrading should be done via the migration script or regenerating the templates. ```bash -curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/v0.12/cookiecutter/migrate.py | python3 +curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python//cookiecutter/migrate.py | python3 ``` But you might still need to adapt your code: diff --git a/.github/cookiecutter-migrate.template.py b/.github/cookiecutter-migrate.template.py index c0e1dba0..8e560bc8 100644 --- a/.github/cookiecutter-migrate.template.py +++ b/.github/cookiecutter-migrate.template.py @@ -20,20 +20,45 @@ And remember to follow any manual instructions for each run. """ # noqa: E501 +# pylint: disable=too-many-lines, too-many-locals, too-many-branches + import hashlib +import json import os import subprocess +import sys import tempfile from pathlib import Path -from typing import SupportsIndex +from typing import Any, SupportsIndex + +_manual_steps: list[str] = [] # pylint: disable=invalid-name def main() -> None: """Run the migration steps.""" # Add a separation line like this one after each migration step. print("=" * 72) - print("Migration script finished. Remember to follow any manual instructions.") - print("=" * 72) + print() + + if _manual_steps: + print( + "\033[5;33m⚠️⚠️⚠️\033[0;33m Remember to check the manual steps: \033[5;33m⚠️⚠️⚠️\033[0m" + ) + for n, step in enumerate(_manual_steps, start=1): + print(f"\033[5;33m⚠️⚠️⚠️ \033[0;33m{n}. {step}\033[0m") + print() + + print( + "\033[5;31m❌\033[0;31m Migration script finished but requires manual " + "intervention \033[5;31m❌\033[0m" + ) + print() + + sys.exit(len(_manual_steps)) + + print("\033[0;32m ✅ Migration script finished successfully ✅\033[0m") + print() + return project_type def apply_patch(patch_content: str) -> None: @@ -41,66 +66,80 @@ def apply_patch(patch_content: str) -> None: subprocess.run(["patch", "-p1"], input=patch_content.encode(), check=True) -def replace_file_contents_atomically( # noqa; DOC501 - filepath: str | Path, - old: str, - new: str, - count: SupportsIndex = -1, - *, - content: str | None = None, +def replace_file_atomically( # noqa; DOC501, DOC503 + filepath: str | Path, new_content: str ) -> None: - """Replace a file atomically with new content. + """Replace a file atomically with the given content. + + The replacement is done atomically by writing to a temporary file in the + same directory and then moving it to the target location. Args: filepath: The path to the file to replace. - old: The string to replace. - new: The string to replace it with. - count: The maximum number of occurrences to replace. If negative, all occurrences are - replaced. - content: The content to replace. If not provided, the file is read from disk. - - The replacement is done atomically by writing to a temporary file and - then moving it to the target location. + new_content: The content to write to the file. """ if isinstance(filepath, str): filepath = Path(filepath) - if content is None: - content = filepath.read_text(encoding="utf-8") - - content = content.replace(old, new, count) - - # Create temporary file in the same directory to ensure atomic move tmp_dir = filepath.parent + tmp_dir.mkdir(parents=True, exist_ok=True) # pylint: disable-next=consider-using-with tmp = tempfile.NamedTemporaryFile(mode="w", dir=tmp_dir, delete=False) try: - # Copy original file permissions - st = os.stat(filepath) - - # Write the new content - tmp.write(content) + st = None + try: + st = os.stat(filepath) + except FileNotFoundError: + st = None - # Ensure all data is written to disk + tmp.write(new_content) tmp.flush() os.fsync(tmp.fileno()) tmp.close() - # Copy original file permissions to the new file - os.chmod(tmp.name, st.st_mode) + if st is not None: + os.chmod(tmp.name, st.st_mode) - # Perform atomic replace - os.rename(tmp.name, filepath) + os.replace(tmp.name, filepath) except BaseException: - # Clean up the temporary file in case of errors tmp.close() os.unlink(tmp.name) raise +def replace_file_contents_atomically( # noqa; DOC501 + filepath: str | Path, + old: str, + new: str, + count: SupportsIndex = -1, + *, + content: str | None = None, +) -> None: + """Replace a file atomically with new content. + + The replacement is done atomically by writing to a temporary file and + then moving it to the target location. + + Args: + filepath: The path to the file to replace. + old: The string to replace. + new: The string to replace it with. + count: The maximum number of occurrences to replace. If negative, all occurrences are + replaced. + content: The content to replace. If not provided, the file is read from disk. + """ + if isinstance(filepath, str): + filepath = Path(filepath) + + if content is None: + content = filepath.read_text(encoding="utf-8") + + replace_file_atomically(filepath, content.replace(old, new, count)) + + def calculate_file_sha256_skip_lines(filepath: Path, skip_lines: int) -> str | None: """Calculate SHA256 of file contents excluding the first N lines. @@ -121,8 +160,157 @@ def calculate_file_sha256_skip_lines(filepath: Path, skip_lines: int) -> str | N return hashlib.sha256(remaining_content.encode()).hexdigest() +def find_ruleset(name: str) -> dict[str, Any] | None: + """Find a repository ruleset by name using the GitHub API. + + Args: + name: The name of the ruleset to search for. + + Returns: + The ruleset summary dict (id, name, …) if found, or ``None`` if not + found or if the API call failed (a diagnostic is printed in the latter + case). + """ + try: + stdout = subprocess.check_output( + ["gh", "api", "repos/:owner/:repo/rulesets"], + text=True, + stderr=subprocess.PIPE, + ) + except FileNotFoundError: + print(" gh CLI not found; cannot query rulesets via the GitHub API.") + return None + except subprocess.CalledProcessError as exc: + print(f" Failed to list rulesets: {exc.stderr.strip()}") + return None + + rulesets: list[dict[str, Any]] = json.loads(stdout) + return next((r for r in rulesets if r.get("name") == name), None) + + +def get_ruleset(ruleset: str | int) -> dict[str, Any] | None: + """Fetch the full details of a repository ruleset by name or ID. + + Args: + ruleset: The ruleset name (``str``) or numeric ruleset ID (``int``). + + Returns: + The full ruleset dict, or ``None`` if the ruleset could not be found + or the API call failed (a diagnostic is printed). + """ + ruleset_id = ruleset + if isinstance(ruleset, str): + entry = find_ruleset(ruleset) + if entry is None: + return None + ruleset_id = entry["id"] + + try: + stdout = subprocess.check_output( + ["gh", "api", f"repos/:owner/:repo/rulesets/{ruleset_id}"], + text=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as exc: + print(f" Failed to fetch ruleset {ruleset_id}: {exc.stderr.strip()}") + return None + + return json.loads(stdout) # type: ignore[no-any-return] + + +def update_ruleset(ruleset_id: int, config: dict[str, Any]) -> bool: + """Update a repository ruleset via the GitHub API. + + Only ``name``, ``target``, ``enforcement``, ``conditions``, ``rules``, + and ``bypass_actors`` are sent (explicit allowlist to avoid sending + read-only fields back to the API). + + Args: + ruleset_id: The numeric ruleset ID to update. + config: The full ruleset dict (as returned by :func:`get_ruleset`) + with the desired changes already applied in-memory. + + Returns: + ``True`` on success, ``False`` if the API call failed (a diagnostic + is printed). + """ + payload: dict[str, Any] = { + "name": config["name"], + "target": config["target"], + "enforcement": config["enforcement"], + "conditions": config["conditions"], + "rules": config["rules"], + } + if "bypass_actors" in config: + payload["bypass_actors"] = config["bypass_actors"] + + try: + subprocess.check_output( + [ + "gh", + "api", + "-X", + "PUT", + f"repos/:owner/:repo/rulesets/{ruleset_id}", + "--input", + "-", + ], + input=json.dumps(payload), + text=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as exc: + print(f" Failed to update ruleset {ruleset_id}: {exc.stderr.strip()}") + return False + + return True + + +def get_ruleset_settings_url() -> str | None: + """Return the URL to the repository's ruleset settings page. + + Returns: + The URL as a string, or ``None`` if it could not be determined. + """ + try: + stdout = subprocess.check_output( + ["gh", "repo", "view", "--json", "owner,name"], + text=True, + stderr=subprocess.PIPE, + ) + info: dict[str, Any] = json.loads(stdout) + org = info["owner"]["login"] + repo = info["name"] + return f"https://github.com/{org}/{repo}/settings/rules" + except (subprocess.CalledProcessError, KeyError, json.JSONDecodeError): + return None + + +def read_cookiecutter_str_var(name: str) -> str | None: + """Read a cookiecutter variable from the replay file.""" + replay_path = Path(".cookiecutter-replay.json") + if not replay_path.exists(): + return None + + try: + data = json.loads(replay_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + + cookiecutter_data = data.get("cookiecutter") + if not isinstance(cookiecutter_data, dict): + return None + + value = cookiecutter_data.get(name) + if not isinstance(value, str): + return None + + return value + + def manual_step(message: str) -> None: """Print a manual step message in yellow.""" + _manual_steps.append(message) print(f"\033[0;33m>>> {message}\033[0m") diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a1c0dac6..aacbb962 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -38,11 +38,13 @@ updates: - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/.github/workflows/auto-dependabot.yaml b/.github/workflows/auto-dependabot.yaml index ec39a854..60309295 100644 --- a/.github/workflows/auto-dependabot.yaml +++ b/.github/workflows/auto-dependabot.yaml @@ -1,21 +1,37 @@ name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: + name: Auto-merge Dependabot PR if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' diff --git a/.github/workflows/ci-pr.yaml b/.github/workflows/ci-pr.yaml index 465d384e..059236ff 100644 --- a/.github/workflows/ci-pr.yaml +++ b/.github/workflows/ci-pr.yaml @@ -35,7 +35,7 @@ jobs: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.1 + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: .[dev-mkdocs] @@ -48,7 +48,7 @@ jobs: mike set-default $MIKE_VERSION - name: Upload site - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: docs-site path: site/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bacf5b9b..3ab9c6d4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,11 +28,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -41,7 +39,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -60,7 +58,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -84,7 +82,7 @@ jobs: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.1 + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: build @@ -93,7 +91,7 @@ jobs: run: python -m build - name: Upload distribution files - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: dist-packages path: dist/ @@ -105,15 +103,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -123,7 +119,7 @@ jobs: run: env - name: Download package - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: name: dist-packages path: dist @@ -143,7 +139,7 @@ jobs: > pyproject.toml - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.1 + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.2 with: python-version: ${{ matrix.python }} dependencies: dist/*.whl @@ -161,7 +157,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -182,7 +178,7 @@ jobs: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.1 + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: .[dev-mkdocs] @@ -195,7 +191,7 @@ jobs: mike set-default $MIKE_VERSION - name: Upload site - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: docs-site path: site/ @@ -218,7 +214,7 @@ jobs: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.1 + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: .[dev-mkdocs] @@ -276,10 +272,10 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: name: dist-packages path: dist @@ -325,7 +321,7 @@ jobs: id-token: write steps: - name: Download distribution files - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: name: dist-packages path: dist diff --git a/.github/workflows/dco-merge-queue.yml b/.github/workflows/dco-merge-queue.yml index fb1cd90c..d9597ad0 100644 --- a/.github/workflows/dco-merge-queue.yml +++ b/.github/workflows/dco-merge-queue.yml @@ -5,7 +5,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 8d02c139..c327e7f2 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/.github/workflows/release-notes-check.yml b/.github/workflows/release-notes-check.yml index e97886b6..0f1250af 100644 --- a/.github/workflows/release-notes-check.yml +++ b/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..5b566804 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,99 @@ +# AGENTS.md - AI Coding Agent Instructions + +Condensed instructions for AI agents. See [README.md](README.md) and [CONTRIBUTING.md](CONTRIBUTING.md) for full details. + +## Project Overview + +Python package (3.11+) providing repository configuration and scaffolding for Frequenz projects: cookiecutter templates (actor, api, app, lib, model), nox sessions, protobuf/gRPC utilities, MkDocs and pytest utilities. + +**Package:** `src/frequenz/repo/config/` (namespace package, `setuptools_scm` versioning) + +## Quick Reference + +```sh +# Setup +pip install -e .[dev] # All dev dependencies +pip install -e .[dev-noxfile] # Just noxfile deps + +# Test +nox -s pytest_max # Full test suite +nox -R -s pytest_max # Reuse venv (faster) +pytest tests/ # Direct pytest +UPDATE_GOLDEN=1 pytest tests/integration/test_cookiecutter_generation.py::test_golden + +# Lint (run before committing) +nox # All checks +nox -s formatting # black + isort +nox -s mypy # Type checking +nox -R -s mypy # Reuse venv +``` + +**Markers:** `integration`, `cookiecutter` + +## Project Structure + +``` +src/frequenz/repo/config/ # Main package (_core.py, nox/, mkdocs/, pytest/, setuptools/, cli/) +tests/ # Unit tests, tests/integration/ for integration tests +tests_golden/ # Golden test fixtures +cookiecutter/ # Cookiecutter templates +``` + +## Code Patterns + +```python +# File header (required) +# License: MIT +# Copyright © 2023 Frequenz Energy-as-a-Service GmbH +"""Module docstring.""" + +import dataclasses +from typing import Self, assert_never + +# Dataclasses: always kw_only + frozen if intended to be immutable +@dataclasses.dataclass(kw_only=True, frozen=True) +class Config: + """Docstring.""" + opts: CommandsOptions = dataclasses.field(default_factory=CommandsOptions) + +# Functions: strict typing, Google docstrings +def func(session: _nox.Session, /, *, flag: bool = True) -> list[str]: + """Brief description. + + Args: + session: The nox session. + flag: Optional flag. + + Returns: + List of strings. + """ +``` + +| Type | Convention | Example | +|------|------------|---------| +| Files/functions | snake_case | `api_pages.py`, `find_dirs()` | +| Classes | PascalCase | `RepositoryType` | +| Constants | UPPER_SNAKE | `UPDATE_GOLDEN` | +| Private | `_` prefix | `_impl()` | + +**Formatting:** black formatting using 4 spaces (Python), 2 spaces (YAML/TOML/JSON), 88 line length, 100 max, double quotes + +When changing files that are regularly reset (like `RELEASE_NOTES.md` or `cookiecutter/migrate.py`), consult the git history to match the style used in the past. + +## Commit Messages +- Use imperative mood: "Add", "Fix", "Update" +- Use a 50-character limit for the subject line (can go a bit over if necessary), and wrap body at 80 characters +- Include a brief description of the change in the body if needed, focus on the what and why, and avoid describing how, especially if it is obvious from the code/diff +- Separate changes into logical commits +- Consult the recent git history, specifically of the files being committed, to match style and conventions in commit messages used previously. + +## Cookiecutter Template Changes + +See [CONTRIBUTING.md](CONTRIBUTING.md#modifying-cookiecutter-templates) for the full workflow. Summary: + +1. Edit templates in `cookiecutter/{{cookiecutter.github_repo_name}}/` +2. Update golden files: `UPDATE_GOLDEN=1 pytest tests/integration/test_cookiecutter_generation.py::test_golden` +3. Write migration in `cookiecutter/migrate.py` (idempotent; use `manual_step()` for non-automatable changes) +4. Validate: `python3 cookiecutter/migrate.py && git diff .github/` +5. Update `RELEASE_NOTES.md` +6. Commit separately: templates first, then this repo's migration result diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a3616cd..07e7b61f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,7 +53,7 @@ Or you can use `nox`: nox -R -s pytest -- test/test_*.py ``` -The same appliest to `pylint` or `mypy` for example: +The same applies to `pylint` or `mypy` for example: ```sh nox -R -s pylint -- test/test_*.py @@ -78,27 +78,94 @@ Failures in the golden tests could indicate two things: 2. The generated files don't match the golden files because an intended change was introduced. In this case, the golden files need to be updated. -In the latter case, manually updating files is complicated and error-prone, so -a simpler (though hacky) way is provided. +In the latter case, manually updating files is complicated and error-prone, +please consult [Update the golden test +fixtures](#2-update-the-golden-test-fixtures) below for the recommended way to +update the golden files. -To update the golden files, simply run `pytest` for the tests using golden -files setting the environment variable `UPDATE_GOLDEN` to `1`: +### Modifying Cookiecutter Templates + +When you need to make changes to the cookiecutter templates (located in +`cookiecutter/{{cookiecutter.github_repo_name}}/`), there's a specific workflow +to follow. The templates support different project types (actor, api, app, lib, +model) using Jinja2 conditional sections. + +#### 1. Make your template changes + +Edit the files in the `cookiecutter/` directory. Keep in mind that some files +have conditional sections for different project types. + +#### 2. Update the golden test fixtures + +After modifying templates, you need to regenerate the golden files so the tests +pass: ```sh UPDATE_GOLDEN=1 pytest tests/integration/test_cookiecutter_generation.py::test_golden ``` -This will replace the existing golden files (stored in `tests_golden/`) with -the newly generated files. +If you renamed or removed files, it's safest to wipe the golden directory first: -Note that if you rename, or remove golden files, you should also manually -remove the files that were affected. An easy way to make sure there are no old -unused golden files left is to just wipe the whole `tests_golden/` directory -before running `pytest` to generate the new ones. +```sh +rm -rf tests_golden/ +UPDATE_GOLDEN=1 pytest tests/integration/test_cookiecutter_generation.py::test_golden +``` **Please ensure that all introduced changes are intended before updating the golden files.** +#### 3. Write a migration script + +Existing projects using these templates need a way to migrate to the new +version. Update `cookiecutter/migrate.py` to handle this migration. The script +is reset to a blank template (from `.github/cookiecutter-migrate.template.py`) +after each release. + +A few guidelines for migration scripts: + +- Make them idempotent (safe to run multiple times) +- Print clear messages about what's being done +- Use the helper functions: `replace_file_contents_atomically()` for simple + replacements, `apply_patch()` for more complex changes + +For changes that can't be automated (like GitHub repository settings that +require admin permissions, or changes that would make the script overly +complex), use the `manual_step()` function to clearly communicate to users +what they need to do manually: + +```python +manual_step( + "Update branch protection rules in GitHub settings:\n" + " Settings > Branches > Enable 'Require status checks'" +) +``` + +#### 4. Validate with this repository + +Run the migration script on this repository itself to make sure it works: + +```sh +python3 cookiecutter/migrate.py +git diff .github/workflows/ # Check the changes look correct +``` + +Note that some changes (e.g., API-specific features like protolint) won't be +testable on this repository since it's a `lib` type project. + +#### 5. Update the release notes + +Add an entry to `RELEASE_NOTES.md` describing what changed in the templates. + +#### 6. Commit separately + +Create two separate commits to keep the history clean: + +1. First commit: the template changes, migration script, golden tests, and + release notes +2. Second commit: the changes to this repository from running the migration + +This separation makes it easier to review and understand what changed. + ### Building the documentation To build the documentation, first install the dependencies (if you didn't diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 19846555..d527758b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,7 @@ ## Summary -This release adds a new workflow for Dependabot auto-merge and updates mkdocstrings to v2. +This is a maintenance, template-only, bugfix release. ## Upgrading @@ -11,24 +11,14 @@ This release adds a new workflow for Dependabot auto-merge and updates mkdocstri All upgrading should be done via the migration script or regenerating the templates. ```bash -curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/v0.14/cookiecutter/migrate.py | python3 +curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/v0.16.0/cookiecutter/migrate.py | python3 ``` -But you might still need to adapt your code, just have a look at the script output for further instructions. - -## New Features - -* `mkdocsstrings-python` v2 is now supported. - -### Cookiecutter template - -- Dependencies have been updated. -- New warning ignores for protobuf gencode versions in pytest. -- Added Dependabot auto-merge workflow using `frequenz-floss/dependabot-auto-approve` action. - ## Bug Fixes ### Cookiecutter template -- mkdocstrings: Move `paths` key to the right section in `mkdocs.yml`. -- Fix invalid YAML syntax in Dependabot workflow template. +- Added a migration step for api repositories to fix `mkdocs.yml` when the previous `mkdocstrings-python` v2 migration moved only `paths: ["src"]` under `handlers.python.options` but not `paths: ["py"]`. +- Fixed runners for jobs that require Docker and where wrongly converted to `ubuntu-slim` in v0.15.0, changing them back to `ubuntu-24.04` to avoid Docker-related failures. The template and the migration script were both updated to reflect this change. +- Updated the repo-config migration workflow template and migration script so existing repositories also add the `merge_group` trigger and skip the job unless the event is `pull_request_target`, allowing the workflow to be used as a required merge-queue check. +- Added a migration step to remove the copilot review request from the Protect version branch protection rules. This was also done by v0.15.0 in theory, but the migration step was wrong and didn't update it properly. diff --git a/cookiecutter/migrate.py b/cookiecutter/migrate.py index 426e5e37..9308d5e3 100644 --- a/cookiecutter/migrate.py +++ b/cookiecutter/migrate.py @@ -20,250 +20,352 @@ And remember to follow any manual instructions for each run. """ # noqa: E501 +# pylint: disable=too-many-lines, too-many-locals, too-many-branches + import hashlib import json import os import subprocess +import sys import tempfile from pathlib import Path from typing import Any, SupportsIndex +_manual_steps: list[str] = [] # pylint: disable=invalid-name + def main() -> None: """Run the migration steps.""" # Add a separation line like this one after each migration step. print("=" * 72) - print("Creating Dependabot auto-merge workflow...") - create_dependabot_auto_merge_workflow() + print("Fixing repo-config migration merge queue trigger...") + migrate_repo_config_migration_merge_group_trigger() print("=" * 72) - print("Disabling CODEOWNERS review requirement in GitHub ruleset...") - disable_codeowners_review_requirement() + print("Fixing mkdocstrings-python v2 paths for api repos...") + migrate_api_mkdocs_mkdocstrings_paths() print("=" * 72) - print("Updating the mkdocs.yml for mkdocstrings-python v2 compatibility...") - update_mkdocs_yml_mkdocstrings_python_v2() + print("Migrating protolint and publish-to-pypi runners to ubuntu-24.04...") + migrate_docker_based_runners() print("=" * 72) - print("Migration script finished. Remember to follow any manual instructions.") + print("Updating 'Protect version branches' GitHub ruleset...") + migrate_protect_version_branches_ruleset() print("=" * 72) + print() + if _manual_steps: + print( + "\033[5;33m⚠️⚠️⚠️\033[0;33m Remember to check the manual steps: \033[5;33m⚠️⚠️⚠️\033[0m" + ) + for n, step in enumerate(_manual_steps, start=1): + print(f"\033[5;33m⚠️⚠️⚠️ \033[0;33m{n}. {step}\033[0m") + print() -def update_mkdocs_yml_mkdocstrings_python_v2() -> None: - """Rename 'inventories' imports to 'inventory'.""" - replace_file_contents_atomically( - filepath=Path("mkdocs.yml"), - old=" import:", - new=" inventories:", - ) - replace_file_contents_atomically( - filepath=Path("mkdocs.yml"), - old="""\ - options: - paths: ["src"]""", - new="""\ - paths: ["src"] - options:""", - ) + print( + "\033[5;31m❌\033[0;31m Migration script finished but requires manual " + "intervention \033[5;31m❌\033[0m" + ) + print() + sys.exit(len(_manual_steps)) -def create_dependabot_auto_merge_workflow() -> None: - """Create the Dependabot auto-merge workflow file.""" - workflow_dir = Path(".github") / "workflows" - workflow_dir.mkdir(parents=True, exist_ok=True) - - workflow_content = """\ -name: Auto-merge Dependabot PR - -on: - pull_request: - -permissions: - contents: write - pull-requests: write - -jobs: - auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest - steps: - - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - dependency-type: 'all' - auto-merge: 'true' - merge-method: 'merge' - add-label: 'tool:auto-merged' -""" # noqa: E501 + print("\033[0;32m ✅ Migration script finished successfully ✅\033[0m") + print() - workflow_file = workflow_dir / "auto-dependabot.yaml" - workflow_file.write_text(workflow_content, encoding="utf-8") - print(f"Created/Updated Dependabot auto-merge workflow at {workflow_file}") +def migrate_api_mkdocs_mkdocstrings_paths() -> None: + """Fix the mkdocstrings paths migration for api repositories.""" + project_type = read_cookiecutter_str_var("type") + if project_type is None: + manual_step( + "Unable to detect the cookiecutter project type from " + ".cookiecutter-replay.json; if this is an api project and " + '`mkdocs.yml` still has `paths: ["py"]` nested under ' + "`handlers.python.options`, move it out of `options`." + ) + return -def get_default_branch() -> str | None: - """Get the default branch name from GitHub. + if project_type != "api": + print(" Skipping mkdocs.yml (not an api project)") + return - Returns: - The default branch name, or None if it cannot be determined. - """ - try: - result = subprocess.run( - ["gh", "api", "repos/:owner/:repo", "--jq", ".default_branch"], - capture_output=True, - text=True, - check=True, + filepath = Path("mkdocs.yml") + if not filepath.exists(): + manual_step( + "Unable to find mkdocs.yml; if this project uses mkdocs, " + 'make sure the `paths: ["py"]` config is under ' + "`handlers.python`, not `handlers.python.options`." ) - default_branch = result.stdout.strip() - print(f"Default branch: {default_branch}") - return default_branch - except subprocess.CalledProcessError as e: - print(f"Failed to get default branch: {e}") - return None - + return -def find_version_branch_ruleset() -> dict[str, Any] | None: - """Find the 'Protect version branches' ruleset. + old = ' options:\n paths: ["py"]' + new = ' paths: ["py"]\n options:' + current_template = ( + ' handlers:\n paths: ["py"]\n python:\n options:' + ) + content = filepath.read_text(encoding="utf-8") - Returns: - The ruleset configuration, or None if not found. - """ - try: - result = subprocess.run( - ["gh", "api", "repos/:owner/:repo/rulesets"], - capture_output=True, - text=True, - check=True, - ) - rulesets = json.loads(result.stdout) + if old in content: + replace_file_contents_atomically(filepath, old, new, count=1) + print(f" Updated {filepath}: moved mkdocstrings api paths out of options") + return - for ruleset in rulesets: - if ruleset.get("name") == "Protect version branches": - return ruleset # type: ignore[no-any-return] - return None - except subprocess.CalledProcessError as e: - print(f"Failed to fetch rulesets: {e}") - return None + if new in content or current_template in content: + print(f" Skipped {filepath}: mkdocstrings api paths already updated") + return + manual_step( + f"Could not find the api mkdocstrings path pattern in {filepath}. " + 'If `paths: ["py"]` is still nested under `handlers.python.options`, ' + "move it out of `options` according to the latest template." + ) -def update_ruleset(ruleset_id: int, ruleset_config: dict[str, Any]) -> bool: - """Update a GitHub ruleset configuration. - Args: - ruleset_id: The ID of the ruleset to update. - ruleset_config: The updated ruleset configuration. +def migrate_docker_based_runners() -> None: + """Migrate Docker-based jobs to use ubuntu-24.04 runners. - Returns: - True if the update was successful, False otherwise. + The ``protolint`` and ``publish-to-pypi`` jobs need Docker, which is not + available on ``ubuntu-slim``. They should therefore run on + ``ubuntu-24.04`` instead. """ - update_payload = { - "name": ruleset_config["name"], - "target": ruleset_config["target"], - "enforcement": ruleset_config["enforcement"], - "conditions": ruleset_config["conditions"], - "rules": ruleset_config["rules"], + workflows_dir = Path(".github") / "workflows" + protolint_new = ( + " protolint:\n" + " name: Check proto files with protolint\n" + " runs-on: ubuntu-24.04" + ) + publish_to_pypi_new = ( + ' needs: ["create-github-release"]\n runs-on: ubuntu-24.04' + ) + migrations: dict[str, list[dict[str, Any]]] = {} + + protolint_rule = { + "job": "protolint", + "required_for": "api repos", + "job_marker": " protolint:\n", + "old": [ + ( + " protolint:\n" + " name: Check proto files with protolint\n" + " runs-on: ubuntu-slim" + ), + ( + " protolint:\n" + " name: Check proto files with protolint\n" + " runs-on: ubuntu-latest" + ), + ], + "new": protolint_new, } + project_type = read_cookiecutter_str_var("type") + if project_type is None: + manual_step( + "Unable to detect the cookiecutter project type from " + ".cookiecutter-replay.json; cannot determine whether the protolint " + "runner migration applies." + ) + elif project_type == "api": + migrations.setdefault("ci-pr.yaml", []).append(protolint_rule) + migrations.setdefault("ci.yaml", []).append(protolint_rule) + else: + print(" Skipping protolint runner migration (not an api project)") - if "bypass_actors" in ruleset_config: - update_payload["bypass_actors"] = ruleset_config["bypass_actors"] + github_org = read_cookiecutter_str_var("github_org") + if github_org is None: + manual_step( + "Unable to detect the cookiecutter GitHub organization from " + ".cookiecutter-replay.json; cannot determine whether the " + "publish-to-pypi runner migration applies." + ) + elif github_org == "frequenz-floss": + migrations.setdefault("ci.yaml", []).append( + { + "job": "publish-to-pypi", + "required_for": "frequenz-floss repos", + "job_marker": " publish-to-pypi:\n", + "old": [ + (' needs: ["create-github-release"]\n runs-on: ubuntu-slim'), + ( + ' needs: ["create-github-release"]\n' + " runs-on: ubuntu-latest" + ), + ], + "new": publish_to_pypi_new, + } + ) + else: + print(" Skipping publish-to-pypi runner migration (not a frequenz-floss repo)") + + for filename, rules in migrations.items(): + filepath = workflows_dir / filename + if not filepath.exists(): + for rule in rules: + manual_step( + f" Expected to find {filepath} for job {rule['job']} in " + f"{rule['required_for']}. Please add or update that job to use " + "`runs-on: ubuntu-24.04`." + ) + continue + + for rule in rules: + job = rule["job"] + required_for = rule["required_for"] + job_marker = rule["job_marker"] + new = rule["new"] + content = filepath.read_text(encoding="utf-8") + + if job_marker not in content: + manual_step( + f" Expected to find job {job} in {filepath} for " + f"{required_for}. Please update it to use " + "`runs-on: ubuntu-24.04`." + ) + continue + + if new in content: + print(f" Skipped {filepath}: runner already up to date for job {job}") + continue + + for old in rule["old"]: + if old in content: + replace_file_contents_atomically( + filepath, old, new, content=content + ) + print(f" Updated {filepath}: migrated runner for job {job}") + break + else: + manual_step( + f" Pattern not found in {filepath}: please switch the runner " + f"for job {job} to `runs-on: ubuntu-24.04`." + ) + + +def migrate_repo_config_migration_merge_group_trigger() -> None: + """Trigger repo-config migration in the merge queue.""" + filepath = Path(".github/workflows/repo-config-migration.yaml") + if not filepath.exists(): + manual_step( + "Unable to find .github/workflows/repo-config-migration.yaml; if this " + "project uses the repo-config migration workflow, update it to trigger " + "on `merge_group` and skip the job unless the event is " + "`pull_request_target`." + ) + return - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump(update_payload, f, indent=2) - temp_file = f.name + content = filepath.read_text(encoding="utf-8") + old_on = ( + "on:\n" + " pull_request_target:\n" + " types: [opened, synchronize, reopened, labeled, unlabeled]\n" + ) + new_on = ( + "on:\n" + " merge_group: # To allow using this as a required check for merging\n" + " pull_request_target:\n" + " types: [opened, synchronize, reopened, labeled, unlabeled]\n" + ) + old_if = ( + " if: contains(github.event.pull_request.title, 'the repo-config group')" + ) + new_if = ( + " # Skip if it was triggered by the merge queue. We only need the workflow to\n" + ' # be executed to meet the "Required check" condition for merging, but we\n' + " # don't need to actually run the job, having the job present as Skipped is\n" + " # enough.\n" + " if: |\n" + " github.event_name == 'pull_request_target' &&\n" + " contains(github.event.pull_request.title, 'the repo-config group')" + ) - try: - subprocess.run( - [ - "gh", - "api", - "-X", - "PUT", - f"repos/:owner/:repo/rulesets/{ruleset_id}", - "--input", - temp_file, - ], - capture_output=True, - check=True, - ) - return True - except subprocess.CalledProcessError as e: - print(f"Error updating ruleset: {e}") - return False - finally: - os.unlink(temp_file) + updated = content + if old_on in updated: + updated = updated.replace(old_on, new_on, 1) + if old_if in updated: + updated = updated.replace(old_if, new_if, 1) -def disable_codeowners_review_requirement() -> None: - """Disable CODEOWNERS review requirement in GitHub repository ruleset.""" - # Get repository info - try: - result = subprocess.run( - ["gh", "repo", "view", "--json", "owner,name"], - capture_output=True, - text=True, - check=True, - ) - repo_info = json.loads(result.stdout) - org = repo_info["owner"]["login"] - repo = repo_info["name"] - ruleset_url = f"https://github.com/{org}/{repo}/settings/rules" - except subprocess.CalledProcessError: - ruleset_url = "GitHub repository settings > Rules" - - if get_default_branch() is None: - manual_step( - "Failed to get default branch. " - "Please manually disable the CODEOWNERS review requirement in the " - f"'Protect version branches' ruleset at: {ruleset_url}" + if updated != content: + replace_file_atomically(filepath, updated) + print( + " Updated .github/workflows/repo-config-migration.yaml: added " + "merge_group trigger" ) return - version_branch_ruleset = find_version_branch_ruleset() - if not version_branch_ruleset: - manual_step( - "'Protect version branches' ruleset not found. " - "Please manually disable the CODEOWNERS review requirement at: " - f"{ruleset_url}" + if new_on in content and new_if in content: + print( + " Skipped .github/workflows/repo-config-migration.yaml: merge queue " + "trigger already configured" ) return - ruleset_id = version_branch_ruleset["id"] - print(f"Found ruleset ID: {ruleset_id}") + manual_step( + "Could not find the expected repo-config migration workflow pattern in " + ".github/workflows/repo-config-migration.yaml. If this repository uses " + "that workflow, add the `merge_group` trigger and make the job run only " + "for `pull_request_target` events according to the latest template." + ) - try: - result = subprocess.run( - ["gh", "api", f"repos/:owner/:repo/rulesets/{ruleset_id}"], - capture_output=True, - text=True, - check=True, - ) - ruleset_config = json.loads(result.stdout) - except subprocess.CalledProcessError as e: + +def migrate_protect_version_branches_ruleset() -> None: + """Update the 'Protect version branches' GitHub ruleset. + + Uses the GitHub API (via ``gh`` CLI) to check whether the + 'Protect version branches' ruleset on the current repository is aligned + with the current template. Recent template changes include: + + * Removing the ``copilot_code_review`` rule. + + If the ruleset is already aligned, prints an informational message. + If it needs updating, applies the changes via the API without removing + any existing required status checks. + If the ruleset is not found at all, issues a manual-step message that + points the user to the docs. + """ + rule_name = "Protect version branches" + docs_url = ( + "https://frequenz-floss.github.io/frequenz-repo-config-python/" + "user-guide/start-a-new-project/configure-github/#rulesets" + ) + + # Build a link to the repo's ruleset settings for manual-step messages. + ruleset_url = get_ruleset_settings_url() or docs_url + + # ── Fetch ruleset details ──────────────────────────────────────── + ruleset = get_ruleset(rule_name) + if ruleset is None: manual_step( - f"Failed to fetch ruleset configuration: {e}. " - "This action requires admin permissions. " - f"Please manually disable the CODEOWNERS review requirement at: {ruleset_url}" + f"The '{rule_name}' GitHub ruleset was not found (or the gh CLI " + "is not available / the API call failed). " + "Please check whether it should exist for this repository. " + f"If it should, import it following the instructions at: {docs_url}" ) return - updated = False - for rule in ruleset_config.get("rules", []): - if rule.get("type") == "pull_request": - if rule.get("parameters", {}).get("require_code_owner_review"): - rule["parameters"]["require_code_owner_review"] = False - updated = True - break + # ── Detect and apply changes in-memory ─────────────────────────────── + changes: list[str] = [] + updated_rules = [] + + for rule in ruleset.get("rules", []): + if rule.get("type") == "copilot_code_review": + changes.append("remove copilot_code_review") + continue + updated_rules.append(rule) - if not updated: - print("CODEOWNERS review requirement already disabled.") + if not changes: + print(f" Ruleset '{rule_name}' is already up to date") return - if update_ruleset(ruleset_id, ruleset_config): - print("Successfully disabled CODEOWNERS review requirement in GitHub ruleset.") - else: + # ── Push the update ─────────────────────────────────────────────────── + ruleset["rules"] = updated_rules + if not update_ruleset(ruleset["id"], ruleset): manual_step( - "Failed to update GitHub ruleset. This action requires admin permissions. " - "Please manually disable the CODEOWNERS review requirement in the " - f"'Protect version branches' ruleset at: {ruleset_url}" + f"Failed to update the '{rule_name}' ruleset via the GitHub API. " + f"Please apply the following changes manually at {ruleset_url}: " + + "; ".join(changes) ) + return + + print(f" Updated ruleset '{rule_name}': " + ", ".join(changes)) def apply_patch(patch_content: str) -> None: @@ -271,66 +373,80 @@ def apply_patch(patch_content: str) -> None: subprocess.run(["patch", "-p1"], input=patch_content.encode(), check=True) -def replace_file_contents_atomically( # noqa; DOC501 - filepath: str | Path, - old: str, - new: str, - count: SupportsIndex = -1, - *, - content: str | None = None, +def replace_file_atomically( # noqa; DOC501, DOC503 + filepath: str | Path, new_content: str ) -> None: - """Replace a file atomically with new content. + """Replace a file atomically with the given content. + + The replacement is done atomically by writing to a temporary file in the + same directory and then moving it to the target location. Args: filepath: The path to the file to replace. - old: The string to replace. - new: The string to replace it with. - count: The maximum number of occurrences to replace. If negative, all occurrences are - replaced. - content: The content to replace. If not provided, the file is read from disk. - - The replacement is done atomically by writing to a temporary file and - then moving it to the target location. + new_content: The content to write to the file. """ if isinstance(filepath, str): filepath = Path(filepath) - if content is None: - content = filepath.read_text(encoding="utf-8") - - content = content.replace(old, new, count) - - # Create temporary file in the same directory to ensure atomic move tmp_dir = filepath.parent + tmp_dir.mkdir(parents=True, exist_ok=True) # pylint: disable-next=consider-using-with tmp = tempfile.NamedTemporaryFile(mode="w", dir=tmp_dir, delete=False) try: - # Copy original file permissions - st = os.stat(filepath) + st = None + try: + st = os.stat(filepath) + except FileNotFoundError: + st = None - # Write the new content - tmp.write(content) - - # Ensure all data is written to disk + tmp.write(new_content) tmp.flush() os.fsync(tmp.fileno()) tmp.close() - # Copy original file permissions to the new file - os.chmod(tmp.name, st.st_mode) + if st is not None: + os.chmod(tmp.name, st.st_mode) - # Perform atomic replace - os.rename(tmp.name, filepath) + os.replace(tmp.name, filepath) except BaseException: - # Clean up the temporary file in case of errors tmp.close() os.unlink(tmp.name) raise +def replace_file_contents_atomically( # noqa; DOC501 + filepath: str | Path, + old: str, + new: str, + count: SupportsIndex = -1, + *, + content: str | None = None, +) -> None: + """Replace a file atomically with new content. + + The replacement is done atomically by writing to a temporary file and + then moving it to the target location. + + Args: + filepath: The path to the file to replace. + old: The string to replace. + new: The string to replace it with. + count: The maximum number of occurrences to replace. If negative, all occurrences are + replaced. + content: The content to replace. If not provided, the file is read from disk. + """ + if isinstance(filepath, str): + filepath = Path(filepath) + + if content is None: + content = filepath.read_text(encoding="utf-8") + + replace_file_atomically(filepath, content.replace(old, new, count)) + + def calculate_file_sha256_skip_lines(filepath: Path, skip_lines: int) -> str | None: """Calculate SHA256 of file contents excluding the first N lines. @@ -351,8 +467,157 @@ def calculate_file_sha256_skip_lines(filepath: Path, skip_lines: int) -> str | N return hashlib.sha256(remaining_content.encode()).hexdigest() +def find_ruleset(name: str) -> dict[str, Any] | None: + """Find a repository ruleset by name using the GitHub API. + + Args: + name: The name of the ruleset to search for. + + Returns: + The ruleset summary dict (id, name, …) if found, or ``None`` if not + found or if the API call failed (a diagnostic is printed in the latter + case). + """ + try: + stdout = subprocess.check_output( + ["gh", "api", "repos/:owner/:repo/rulesets"], + text=True, + stderr=subprocess.PIPE, + ) + except FileNotFoundError: + print(" gh CLI not found; cannot query rulesets via the GitHub API.") + return None + except subprocess.CalledProcessError as exc: + print(f" Failed to list rulesets: {exc.stderr.strip()}") + return None + + rulesets: list[dict[str, Any]] = json.loads(stdout) + return next((r for r in rulesets if r.get("name") == name), None) + + +def get_ruleset(ruleset: str | int) -> dict[str, Any] | None: + """Fetch the full details of a repository ruleset by name or ID. + + Args: + ruleset: The ruleset name (``str``) or numeric ruleset ID (``int``). + + Returns: + The full ruleset dict, or ``None`` if the ruleset could not be found + or the API call failed (a diagnostic is printed). + """ + ruleset_id = ruleset + if isinstance(ruleset, str): + entry = find_ruleset(ruleset) + if entry is None: + return None + ruleset_id = entry["id"] + + try: + stdout = subprocess.check_output( + ["gh", "api", f"repos/:owner/:repo/rulesets/{ruleset_id}"], + text=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as exc: + print(f" Failed to fetch ruleset {ruleset_id}: {exc.stderr.strip()}") + return None + + return json.loads(stdout) # type: ignore[no-any-return] + + +def update_ruleset(ruleset_id: int, config: dict[str, Any]) -> bool: + """Update a repository ruleset via the GitHub API. + + Only ``name``, ``target``, ``enforcement``, ``conditions``, ``rules``, + and ``bypass_actors`` are sent (explicit allowlist to avoid sending + read-only fields back to the API). + + Args: + ruleset_id: The numeric ruleset ID to update. + config: The full ruleset dict (as returned by :func:`get_ruleset`) + with the desired changes already applied in-memory. + + Returns: + ``True`` on success, ``False`` if the API call failed (a diagnostic + is printed). + """ + payload: dict[str, Any] = { + "name": config["name"], + "target": config["target"], + "enforcement": config["enforcement"], + "conditions": config["conditions"], + "rules": config["rules"], + } + if "bypass_actors" in config: + payload["bypass_actors"] = config["bypass_actors"] + + try: + subprocess.check_output( + [ + "gh", + "api", + "-X", + "PUT", + f"repos/:owner/:repo/rulesets/{ruleset_id}", + "--input", + "-", + ], + input=json.dumps(payload), + text=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as exc: + print(f" Failed to update ruleset {ruleset_id}: {exc.stderr.strip()}") + return False + + return True + + +def get_ruleset_settings_url() -> str | None: + """Return the URL to the repository's ruleset settings page. + + Returns: + The URL as a string, or ``None`` if it could not be determined. + """ + try: + stdout = subprocess.check_output( + ["gh", "repo", "view", "--json", "owner,name"], + text=True, + stderr=subprocess.PIPE, + ) + info: dict[str, Any] = json.loads(stdout) + org = info["owner"]["login"] + repo = info["name"] + return f"https://github.com/{org}/{repo}/settings/rules" + except (subprocess.CalledProcessError, KeyError, json.JSONDecodeError): + return None + + +def read_cookiecutter_str_var(name: str) -> str | None: + """Read a cookiecutter variable from the replay file.""" + replay_path = Path(".cookiecutter-replay.json") + if not replay_path.exists(): + return None + + try: + data = json.loads(replay_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + + cookiecutter_data = data.get("cookiecutter") + if not isinstance(cookiecutter_data, dict): + return None + + value = cookiecutter_data.get(name) + if not isinstance(value, str): + return None + + return value + + def manual_step(message: str) -> None: """Print a manual step message in yellow.""" + _manual_steps.append(message) print(f"\033[0;33m>>> {message}\033[0m") diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/dependabot.yml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/dependabot.yml index 1341d523..794a09e7 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/dependabot.yml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/dependabot.yml @@ -37,11 +37,14 @@ updates: {%- if cookiecutter.type == "api" %} - "frequenz-api-common" {%- endif %} - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[{{ cookiecutter.type }}]" + - "frequenz-repo-config[extra-lint-examples]" - "markdown-callouts" - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" # We group repo-config updates as it uses optional dependencies that are @@ -49,10 +52,13 @@ updates: # each if we don't group them. repo-config: patterns: - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[{{ cookiecutter.type }}]" + - "frequenz-repo-config[extra-lint-examples]" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/auto-dependabot.yaml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/auto-dependabot.yaml index bde9b5ef..94e4169d 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/auto-dependabot.yaml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/auto-dependabot.yaml @@ -1,23 +1,42 @@ -{% raw %}name: Auto-merge Dependabot PR +{% raw -%} +name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + name: Auto-merge Dependabot PR + if: | + github.actor == 'dependabot[bot]' && + !contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' add-label: 'tool:auto-merged' -{% endraw %} +{%- endraw %} diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/ci.yaml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/ci.yaml index 41d4f797..39c4267b 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/ci.yaml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/ci.yaml @@ -61,11 +61,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -74,7 +72,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -96,7 +94,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -145,15 +143,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -205,7 +201,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -328,7 +324,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files uses: actions/download-artifact@v4 diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/dco-merge-queue.yml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/dco-merge-queue.yml index ce9b21a4..cd596d5e 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/dco-merge-queue.yml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/dco-merge-queue.yml @@ -6,7 +6,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/labeler.yml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/labeler.yml index 8cfbe211..b1bb1401 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/labeler.yml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/labeler.yml @@ -8,7 +8,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/release-notes-check.yml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/release-notes-check.yml index e972fb06..8e748d56 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/release-notes-check.yml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/repo-config-migration.yaml b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/repo-config-migration.yaml new file mode 100644 index 00000000..306828b1 --- /dev/null +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/.github/workflows/repo-config-migration.yaml @@ -0,0 +1,62 @@ +{% raw -%} +# Automatic repo-config migrations for Dependabot PRs +# +# The companion auto-dependabot workflow skips repo-config group PRs so +# they're handled exclusively by the migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (migrate.py is fetched from an +# upstream tag, not from the checked-out branch). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Repo Config Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" +{%- endraw %} diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml b/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml index cbe3edad..e2b98382 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ - "setuptools == 75.8.0", + "setuptools == 80.9.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[{{cookiecutter.type}}] == 0.14.0", + "frequenz-repo-config[{{cookiecutter.type}}] == 0.16.0", {%- if cookiecutter.type == "api" %} # We need to pin the protobuf, grpcio and grpcio-tools dependencies to make # sure the code is generated using the minimum supported versions, as older @@ -22,17 +22,19 @@ build-backend = "setuptools.build_meta" name = "{{cookiecutter.pypi_package_name}}" description = "{{cookiecutter.description}}" readme = "README.md" -license = { text = "{{cookiecutter.license}}" } +{%- if cookiecutter.license == "MIT" %} +license = "MIT" +{%- elif cookiecutter.license == "Proprietary" %} +license = "LicenseRef-Proprietary" +{%- else %} +license = "{{cookiecutter.license}}" +{%- endif %} +license-files = ["LICENSE"] keywords = {{cookiecutter | keywords}} # TODO(cookiecutter): Remove and add more classifiers if appropriate classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - {%- if cookiecutter.license == "MIT" %} - "License :: OSI Approved :: MIT License", - {%- elif cookiecutter.license == "Propietary" %} - "License :: Other/Proprietary License", - {%- endif %} "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", {%- if cookiecutter.type != "app" %} @@ -85,6 +87,7 @@ email = "{{cookiecutter.author_email}}" [project.optional-dependencies] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.3", # For reading the flake8 config from pyproject.toml "pydoclint == 0.6.10", @@ -101,7 +104,7 @@ dev-mkdocs = [ "mkdocs-material == 9.6.18", "mkdocstrings[python] == 1.0.0", "mkdocstrings-python == 2.0.1", - "frequenz-repo-config[{{cookiecutter.type}}] == 0.14.0", + "frequenz-repo-config[{{cookiecutter.type}}] == 0.16.0", ] dev-mypy = [ "mypy == 1.9.0", @@ -114,7 +117,7 @@ dev-mypy = [ ] dev-noxfile = [ "nox == 2025.5.1", - "frequenz-repo-config[{{cookiecutter.type}}] == 0.14.0", + "frequenz-repo-config[{{cookiecutter.type}}] == 0.16.0", ] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples @@ -124,7 +127,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 8.4.1", "pylint == 3.3.8", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.14.0", + "frequenz-repo-config[extra-lint-examples] == 0.16.0", {%- if cookiecutter.type != "api" %} "pytest-mock == 3.14.0", "pytest-asyncio == 1.1.0", diff --git a/docs/user-guide/SUMMARY.md b/docs/user-guide/SUMMARY.md index b851adea..44a5eb5d 100644 --- a/docs/user-guide/SUMMARY.md +++ b/docs/user-guide/SUMMARY.md @@ -1,5 +1,5 @@ * [Introduction](index.md) * [Start a new project](start-a-new-project/) -* [Migrate an existing project](migrate-an-existing-project.md) -* [Update an existing project](update-an-existing-project.md) +* [Update to a new version](update-to-a-new-version.md) +* [Convert an existing project](convert-an-existing-project.md) * [Advanced usage](advanced-usage.md) diff --git a/docs/user-guide/advanced-usage.md b/docs/user-guide/advanced-usage.md index 7dbe40c0..473b27da 100644 --- a/docs/user-guide/advanced-usage.md +++ b/docs/user-guide/advanced-usage.md @@ -1,3 +1,5 @@ +{% set ref_name = version.ref_name if version else default_branch %} + # Advanced usage The [Cookiecutter] template uses some tools provided as a library by this @@ -10,4 +12,168 @@ using different CLI options for some tools), then you'll need to. You can find information about the extra features in the [API reference](reference/frequenz/repo/config/). +## Migration workflow + +Projects generated from the template already include the automated migration +workflow (`repo-config-migration.yaml`). If you need to set it up in a +project that wasn't generated from the template (or need to recreate it), +follow the steps below. + +The migration workflow uses the +[`gh-action-dependabot-migrate`][gh-action-dependabot-migrate] GitHub Action, +which contains all the migration logic (running scripts, committing changes, +posting comments, managing labels, approving and merging). The workflow in +your repository triggers the action with repo-config-specific inputs. + +See the [`gh-action-dependabot-migrate` +documentation][gh-action-dependabot-migrate] for full details on the action's +inputs, authentication options, trust model, and security design. + +For general information about how the workflow works and how to interact with +it as a user, see the [automated migration workflow][migration-workflow] +section in the update guide. + +### Creating the caller workflow + +Create `.github/workflows/repo-config-migration.yaml` in your repository: + +{% raw %} +```yaml +name: Repo Config Migration + +on: + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + if: contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" +``` +{% endraw %} + +!!! Note + + Keep third-party actions pinned to commit hashes. + [Dependabot] updates those references through the `github-actions` + ecosystem. + +The key repo-config-specific settings are: + +* **`script-url-template`** — points to the `migrate.py` script in this + repository, with `{version}` as a placeholder for the version tag (e.g. + `v0.15.0`). +* **`token`** — a [GitHub App][GitHub App] installation token used for + pushing migration commits and managing PR state (approval and auto-merge, + when applicable). + Because it is not `GITHUB_TOKEN`, API calls made with this token trigger + follow-up workflows (merge queue CI, status checks, etc.). +* **`migration-token`** — a token exposed to the migration script as + `GH_TOKEN` / `GITHUB_TOKEN` for authenticated GitHub API calls (e.g. + updating repository settings or branch rulesets). +* **`auto-merge-on-changes`** — intentionally omitted, so it uses the + action's default (`false`). If the migration produces file changes + (commits), the PR is left for manual review, approval, and merge. +* **`auto-merged-label`** — set to `tool:auto-merged` (shared with the + `auto-dependabot.yaml` workflow) so auto-merged PRs are consistently + labelled. +* **`migrated-label`**, **`intervention-pending-label`**, + **`intervention-done-label`** — custom label names following the + `tool:repo-config:migration:*` naming convention. +* **`if` condition** — matches PRs with `the repo-config group` in the + title, which is how [Dependabot] names PRs for the `repo-config` + dependency group. + +!!! Warning "Security" + + The workflow uses `pull_request_target` because [Dependabot] PRs are + treated as fork PRs: `GITHUB_TOKEN` is read-only and secrets are + unavailable with a plain `pull_request` trigger. The action mitigates + the risk by never executing code from the PR — the migration script is + fetched from an upstream tag. For details, see [Preventing pwn + requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/). + +### Requirements + +The workflow requires: + +* **A [GitHub App][GitHub App]** configured as described in the [GitHub App + for Dependabot + workflows](start-a-new-project/configure-github.md#github-app-for-dependabot-workflows) + section. + +* **A `REPO_CONFIG_MIGRATION_TOKEN` secret** — a token (e.g. a fine-grained PAT) that + gives the migration script access to the GitHub API. See the [GitHub App + for Dependabot + workflows](start-a-new-project/configure-github.md#github-app-for-dependabot-workflows) + section for details. + + **Only necessary if the migration script needs to make authenticated API + calls, it can be left empty otherwise.** + +* **Auto-merge enabled** in the repository settings (Settings > General > + Pull Requests > Allow auto-merge). + +* **A `repo-config` dependency group** in `.github/dependabot.yml` that + groups `frequenz-repo-config*` packages: + + ```yaml + groups: + repo-config: + patterns: + - "frequenz-repo-config*" + ``` + +* **A `github-actions` ecosystem** in `.github/dependabot.yml` so the + action version stays up to date: + + ```yaml + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + ``` + +### Interaction with other workflows + +The `auto-dependabot.yaml` workflow has a job-level `if` condition that +**skips** PRs containing `the repo-config group` in the title. This ensures +repo-config dependency updates are handled exclusively by the migration +workflow, while all other [Dependabot] PRs continue to be auto-approved and +merged by the existing workflow. + +The `github-actions` ecosystem updates to the *caller workflow itself* (i.e. +bumping the action's `@` reference) produce PRs with `the compatible +group` in the title, which does **not** match the migration workflow's `if` +condition. These PRs are handled normally by `auto-dependabot.yaml`. + [Cookiecutter]: https://cookiecutter.readthedocs.io/en/stable +[Dependabot]: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates +[GitHub App]: https://docs.github.com/en/apps/creating-github-apps +[gh-action-dependabot-migrate]: https://github.com/frequenz-floss/gh-action-dependabot-migrate +[migration-workflow]: update-to-a-new-version.md#automated-migration-workflow diff --git a/docs/user-guide/migrate-an-existing-project.md b/docs/user-guide/convert-an-existing-project.md similarity index 72% rename from docs/user-guide/migrate-an-existing-project.md rename to docs/user-guide/convert-an-existing-project.md index 2cc92beb..61df8f4b 100644 --- a/docs/user-guide/migrate-an-existing-project.md +++ b/docs/user-guide/convert-an-existing-project.md @@ -1,6 +1,6 @@ -# Migrate an existing project +# Convert an existing project -The easiest way to migrate an existing project is to generate a new one based +The easiest way to convert an existing project is to generate a new one based on the current project metadata and then overwrite the existing files. It is recommended to commit all changes before doing this, so you can then use @@ -31,5 +31,7 @@ git commit -a !!! Tip Please have a look at the follow-up steps listed in the [Start a new - project](#create-the-local-development-environment) section to finish the - setup. + project](start-a-new-project/#create-the-local-environment) section to + finish the setup, including the [GitHub App + configuration](start-a-new-project/configure-github.md#github-app-for-dependabot-workflows) + required for the automated migration and auto-merge workflows. diff --git a/docs/user-guide/start-a-new-project/configure-github.md b/docs/user-guide/start-a-new-project/configure-github.md index b0d2c84c..31275ea5 100644 --- a/docs/user-guide/start-a-new-project/configure-github.md +++ b/docs/user-guide/start-a-new-project/configure-github.md @@ -125,3 +125,41 @@ The basic code configuration should be generate using No special configuration is needed for GitHub Pages, but you need to initialize the `gh-pages` branch. You can read how to do this in the [Initialize GitHub Pages](index.md#initialize-github-pages.md) section. + +## GitHub Actions + +### GitHub App for Dependabot workflows + +The templates include two workflows that act on [Dependabot] PRs: + +* **`auto-dependabot.yaml`** — auto-approves and enables auto-merge for + routine dependency updates. +* **`repo-config-migration.yaml`** — runs the repo-config migration script + and auto-merges only when the migration does not create commits; otherwise + it requires manual approval and merge (see + [Automated migration workflow](../update-to-a-new-version.md#automated-migration-workflow) + for details). + +Both workflows use a [GitHub App][GitHub App] installation token instead of +`GITHUB_TOKEN`. This is intentional: actions performed with `GITHUB_TOKEN` do +not trigger certain follow-up workflow runs, which can prevent merge queue CI +(`merge_group`) from starting. + +To make them work, ensure a [GitHub App][GitHub App] is installed on the +repository with at least `Pull requests: write` and `Contents: write` +permissions (add `Workflows: write` if migrations can touch +`.github/workflows/*` files). + +If the migration script needs to make authenticated API calls (e.g. updating +repository settings or branch rulesets), you should also provide the +`REPO_CONFIG_MIGRATION_TOKEN` secret with a dedicated, least-privilege token +scoped to exactly what the script needs. The `REPO_CONFIG_MIGRATION_TOKEN` is +exposed to the migration script as `GH_TOKEN` / `GITHUB_TOKEN` + +> [!TIP] +> It is recommended to have this secret set only when needed, and remove it +> again as soon as the migration is done. This minimizes the risk of abuse in +> case of a security issue in the migration script. + +[Dependabot]: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates +[GitHub App]: https://docs.github.com/en/apps/creating-github-apps diff --git a/docs/user-guide/update-an-existing-project.md b/docs/user-guide/update-to-a-new-version.md similarity index 50% rename from docs/user-guide/update-an-existing-project.md rename to docs/user-guide/update-to-a-new-version.md index 44029407..96b434d2 100644 --- a/docs/user-guide/update-an-existing-project.md +++ b/docs/user-guide/update-to-a-new-version.md @@ -1,15 +1,111 @@ {% set ref_name = version.ref if version else 'HEAD' %} -# Update an existing project +# Update to a new version + +To upgrade an existing project to a new repo-config version, there are three +approaches: let the automated migration workflow handle it (default and +recommended), run the migration script manually, or re-run the [Cookiecutter] +command. + +## Automated migration workflow + +Projects generated from the template include a GitHub Actions workflow +(`repo-config-migration.yaml`) that runs the migration script automatically +when [Dependabot] opens a pull request for the `repo-config` dependency group. + +For updates where the migration script does not create additional commits and +does not report manual steps, the workflow auto-approves and enables +auto-merge. If the migration creates commits or reports manual steps, the PR +always requires manual review, approval, and merge. + +```mermaid +flowchart TD + A((Start)) --> B{Migration} + + %% transitions out of Migration + B -->|succeeded without changes| C["Approved + auto-merge enabled"] + B -->|intervention needed| D["Pending intervention (merges blocked)"] + B -->|succeeded with changes| F["Awaiting review"] + + %% normal flow + C -->|checks pass| E((("Merged"))) + + %% intervention loop + D -->|intervention done| F + F -->|intervention undone| D + + %% review/merge + F -->|checks pass + approval/merge| E + + %% layout: enforce the desired rows + subgraph r1[ ] + direction LR + A + end + + subgraph r2[ ] + direction LR + B + end + + subgraph r3[ ] + direction LR + C --- D + end + + subgraph r4[ ] + direction LR + F + end + + subgraph r5[ ] + direction LR + E + end + + %% keep row boxes invisible + style r1 fill:transparent,stroke:transparent + style r2 fill:transparent,stroke:transparent + style r3 fill:transparent,stroke:transparent + style r4 fill:transparent,stroke:transparent + style r5 fill:transparent,stroke:transparent +``` + +If the migration script exits with a non-zero status, the workflow: + +* Posts a PR comment with the full migration output. +* Adds the `tool:repo-config:migration:intervention-pending` label. +* Fails the job, which blocks merging. -To upgrade an existing project, there are two main approaches: use a migration -script or re-run the [Cookiecutter] command. +After you complete the required manual steps, push your changes to the PR +branch and signal resolution in **either** of these two ways: -## Use a migration script +* **Remove** the `tool:repo-config:migration:intervention-pending` label from + the PR. +* **Add** the `tool:repo-config:migration:intervention-done` label to the PR. -This is the recommended approach, as it should be much less work. Only when -extremely deep changes are made to the template, or when the project is very -old, should you consider re-running the [Cookiecutter] command. Usually release -notes will warn you about the former. +Either action triggers the workflow again, which normalises the labels. +Whenever a migration produces commits or requires manual steps, you should +review, approve, and merge the PR yourself. + +If intervention is marked as done and you later need more manual work, either +removing `tool:repo-config:migration:intervention-done` or adding +`tool:repo-config:migration:intervention-pending` marks intervention as +pending again. + +!!! Note + + If you need to set up this workflow in a project that wasn't generated + from the template, see the + [migration workflow setup](advanced-usage.md#migration-workflow) + section in the advanced usage page. You will also need to configure the + [GitHub App for Dependabot + workflows](start-a-new-project/configure-github.md#github-app-for-dependabot-workflows). + +## Use a migration script manually + +If you prefer to run the migration yourself (or if your project doesn't use +the automated workflow), you can fetch the migration script from GitHub and +run it directly. The script can't always perform all the changes necessary to migrate to a new version. In this case, you will have to manually apply the changes. The script @@ -18,9 +114,6 @@ will guide you through the process, so please read the script output carefully. The script can also only migrate from one version to the next. If you are skipping versions, you will have to run the script multiple times. -The easiest way to run the migration script is to fetch it from GitHub and run -it directly. - ```sh curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{{ ref_name }}/cookiecutter/migrate.py \ | python3 @@ -108,5 +201,8 @@ templates update commit. !!! Tip Please have a look at the follow-up steps listed in the [Start a new - project](#create-the-local-development-environment) section to finish the - setup. + project](start-a-new-project/#create-the-local-environment) section to + finish the setup. + +[Cookiecutter]: https://cookiecutter.readthedocs.io/en/stable +[Dependabot]: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates diff --git a/github-rulesets/python/Protect version branches.json b/github-rulesets/python/Protect version branches.json index a07ba535..a62a4944 100644 --- a/github-rulesets/python/Protect version branches.json +++ b/github-rulesets/python/Protect version branches.json @@ -23,12 +23,12 @@ { "type": "pull_request", "parameters": { - "require_code_owner_review": true, + "require_code_owner_review": false, "require_last_push_approval": true, "dismiss_stale_reviews_on_push": true, "required_approving_review_count": 1, + "required_reviewers": [], "required_review_thread_resolution": false, - "automatic_copilot_code_review_enabled": true, "allowed_merge_methods": [ "merge", "squash", @@ -55,6 +55,10 @@ { "context": "Check release notes are updated", "integration_id": 15368 + }, + { + "context": "Migrate Repo Config", + "integration_id": 15368 } ], "strict_required_status_checks_policy": false @@ -76,7 +80,7 @@ "bypass_mode": "always" }, { - "actor_id": 1, + "actor_id": null, "actor_type": "OrganizationAdmin", "bypass_mode": "always" } diff --git a/pyproject.toml b/pyproject.toml index b80609c8..dd928632 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,15 @@ # Copyright © 2023 Frequenz Energy-as-a-Service GmbH [build-system] -requires = ["setuptools == 80.9.0", "setuptools_scm[toml] == 9.2.2"] +requires = ["setuptools == 82.0.0", "setuptools_scm[toml] == 9.2.2"] build-backend = "setuptools.build_meta" [project] name = "frequenz-repo-config" description = "Frequenz repository setup tools and common configuration" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = [ "config", "frequenz", @@ -29,7 +30,6 @@ keywords = [ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries", @@ -57,7 +57,7 @@ actor = [] api = [ "grpcio-tools >= 1.47.0, < 2", "mypy-protobuf >= 3.0.0, < 6", - "setuptools >= 67.6.0, < 81", + "setuptools >= 67.6.0, < 83", ] app = [] lib = [] @@ -69,46 +69,50 @@ extra-lint-examples = [ ] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.4", # For reading the flake8 config from pyproject.toml "pydoclint == 0.8.3", "pydocstyle == 6.3.0", ] -dev-formatting = ["black == 25.12.0", "isort == 7.0.0"] +dev-formatting = ["black == 26.1.0", "isort == 8.0.1"] dev-mkdocs = [ - "black == 25.12.0", - "Markdown == 3.10", + "black == 26.1.0", + "Markdown == 3.10.2", "mike == 2.1.3", "mkdocs-gen-files == 0.6.0", "mkdocs-literate-nav == 0.6.2", "mkdocs-macros-plugin == 1.5.0", - "mkdocs-material == 9.7.1", - "mkdocstrings[python] == 1.0.0", - "mkdocstrings-python == 2.0.1", + "mkdocs-material == 9.7.3", + "mkdocstrings[python] == 1.0.3", + "mkdocstrings-python == 2.0.3", ] dev-mypy = [ "mypy == 1.19.1", - "types-setuptools >= 67.6.0, < 81", # Should match the api dependency - "types-Markdown == 3.10.0.20251106", + "types-setuptools >= 67.6.0, < 83", # Should match the api dependency + "types-Markdown == 3.10.2.20260211", "types-PyYAML == 6.0.12.20250915", "types-babel == 2.11.0.15", "types-colorama == 0.4.15.20250801", # For checking the noxfile, docs/ script, and tests "frequenz-repo-config[dev-mkdocs,dev-noxfile,dev-pytest]", ] -dev-noxfile = ["nox == 2025.11.12"] +dev-noxfile = ["nox == 2026.2.9"] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples # For checking the noxfile, docs/ script, and tests "frequenz-repo-config[dev-mkdocs,dev-noxfile,dev-pytest]", - "setuptools >= 67.6.0, < 81", # Should match the api dependency + "setuptools >= 67.6.0, < 83", # Should match the api dependency ] dev-pytest = [ "pytest == 9.0.2", - "pylint == 4.0.4", # We need this to check for the examples + "pylint == 4.0.5", # We need this to check for the examples "cookiecutter == 2.6.0", # For checking the cookiecutter scripts "jinja2 == 3.1.6", # For checking the cookiecutter scripts "sybil >= 6.1.1, < 10", # Should be consistent with the extra-lint-examples dependency + # This is a hack to overcome an outdated version check in requests, see + # https://github.com/frequenz-floss/frequenz-repo-config-python/issues/527 + "chardet < 6", ] dev = [ "frequenz-repo-config[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]", diff --git a/src/frequenz/repo/config/cli/version/mike/sort.py b/src/frequenz/repo/config/cli/version/mike/sort.py index 1d7d2b27..615eaffd 100644 --- a/src/frequenz/repo/config/cli/version/mike/sort.py +++ b/src/frequenz/repo/config/cli/version/mike/sort.py @@ -3,7 +3,6 @@ """Sort `mike`'s `version.json` file with a custom order.""" - import json import sys from typing import Any, TextIO diff --git a/src/frequenz/repo/config/github.py b/src/frequenz/repo/config/github.py index 1e7754cb..2b07b62f 100644 --- a/src/frequenz/repo/config/github.py +++ b/src/frequenz/repo/config/github.py @@ -14,7 +14,6 @@ logging for GitHub Actions. """ - import logging import os import subprocess diff --git a/src/frequenz/repo/config/mkdocs/mkdocstrings_macros.py b/src/frequenz/repo/config/mkdocs/mkdocstrings_macros.py index 46e8e88d..5d869398 100644 --- a/src/frequenz/repo/config/mkdocs/mkdocstrings_macros.py +++ b/src/frequenz/repo/config/mkdocs/mkdocstrings_macros.py @@ -108,7 +108,6 @@ def define_env(env: macros.MacrosPlugin) -> None: ``` """ - import logging from typing import Any diff --git a/src/frequenz/repo/config/nox/util.py b/src/frequenz/repo/config/nox/util.py index adba040b..e4f1a12a 100644 --- a/src/frequenz/repo/config/nox/util.py +++ b/src/frequenz/repo/config/nox/util.py @@ -7,7 +7,6 @@ modules in this package. """ - import pathlib as _pathlib import tomllib as _tomllib from collections.abc import Iterable, Mapping diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/dependabot.yml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/dependabot.yml index c5b9db3e..ead76f6b 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/dependabot.yml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/dependabot.yml @@ -34,11 +34,14 @@ updates: - "minor" exclude-patterns: - "async-solipsism" - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[actor]" + - "frequenz-repo-config[extra-lint-examples]" - "markdown-callouts" - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" # We group repo-config updates as it uses optional dependencies that are @@ -46,10 +49,13 @@ updates: # each if we don't group them. repo-config: patterns: - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[actor]" + - "frequenz-repo-config[extra-lint-examples]" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/auto-dependabot.yaml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/auto-dependabot.yaml index 79a6fafc..a6c76658 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/auto-dependabot.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/auto-dependabot.yaml @@ -1,23 +1,40 @@ name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + name: Auto-merge Dependabot PR + if: | + github.actor == 'dependabot[bot]' && + !contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' add-label: 'tool:auto-merged' - diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/ci.yaml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/ci.yaml index baeb64e4..0108d706 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/ci.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/ci.yaml @@ -28,11 +28,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -41,7 +39,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -63,7 +61,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -112,15 +110,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -172,7 +168,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -295,7 +291,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files uses: actions/download-artifact@v4 diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/dco-merge-queue.yml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/dco-merge-queue.yml index fb1cd90c..d9597ad0 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/dco-merge-queue.yml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/dco-merge-queue.yml @@ -5,7 +5,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/labeler.yml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/labeler.yml index c844b8d2..f6692548 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/labeler.yml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/labeler.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/release-notes-check.yml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/release-notes-check.yml index 9f7ee31b..545d537a 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/release-notes-check.yml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/repo-config-migration.yaml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/repo-config-migration.yaml new file mode 100644 index 00000000..57a54c32 --- /dev/null +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/.github/workflows/repo-config-migration.yaml @@ -0,0 +1,60 @@ +# Automatic repo-config migrations for Dependabot PRs +# +# The companion auto-dependabot workflow skips repo-config group PRs so +# they're handled exclusively by the migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (migrate.py is fetched from an +# upstream tag, not from the checked-out branch). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Repo Config Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml index bb8c42aa..d487fb2b 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ - "setuptools == 75.8.0", + "setuptools == 80.9.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[actor] == 0.14.0", + "frequenz-repo-config[actor] == 0.16.0", ] build-backend = "setuptools.build_meta" @@ -13,13 +13,13 @@ build-backend = "setuptools.build_meta" name = "frequenz-actor-test" description = "Test description" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["frequenz", "python", "actor", "test"] # TODO(cookiecutter): Remove and add more classifiers if appropriate classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries", @@ -44,6 +44,7 @@ email = "floss@frequenz.com" [project.optional-dependencies] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.3", # For reading the flake8 config from pyproject.toml "pydoclint == 0.6.10", @@ -60,7 +61,7 @@ dev-mkdocs = [ "mkdocs-material == 9.6.18", "mkdocstrings[python] == 1.0.0", "mkdocstrings-python == 2.0.1", - "frequenz-repo-config[actor] == 0.14.0", + "frequenz-repo-config[actor] == 0.16.0", ] dev-mypy = [ "mypy == 1.9.0", @@ -70,7 +71,7 @@ dev-mypy = [ ] dev-noxfile = [ "nox == 2025.5.1", - "frequenz-repo-config[actor] == 0.14.0", + "frequenz-repo-config[actor] == 0.16.0", ] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples @@ -80,7 +81,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 8.4.1", "pylint == 3.3.8", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.14.0", + "frequenz-repo-config[extra-lint-examples] == 0.16.0", "pytest-mock == 3.14.0", "pytest-asyncio == 1.1.0", "async-solipsism == 0.8", diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/dependabot.yml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/dependabot.yml index d091a66c..101f7843 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/dependabot.yml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/dependabot.yml @@ -35,11 +35,14 @@ updates: exclude-patterns: - "async-solipsism" - "frequenz-api-common" - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[api]" + - "frequenz-repo-config[extra-lint-examples]" - "markdown-callouts" - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" # We group repo-config updates as it uses optional dependencies that are @@ -47,10 +50,13 @@ updates: # each if we don't group them. repo-config: patterns: - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[api]" + - "frequenz-repo-config[extra-lint-examples]" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/auto-dependabot.yaml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/auto-dependabot.yaml index 79a6fafc..a6c76658 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/auto-dependabot.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/auto-dependabot.yaml @@ -1,23 +1,40 @@ name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + name: Auto-merge Dependabot PR + if: | + github.actor == 'dependabot[bot]' && + !contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' add-label: 'tool:auto-merged' - diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/ci.yaml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/ci.yaml index ddf321ef..060c331a 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/ci.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/ci.yaml @@ -58,11 +58,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -71,7 +69,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -93,7 +91,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -142,15 +140,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -202,7 +198,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -325,7 +321,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files uses: actions/download-artifact@v4 diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/dco-merge-queue.yml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/dco-merge-queue.yml index fb1cd90c..d9597ad0 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/dco-merge-queue.yml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/dco-merge-queue.yml @@ -5,7 +5,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/labeler.yml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/labeler.yml index c844b8d2..f6692548 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/labeler.yml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/labeler.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/release-notes-check.yml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/release-notes-check.yml index 1d18e8e5..2515b127 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/release-notes-check.yml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/repo-config-migration.yaml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/repo-config-migration.yaml new file mode 100644 index 00000000..57a54c32 --- /dev/null +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/.github/workflows/repo-config-migration.yaml @@ -0,0 +1,60 @@ +# Automatic repo-config migrations for Dependabot PRs +# +# The companion auto-dependabot workflow skips repo-config group PRs so +# they're handled exclusively by the migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (migrate.py is fetched from an +# upstream tag, not from the checked-out branch). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Repo Config Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml index a7a7b4f7..8bff8179 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ - "setuptools == 75.8.0", + "setuptools == 80.9.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[api] == 0.14.0", + "frequenz-repo-config[api] == 0.16.0", # We need to pin the protobuf, grpcio and grpcio-tools dependencies to make # sure the code is generated using the minimum supported versions, as older # versions can't work with code that was generated with newer versions. @@ -20,13 +20,13 @@ build-backend = "setuptools.build_meta" name = "frequenz-api-test" description = "Test description" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["frequenz", "python", "api", "grpc", "protobuf", "rpc", "test"] # TODO(cookiecutter): Remove and add more classifiers if appropriate classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries", @@ -55,6 +55,7 @@ email = "floss@frequenz.com" [project.optional-dependencies] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.3", # For reading the flake8 config from pyproject.toml "pydoclint == 0.6.10", @@ -71,7 +72,7 @@ dev-mkdocs = [ "mkdocs-material == 9.6.18", "mkdocstrings[python] == 1.0.0", "mkdocstrings-python == 2.0.1", - "frequenz-repo-config[api] == 0.14.0", + "frequenz-repo-config[api] == 0.16.0", ] dev-mypy = [ "mypy == 1.9.0", @@ -82,7 +83,7 @@ dev-mypy = [ ] dev-noxfile = [ "nox == 2025.5.1", - "frequenz-repo-config[api] == 0.14.0", + "frequenz-repo-config[api] == 0.16.0", ] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples @@ -92,7 +93,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 8.4.1", "pylint == 3.3.8", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.14.0", + "frequenz-repo-config[extra-lint-examples] == 0.16.0", ] dev = [ "frequenz-api-test[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]", diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/dependabot.yml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/dependabot.yml index c5b9db3e..dcf2b789 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/dependabot.yml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/dependabot.yml @@ -34,11 +34,14 @@ updates: - "minor" exclude-patterns: - "async-solipsism" - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[app]" + - "frequenz-repo-config[extra-lint-examples]" - "markdown-callouts" - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" # We group repo-config updates as it uses optional dependencies that are @@ -46,10 +49,13 @@ updates: # each if we don't group them. repo-config: patterns: - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[app]" + - "frequenz-repo-config[extra-lint-examples]" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/auto-dependabot.yaml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/auto-dependabot.yaml index 79a6fafc..a6c76658 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/auto-dependabot.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/auto-dependabot.yaml @@ -1,23 +1,40 @@ name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + name: Auto-merge Dependabot PR + if: | + github.actor == 'dependabot[bot]' && + !contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' add-label: 'tool:auto-merged' - diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/ci.yaml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/ci.yaml index baeb64e4..0108d706 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/ci.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/ci.yaml @@ -28,11 +28,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -41,7 +39,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -63,7 +61,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -112,15 +110,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -172,7 +168,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -295,7 +291,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files uses: actions/download-artifact@v4 diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/dco-merge-queue.yml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/dco-merge-queue.yml index fb1cd90c..d9597ad0 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/dco-merge-queue.yml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/dco-merge-queue.yml @@ -5,7 +5,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/labeler.yml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/labeler.yml index c844b8d2..f6692548 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/labeler.yml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/labeler.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/release-notes-check.yml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/release-notes-check.yml index 9f7ee31b..545d537a 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/release-notes-check.yml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/repo-config-migration.yaml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/repo-config-migration.yaml new file mode 100644 index 00000000..57a54c32 --- /dev/null +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/.github/workflows/repo-config-migration.yaml @@ -0,0 +1,60 @@ +# Automatic repo-config migrations for Dependabot PRs +# +# The companion auto-dependabot workflow skips repo-config group PRs so +# they're handled exclusively by the migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (migrate.py is fetched from an +# upstream tag, not from the checked-out branch). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Repo Config Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml index 1f540a3b..9c89d486 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ - "setuptools == 75.8.0", + "setuptools == 80.9.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[app] == 0.14.0", + "frequenz-repo-config[app] == 0.16.0", ] build-backend = "setuptools.build_meta" @@ -13,13 +13,13 @@ build-backend = "setuptools.build_meta" name = "frequenz-app-test" description = "Test description" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["frequenz", "python", "app", "application", "test"] # TODO(cookiecutter): Remove and add more classifiers if appropriate classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Typing :: Typed", @@ -43,6 +43,7 @@ email = "floss@frequenz.com" [project.optional-dependencies] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.3", # For reading the flake8 config from pyproject.toml "pydoclint == 0.6.10", @@ -59,7 +60,7 @@ dev-mkdocs = [ "mkdocs-material == 9.6.18", "mkdocstrings[python] == 1.0.0", "mkdocstrings-python == 2.0.1", - "frequenz-repo-config[app] == 0.14.0", + "frequenz-repo-config[app] == 0.16.0", ] dev-mypy = [ "mypy == 1.9.0", @@ -69,7 +70,7 @@ dev-mypy = [ ] dev-noxfile = [ "nox == 2025.5.1", - "frequenz-repo-config[app] == 0.14.0", + "frequenz-repo-config[app] == 0.16.0", ] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples @@ -79,7 +80,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 8.4.1", "pylint == 3.3.8", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.14.0", + "frequenz-repo-config[extra-lint-examples] == 0.16.0", "pytest-mock == 3.14.0", "pytest-asyncio == 1.1.0", "async-solipsism == 0.8", diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/dependabot.yml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/dependabot.yml index c5b9db3e..19232b73 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/dependabot.yml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/dependabot.yml @@ -34,11 +34,14 @@ updates: - "minor" exclude-patterns: - "async-solipsism" - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[lib]" + - "frequenz-repo-config[extra-lint-examples]" - "markdown-callouts" - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" # We group repo-config updates as it uses optional dependencies that are @@ -46,10 +49,13 @@ updates: # each if we don't group them. repo-config: patterns: - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[lib]" + - "frequenz-repo-config[extra-lint-examples]" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/auto-dependabot.yaml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/auto-dependabot.yaml index 79a6fafc..a6c76658 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/auto-dependabot.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/auto-dependabot.yaml @@ -1,23 +1,40 @@ name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + name: Auto-merge Dependabot PR + if: | + github.actor == 'dependabot[bot]' && + !contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' add-label: 'tool:auto-merged' - diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/ci.yaml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/ci.yaml index baeb64e4..0108d706 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/ci.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/ci.yaml @@ -28,11 +28,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -41,7 +39,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -63,7 +61,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -112,15 +110,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -172,7 +168,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -295,7 +291,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files uses: actions/download-artifact@v4 diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/dco-merge-queue.yml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/dco-merge-queue.yml index fb1cd90c..d9597ad0 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/dco-merge-queue.yml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/dco-merge-queue.yml @@ -5,7 +5,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/labeler.yml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/labeler.yml index c844b8d2..f6692548 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/labeler.yml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/labeler.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/release-notes-check.yml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/release-notes-check.yml index 9f7ee31b..545d537a 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/release-notes-check.yml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/repo-config-migration.yaml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/repo-config-migration.yaml new file mode 100644 index 00000000..57a54c32 --- /dev/null +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/.github/workflows/repo-config-migration.yaml @@ -0,0 +1,60 @@ +# Automatic repo-config migrations for Dependabot PRs +# +# The companion auto-dependabot workflow skips repo-config group PRs so +# they're handled exclusively by the migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (migrate.py is fetched from an +# upstream tag, not from the checked-out branch). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Repo Config Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml index 1789b66b..973421cc 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ - "setuptools == 75.8.0", + "setuptools == 80.9.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[lib] == 0.14.0", + "frequenz-repo-config[lib] == 0.16.0", ] build-backend = "setuptools.build_meta" @@ -13,13 +13,13 @@ build-backend = "setuptools.build_meta" name = "frequenz-test" description = "Test description" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["frequenz", "python", "lib", "library", "test"] # TODO(cookiecutter): Remove and add more classifiers if appropriate classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries", @@ -40,6 +40,7 @@ email = "floss@frequenz.com" [project.optional-dependencies] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.3", # For reading the flake8 config from pyproject.toml "pydoclint == 0.6.10", @@ -56,7 +57,7 @@ dev-mkdocs = [ "mkdocs-material == 9.6.18", "mkdocstrings[python] == 1.0.0", "mkdocstrings-python == 2.0.1", - "frequenz-repo-config[lib] == 0.14.0", + "frequenz-repo-config[lib] == 0.16.0", ] dev-mypy = [ "mypy == 1.9.0", @@ -66,7 +67,7 @@ dev-mypy = [ ] dev-noxfile = [ "nox == 2025.5.1", - "frequenz-repo-config[lib] == 0.14.0", + "frequenz-repo-config[lib] == 0.16.0", ] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples @@ -76,7 +77,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 8.4.1", "pylint == 3.3.8", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.14.0", + "frequenz-repo-config[extra-lint-examples] == 0.16.0", "pytest-mock == 3.14.0", "pytest-asyncio == 1.1.0", "async-solipsism == 0.8", diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/dependabot.yml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/dependabot.yml index c5b9db3e..ca7c8794 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/dependabot.yml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/dependabot.yml @@ -34,11 +34,14 @@ updates: - "minor" exclude-patterns: - "async-solipsism" - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[model]" + - "frequenz-repo-config[extra-lint-examples]" - "markdown-callouts" - "mkdocs-gen-files" - "mkdocs-literate-nav" - "mkdocstrings*" + - "mkdocstrings[python]" - "pydoclint" - "pytest-asyncio" # We group repo-config updates as it uses optional dependencies that are @@ -46,10 +49,13 @@ updates: # each if we don't group them. repo-config: patterns: - - "frequenz-repo-config*" + - "frequenz-repo-config" + - "frequenz-repo-config[model]" + - "frequenz-repo-config[extra-lint-examples]" mkdocstrings: patterns: - "mkdocstrings*" + - "mkdocstrings[python]" - package-ecosystem: "github-actions" directory: "/" diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/auto-dependabot.yaml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/auto-dependabot.yaml index 79a6fafc..a6c76658 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/auto-dependabot.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/auto-dependabot.yaml @@ -1,23 +1,40 @@ name: Auto-merge Dependabot PR on: - pull_request: + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: permissions: - contents: write + contents: read pull-requests: write jobs: auto-merge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + name: Auto-merge Dependabot PR + if: | + github.actor == 'dependabot[bot]' && + !contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-slim steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Auto-merge Dependabot PR - uses: frequenz-floss/dependabot-auto-approve@3cad5f42e79296505473325ac6636be897c8b8a1 # v1.3.2 + uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app-token.outputs.token }} dependency-type: 'all' auto-merge: 'true' merge-method: 'merge' add-label: 'tool:auto-merged' - diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/ci.yaml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/ci.yaml index baeb64e4..0108d706 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/ci.yaml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/ci.yaml @@ -28,11 +28,9 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" @@ -41,7 +39,7 @@ jobs: # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Run nox @@ -63,7 +61,7 @@ jobs: needs: ["nox"] # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -112,15 +110,13 @@ jobs: strategy: fail-fast: false matrix: - arch: - - amd64 - - arm - os: + platform: - ubuntu-24.04 + - ubuntu-24.04-arm python: - "3.11" - "3.12" - runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + runs-on: ${{ matrix.platform }} steps: - name: Setup Git @@ -172,7 +168,7 @@ jobs: needs: ["test-installation"] # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -295,7 +291,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Download distribution files uses: actions/download-artifact@v4 diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/dco-merge-queue.yml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/dco-merge-queue.yml index fb1cd90c..d9597ad0 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/dco-merge-queue.yml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/dco-merge-queue.yml @@ -5,7 +5,7 @@ on: jobs: DCO: - runs-on: ubuntu-latest + runs-on: ubuntu-slim if: ${{ github.actor != 'dependabot[bot]' }} steps: - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/labeler.yml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/labeler.yml index c844b8d2..f6692548 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/labeler.yml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/labeler.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Labeler # XXX: !!! SECURITY WARNING !!! diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/release-notes-check.yml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/release-notes-check.yml index 9f7ee31b..545d537a 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/release-notes-check.yml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/release-notes-check.yml @@ -16,7 +16,7 @@ on: jobs: check-release-notes: name: Check release notes are updated - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read steps: diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/repo-config-migration.yaml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/repo-config-migration.yaml new file mode 100644 index 00000000..57a54c32 --- /dev/null +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/.github/workflows/repo-config-migration.yaml @@ -0,0 +1,60 @@ +# Automatic repo-config migrations for Dependabot PRs +# +# The companion auto-dependabot workflow skips repo-config group PRs so +# they're handled exclusively by the migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (migrate.py is fetched from an +# upstream tag, not from the checked-out branch). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Repo Config Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + repo-config-migration: + name: Migrate Repo Config + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + contains(github.event.pull_request.title, 'the repo-config group') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@07dc7e74726498c50726a80cc2167a04d896508f # v1.0.0 + with: + script-url-template: >- + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py + token: ${{ steps.create-app-token.outputs.token }} + migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:repo-config:migration:executed" + intervention-pending-label: "tool:repo-config:migration:intervention-pending" + intervention-done-label: "tool:repo-config:migration:intervention-done" diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml index 9116df25..af78dadd 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml @@ -3,9 +3,9 @@ [build-system] requires = [ - "setuptools == 75.8.0", + "setuptools == 80.9.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[model] == 0.14.0", + "frequenz-repo-config[model] == 0.16.0", ] build-backend = "setuptools.build_meta" @@ -13,13 +13,13 @@ build-backend = "setuptools.build_meta" name = "frequenz-model-test" description = "Test description" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] keywords = ["frequenz", "python", "model", "ai", "ml", "machine-learning", "test"] # TODO(cookiecutter): Remove and add more classifiers if appropriate classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries", @@ -44,6 +44,7 @@ email = "floss@frequenz.com" [project.optional-dependencies] dev-flake8 = [ "flake8 == 7.3.0", + "flake8-datetimez == 20.10.0", "flake8-docstrings == 1.7.0", "flake8-pyproject == 1.2.3", # For reading the flake8 config from pyproject.toml "pydoclint == 0.6.10", @@ -60,7 +61,7 @@ dev-mkdocs = [ "mkdocs-material == 9.6.18", "mkdocstrings[python] == 1.0.0", "mkdocstrings-python == 2.0.1", - "frequenz-repo-config[model] == 0.14.0", + "frequenz-repo-config[model] == 0.16.0", ] dev-mypy = [ "mypy == 1.9.0", @@ -70,7 +71,7 @@ dev-mypy = [ ] dev-noxfile = [ "nox == 2025.5.1", - "frequenz-repo-config[model] == 0.14.0", + "frequenz-repo-config[model] == 0.16.0", ] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples @@ -80,7 +81,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 8.4.1", "pylint == 3.3.8", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.14.0", + "frequenz-repo-config[extra-lint-examples] == 0.16.0", "pytest-mock == 3.14.0", "pytest-asyncio == 1.1.0", "async-solipsism == 0.8",