add update-nixpkgs-tooling

PL-133100

Co-Authored-By: Max Bosch <[email protected]>

.github/workflows/update-nixpkgs.yaml

@@ -0,0 +1,53 @@
+name: update-nixpkgs
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "5 3 * * *"
+
+jobs:
+  run-nixpkgs-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: 'release-tools'
+      - uses: cachix/install-nix-action@v21
+        with:
+          # Nix 2.24 breaks flake update
+          install_url: https://releases.nixos.org/nix/nix-2.18.9/install
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.NIXPKGS_UPDATE_APP_ID }}
+          private-key: ${{ secrets.NIXPKGS_UPDATE_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+      - run: |
+          echo "::add-mask::${{steps.app-token.outputs.token}}"
+      - name: Get GitHub App User ID
+        id: get-user-id
+        run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+      - run: |
+          git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]'
+          git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com>'
+      - uses: actions/checkout@v4
+        with:
+          repository: flyingcircusio/fc-nixos-testing
+          path: 'fc-nixos'
+          token: ${{ steps.app-token.outputs.token }}
+          # fetch all branches and tags
+          fetch-depth: 0
+      - name: build release tooling
+        run: |
+          nix build ./release-tools#
+      - run: |
+          ./result/bin/update-nixpkgs update \
+            --fc-nixos-dir fc-nixos \
+            --nixpkgs-dir nixpkgs \
+            --nixpkgs-upstream-url https://github.com/NixOS/nixpkgs \
+            --nixpkgs-origin-url https://x-access-token:${{steps.app-token.outputs.token}}@github.com/flyingcircusio/nixpkgs-testing.git \
+            --platform-versions 24.05
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}

flake.nix

@@ -9,15 +9,17 @@
     {
       packages = forAllSystems (system: let
         pkgs = nixpkgs.legacyPackages.${system};
-        ppkgs = pkgs.python3.pkgs;
+        pypkgs = pkgs.python3.pkgs;
       in {
-        default = ppkgs.buildPythonApplication {
+        default = pypkgs.buildPythonApplication {
           name = "fc-release";
           src = ./.;
           pyproject = true;
-          build-system = [ ppkgs.setuptools-scm ];
+          build-system = [ pypkgs.setuptools-scm ];
           dependencies = [
-            ppkgs.setuptools
+            pypkgs.setuptools
+            pypkgs.gitpython
+            pypkgs.pygithub
             pkgs.scriv
             pkgs.gh
           ];

pyproject.toml

@@ -19,3 +19,4 @@ dependencies = [
 
 [project.scripts]
 fc-release = "release.fc_release:main"
+update-nixpkgs = "update_nixpkgs:main"

src/update_nixpkgs/__init__.py

@@ -0,0 +1,81 @@
+import argparse
+import os
+import sys
+from logging import basicConfig, INFO
+
+FC_NIXOS_REPO = "flyingcircusio/fc-nixos-testing"
+NIXPKGS_REPO = "flyingcircusio/nixpkgs-testing"
+
+def main():
+    import update_nixpkgs.update
+    import update_nixpkgs.cleanup
+
+    basicConfig(level=INFO)
+    try:
+        github_access_token = os.environ["GH_TOKEN"]
+    except KeyError:
+        raise Exception("Missing `GH_TOKEN` environment variable.")
+
+    parser = argparse.ArgumentParser("nixpkgs updater for fc-nixos")
+    parser.set_defaults(func="print_usage")
+    subparsers = parser.add_subparsers()
+
+    parser_update = subparsers.add_parser('update', help='run nixpkgs update workflow')
+    parser_update.add_argument(
+        "--fc-nixos-dir",
+        help="Directory where the fc-nixos git checkout is in",
+        required=True,
+    )
+    parser_update.add_argument(
+        "--nixpkgs-dir",
+        help="Directory where the nixpkgs git checkout is in",
+        required=True,
+    )
+    parser_update.add_argument(
+        "--nixpkgs-upstream-url",
+        help="URL to the upstream nixpkgs repository",
+        required=True,
+    )
+    parser_update.add_argument(
+        "--nixpkgs-origin-url",
+        help="URL to push the nixpkgs updates to",
+        required=True,
+    )
+    parser_update.add_argument(
+        "--platform-versions",
+        help="Platform versions",
+        required=True,
+        nargs="+",
+    )
+    parser_update.set_defaults(func=update_nixpkgs.update.run)
+
+    parser_cleanup = subparsers.add_parser('cleanup', help='run nixpkgs update cleanup')
+    parser_cleanup.add_argument(
+        "--merged-pr-id", help="merged fc-nixos PR ID", required=True
+    )
+    parser_cleanup.add_argument(
+        "--nixpkgs-dir",
+        help="Directory where the nixpkgs git checkout is in",
+        required=True,
+    )
+    parser_cleanup.add_argument(
+        "--nixpkgs-origin-url",
+        help="URL to push the nixpkgs updates to",
+        required=True,
+    )
+    parser_cleanup.set_defaults(func=update_nixpkgs.cleanup.run)
+
+    args = parser.parse_args()
+    func = args.func
+    if func == "print_usage":
+        parser.print_usage()
+        sys.exit(1)
+
+    kwargs = dict(args._get_kwargs())
+    del kwargs["func"]
+    kwargs["github_access_token"] = github_access_token
+    func(**kwargs)
+
+
+if __name__ == "__main__":
+    main()

src/update_nixpkgs/cleanup.py

@@ -0,0 +1,120 @@
+"""
+This script should be run when an automatic update-nixpkgs PR has been merged.
+It will merge the corresponding flyingcircus/nixpkgs PR and cleanup
+all old fc-nixos and nixpkgs PRs/branches that haven't been merged.
+"""
+import datetime
+import os
+from dataclasses import dataclass
+from logging import info, warning
+
+from git import GitCommandError, Repo
+from github import Auth, Github
+
+from update_nixpkgs import FC_NIXOS_REPO, NIXPKGS_REPO
+
+
+@dataclass
+class Remote:
+    url: str
+    branches: list[str]
+
+
+def nixpkgs_repository(directory: str, remotes: dict[str, Remote]) -> Repo:
+    info("Updating nixpkgs repository.")
+    if os.path.exists(directory):
+        repo = Repo(directory)
+    else:
+        repo = Repo.init(directory, mkdir=True)
+
+    for name, remote in remotes.items():
+        info(f"Updating nixpkgs repository remote `{name}`.")
+        if name in repo.remotes and repo.remotes[name].url != remote.url:
+            repo.delete_remote(repo.remote(name))
+        if name not in repo.remotes:
+            repo.create_remote(name, remote.url)
+
+        for branch in remote.branches:
+            info(
+                f"Fetching nixpkgs repository remote `{name}` - branch `{branch}`."
+            )
+            getattr(repo.remotes, name).fetch(
+                refspec=branch, filter="blob:none"
+            )
+
+    return repo
+
+
+def rebase_nixpkgs(
+    gh: Github, nixpkgs_repo: Repo, target_branch: str, integration_branch: str
+) -> bool:
+    """Rebase nixpkgs repo integration branch onto target branch
+    Returns: True when successful, False when unsuccessful.
+    """
+    info(f"Rebase nixpkgs repo integration branch onto target branch.")
+    if nixpkgs_repo.is_dirty():
+        raise Exception("Repository is dirty!")
+
+    nixpkgs_repo.git.checkout(target_branch)
+
+    try:
+        nixpkgs_repo.git.rebase(f"origin/{integration_branch}")
+    except GitCommandError as e:
+        warning(f"Rebase failed:\n{e.stderr}")
+        nixpkgs_repo.git.rebase(abort=True)
+        warning("Aborted rebase.")
+        return False
+
+    nixpkgs_repo.git.tag(integration_branch, message=integration_branch)
+    nixpkgs_repo.git.push(force_with_lease=True)
+    nixpkgs_repo.git.push(integration_branch)
+    gh.get_repo(NIXPKGS_REPO).get_git_ref(
+        f"heads/{integration_branch}"
+    ).delete()
+    return True
+
+
+def cleanup_old_prs_and_branches(gh: Github, merged_integration_branch: str):
+    info("Cleaning up old PRs and branches.")
+    fc_nixos_repo = gh.get_repo(FC_NIXOS_REPO)
+    nixpkgs_repo = gh.get_repo(NIXPKGS_REPO)
+    merged_integration_branch_date = datetime.date.fromisoformat(
+        merged_integration_branch.split("/")[2]
+    )
+    # branches will be closed automatically by GitHub, when the branch is deleted
+    for repo in [fc_nixos_repo, nixpkgs_repo]:
+        for branch in repo.get_branches():
+            if not branch.name.startswith("nixpkgs-auto-update/"):
+                continue
+            branch_datestr = branch.name.split("/")[2]
+            if (
+                datetime.date.fromisoformat(branch_datestr)
+                < merged_integration_branch_date
+            ):
+                repo.get_git_ref(f"heads/{branch.name}").delete()
+
+
+def run(merged_pr_id: str, nixpkgs_origin_url: str, nixpkgs_dir: str, github_access_token:str):
+    gh = Github(auth=Auth.Token(github_access_token))
+    fc_nixos_pr = gh.get_repo(FC_NIXOS_REPO).get_pull(int(merged_pr_id))
+    pr_platform_version = fc_nixos_pr.base.ref.split("-")[1]
+    integration_branch = fc_nixos_pr.head.ref
+    nixpkgs_target_branch = f"nixos-{pr_platform_version}"
+
+    remotes = {
+        "origin": Remote(
+            nixpkgs_origin_url,
+            [integration_branch, nixpkgs_target_branch],
+        )
+    }
+    nixpkgs_repo = nixpkgs_repository(nixpkgs_dir, remotes)
+    if rebase_nixpkgs(
+        gh,
+        nixpkgs_repo,
+        nixpkgs_target_branch,
+        integration_branch,
+    ):
+        fc_nixos_pr.create_issue_comment(
+            f"Rebased nixpkgs `{nixpkgs_target_branch}` branch successfully."
+        )
+        cleanup_old_prs_and_branches(gh, integration_branch)