Merge pull request #382 from alltilla/package-indexer-rpm

package-indexer: rpm repo
axoflow · Nov 27, 2024 · fd273a4 · fd273a4
2 parents 6dcefd6 + d72c7aa
commit fd273a4
Show file tree

Hide file tree

Showing 16 changed files with 445 additions and 72 deletions.
diff --git a/.github/workflows/create-packages.yml b/.github/workflows/create-packages.yml
@@ -22,7 +22,6 @@ on:
           "ubuntu-jammy",
           "ubuntu-noble",
           "ubuntu-oracular",
-          "ubuntu-lunar",
           "almalinux-8",
           "almalinux-9",
           "fedora-39",

diff --git a/.github/workflows/dbld-images.yml b/.github/workflows/dbld-images.yml
@@ -44,7 +44,6 @@ jobs:
           - ubuntu-jammy
           - ubuntu-noble
           - ubuntu-oracular
-          - ubuntu-lunar
           - devshell
           - tarball
       fail-fast: false

diff --git a/.github/workflows/package-indexer-checks.yml b/.github/workflows/package-indexer-checks.yml
@@ -35,10 +35,10 @@ jobs:
 
       - name: style-check
         run: |
-          docker exec ${DOCKER_CONTAINER_NAME} pip install black
+          docker exec ${DOCKER_CONTAINER_NAME} pip install --break-system-packages black
           docker exec ${DOCKER_CONTAINER_NAME} black --check --diff .
 
       - name: linter
         run: |
-          docker exec ${DOCKER_CONTAINER_NAME} pip install mypy
+          docker exec ${DOCKER_CONTAINER_NAME} pip install --break-system-packages mypy
           docker exec ${DOCKER_CONTAINER_NAME} mypy .
diff --git a/.github/workflows/test-apt-packages.yml b/.github/workflows/test-apt-packages.yml
@@ -23,7 +23,6 @@ jobs:
           - "ubuntu:focal"
           - "ubuntu:jammy"
           - "ubuntu:noble"
-          - "ubuntu:lunar"
           - "ubuntu:oracular"
         upgrade-from:
           - "debian-official"

diff --git a/.github/workflows/upload-packages.yml b/.github/workflows/upload-packages.yml
@@ -10,17 +10,23 @@ on:
       distros:
         required: false
         type: string
-        default: '[
-          "debian-testing",
-          "debian-sid",
-          "debian-bullseye",
-          "debian-bookworm",
-          "ubuntu-focal",
-          "ubuntu-jammy",
-          "ubuntu-noble",
-          "ubuntu-oracular",
-          "ubuntu-lunar",
-        ]'
+        default: >
+          [
+            {"type": "deb", "name": "debian-testing"},
+            {"type": "deb", "name": "debian-sid"},
+            {"type": "deb", "name": "debian-bullseye"},
+            {"type": "deb", "name": "debian-bookworm"},
+            {"type": "deb", "name": "ubuntu-focal"},
+            {"type": "deb", "name": "ubuntu-jammy"},
+            {"type": "deb", "name": "ubuntu-noble"},
+            {"type": "deb", "name": "ubuntu-oracular"},
+
+            {"type": "rpm", "name": "almalinux-8"},
+            {"type": "rpm", "name": "almalinux-9"},
+            {"type": "rpm", "name": "fedora-39"},
+            {"type": "rpm", "name": "fedora-40"},
+            {"type": "rpm", "name": "fedora-41"}
+          ]
     secrets:
       r2-access-key:
         required: true
@@ -32,7 +38,7 @@ on:
 
 jobs:
   upload-packages:
-    name: ${{ matrix.distro }}
+    name: ${{ matrix.distro.name }} (${{ matrix.distro.type }})
 
     runs-on: ubuntu-latest
 
@@ -47,7 +53,7 @@ jobs:
       - name: Download package artifact
         uses: actions/download-artifact@v4
         with:
-          name: package-${{ matrix.distro }}
+          name: package-${{ matrix.distro.name }}
           path: package
 
       - name: Upload packages to R2 incoming
@@ -60,5 +66,5 @@ jobs:
             aws s3api put-object \
               --endpoint-url https://${{ secrets.r2-account-id }}.r2.cloudflarestorage.com \
               --bucket axoflow-packages-build \
-              --key ${{ inputs.pkg-type }}/${{ github.run_id }}/{} \
+              --key ${{ inputs.pkg-type }}/${{ github.run_id }}/${{ matrix.distro.type }}/{} \
               --body {} \;
diff --git a/README.md b/README.md
@@ -141,7 +141,6 @@ The following x86-64 distros are supported:
 | Debian Testing  | debian-testing         |
 | Ubuntu 24.10    | ubuntu-oracular        |
 | Ubuntu 24.04    | ubuntu-noble           |
-| Ubuntu 23.04    | ubuntu-lunar           |
 | Ubuntu 22.04    | ubuntu-jammy           |
 | Ubuntu 20.04    | ubuntu-focal           |
 
@@ -165,6 +164,79 @@ To install AxoSyslog:
 apt install axosyslog
 ```
 
+## RPM packages
+
+You can install AxoSyslog on your RPM-based system from Axoflow's RPM repository.
+AxoSyslog is a drop in replacement for the syslog-ng RPM package, all the binaries
+and configuration files are stored at the same place on your system.
+
+The following x86-64 distros are supported:
+
+| Distro          | axosyslog.repo component |
+|-----------------|--------------------------|
+| Fedora 39       | fedora                   |
+| Fedora 40       | fedora                   |
+| Fedora 41       | fedora                   |
+| AlmaLinux 8     | almalinux                |
+| AlmaLinux 9     | almalinux                |
+
+To add the RPM repo (e.g. Fedora 41):
+
+```
+yum install -y epel-release
+
+tee /etc/yum.repos.d/axosyslog.repo <<< '[axosyslog]
+name=AxoSyslog
+baseurl=https://pkg.axoflow.io/rpm/stable/fedora-$releasever/$basearch
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://pkg.axoflow.io/axoflow-code-signing-pub.asc' > /dev/null
+
+yum update -y
+```
+
+Nightly builds are also available:
+
+```
+tee /etc/yum.repos.d/axosyslog.repo <<< '[axosyslog]
+name=AxoSyslog
+baseurl=https://pkg.axoflow.io/rpm/nightly/fedora-$releasever/$basearch
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://pkg.axoflow.io/axoflow-code-signing-pub.asc' > /dev/null
+```
+
+To install AxoSyslog:
+```
+yum install -y axosyslog
+```
+
+### Extra repos
+
+As the example above showed, EPEL is necessary for AxoSyslog to work.
+Certain AxoSyslog modules need extra dependencies on some of the supported distros.
+
+#### Fedora
+
+```
+dnf install -y dnf-plugins-core
+```
+
+#### AlmaLinux 8
+
+```
+yum install -u yum-plugin-copr
+yum config-manager --set-enabled powertools
+```
+
+#### AlmaLinux 9
+
+```
+dnf config-manager --set-enabled crb
+```
+
 ## Documentation
 
 You can find [comprehensive documentation for AxoSyslog](https://axoflow.com/docs/axosyslog-core)

diff --git a/dbld/Makefile.am b/dbld/Makefile.am
@@ -40,7 +40,6 @@ EXTRA_DIST += \
 	dbld/images/ubuntu-jammy.dockerfile \
 	dbld/images/ubuntu-noble.dockerfile \
 	dbld/images/ubuntu-oracular.dockerfile \
-	dbld/images/ubuntu-lunar.dockerfile \
 	dbld/images/tarball.dockerfile \
 	dbld/images/hooks/build \
 	dbld/images/README.md \

diff --git a/dbld/images/ubuntu-lunar.dockerfile b/dbld/images/ubuntu-lunar.dockerfile
diff --git a/dbld/rules b/dbld/rules
@@ -15,7 +15,6 @@ BUILDER_IMAGES=		\
 	ubuntu-jammy \
 	ubuntu-noble \
 	ubuntu-oracular \
-	ubuntu-lunar	\
 	tarball
 
 IMAGES=${BUILDER_IMAGES} devshell

diff --git a/packaging/package-indexer/Dockerfile b/packaging/package-indexer/Dockerfile
@@ -1,11 +1,15 @@
-FROM ubuntu:focal
+FROM ubuntu:noble
 
 RUN apt-get update && apt-get install -y \
   python3-pip \
   apt-utils \
-  gnupg2
+  gnupg2 \
+  gpg-agent \
+  scdaemon \
+  rpm \
+  createrepo-c
 
-RUN pip install \
+RUN pip install --break-system-packages \
   azure-storage-blob \
   azure-mgmt-cdn \
   azure-identity \

diff --git a/packaging/package-indexer/index-packages.py b/packaging/package-indexer/index-packages.py
@@ -27,7 +27,7 @@
 from sys import stdin
 from typing import List
 
-from indexer import Indexer, NightlyDebIndexer, StableDebIndexer
+from indexer import Indexer, NightlyDebIndexer, StableDebIndexer, NightlyRpmIndexer, StableRpmIndexer
 from config import Config
 
 
@@ -113,10 +113,6 @@ def load_config(args: dict) -> Config:
 def construct_indexers(cfg: Config, args: dict) -> List[Indexer]:
     suite = args["suite"]
 
-    incoming_remote_storage_synchronizer = cfg.create_incoming_remote_storage_synchronizer(suite)
-    indexed_remote_storage_synchronizer = cfg.create_indexed_remote_storage_synchronizer(suite)
-    cdn = cfg.create_cdn(suite)
-
     gpg_key_path = cfg.get_gpg_key_path()
     gpg_key_passphrase = stdin.read() if args["gpg_key_passphrase_from_stdin"] else None
 
@@ -125,9 +121,19 @@ def construct_indexers(cfg: Config, args: dict) -> List[Indexer]:
     if suite == "nightly":
         indexers.append(
             NightlyDebIndexer(
-                incoming_remote_storage_synchronizer=incoming_remote_storage_synchronizer,
-                indexed_remote_storage_synchronizer=indexed_remote_storage_synchronizer,
-                cdn=cdn,
+                incoming_remote_storage_synchronizer=cfg.create_incoming_remote_storage_synchronizer(suite),
+                indexed_remote_storage_synchronizer=cfg.create_indexed_remote_storage_synchronizer(suite),
+                cdn=cfg.create_cdn(suite),
+                run_id=args["run_id"],
+                gpg_key_path=gpg_key_path,
+                gpg_key_passphrase=gpg_key_passphrase,
+            )
+        )
+        indexers.append(
+            NightlyRpmIndexer(
+                incoming_remote_storage_synchronizer=cfg.create_incoming_remote_storage_synchronizer(suite),
+                indexed_remote_storage_synchronizer=cfg.create_indexed_remote_storage_synchronizer(suite),
+                cdn=cfg.create_cdn(suite),
                 run_id=args["run_id"],
                 gpg_key_path=gpg_key_path,
                 gpg_key_passphrase=gpg_key_passphrase,
@@ -136,9 +142,19 @@ def construct_indexers(cfg: Config, args: dict) -> List[Indexer]:
     elif suite == "stable":
         indexers.append(
             StableDebIndexer(
-                incoming_remote_storage_synchronizer=incoming_remote_storage_synchronizer,
-                indexed_remote_storage_synchronizer=indexed_remote_storage_synchronizer,
-                cdn=cdn,
+                incoming_remote_storage_synchronizer=cfg.create_incoming_remote_storage_synchronizer(suite),
+                indexed_remote_storage_synchronizer=cfg.create_indexed_remote_storage_synchronizer(suite),
+                cdn=cfg.create_cdn(suite),
+                run_id=args["run_id"],
+                gpg_key_path=gpg_key_path,
+                gpg_key_passphrase=gpg_key_passphrase,
+            )
+        )
+        indexers.append(
+            StableRpmIndexer(
+                incoming_remote_storage_synchronizer=cfg.create_incoming_remote_storage_synchronizer(suite),
+                indexed_remote_storage_synchronizer=cfg.create_indexed_remote_storage_synchronizer(suite),
+                cdn=cfg.create_cdn(suite),
                 run_id=args["run_id"],
                 gpg_key_path=gpg_key_path,
                 gpg_key_passphrase=gpg_key_passphrase,

diff --git a/packaging/package-indexer/indexer/__init__.py b/packaging/package-indexer/indexer/__init__.py
@@ -22,9 +22,12 @@
 
 from .indexer import Indexer
 from .deb_indexer import StableDebIndexer, NightlyDebIndexer
+from .rpm_indexer import StableRpmIndexer, NightlyRpmIndexer
 
 __all__ = [
     "Indexer",
     "StableDebIndexer",
     "NightlyDebIndexer",
+    "StableRpmIndexer",
+    "NightlyRpmIndexer",
 ]
diff --git a/packaging/package-indexer/indexer/deb_indexer.py b/packaging/package-indexer/indexer/deb_indexer.py
@@ -36,6 +36,7 @@
 from . import utils
 
 CURRENT_DIR = Path(__file__).parent.resolve()
+ARCH = "amd64"
 
 
 class DebIndexer(Indexer):
@@ -67,7 +68,7 @@ def __move_files_from_incoming_to_indexed(self, incoming_dir: Path, indexed_dir:
             platform = relative_path.parent
             file_name = relative_path.name
 
-            new_path = Path(indexed_dir, platform, "binary-amd64", file_name)
+            new_path = Path(indexed_dir, platform, "binary-" + ARCH, file_name)
 
             self._log_info("Moving file.", src_path=str(file), dst_path=str(new_path))
 
@@ -83,7 +84,7 @@ def __create_packages_files(self, indexed_dir: Path) -> None:
         # APT wants to have the `Filename` field in the `Packages` file to start with `dists`.
         dir = indexed_dir.parents[1]
 
-        for pkg_dir in list(indexed_dir.rglob("binary-amd64")):
+        for pkg_dir in list(indexed_dir.rglob("binary-" + ARCH)):
             relative_pkg_dir = pkg_dir.relative_to(dir)
             command = base_command + [str(relative_pkg_dir)]
 
@@ -125,6 +126,7 @@ def __create_release_file(self, indexed_dir: Path) -> None:
     def _index_pkgs(self, indexed_dir: Path) -> None:
         self.__create_packages_files(indexed_dir)
         self.__create_release_file(indexed_dir)
+        self.__sign_pkgs(indexed_dir)
 
     @staticmethod
     def __add_gpg_security_params(command: list) -> list:
@@ -212,7 +214,7 @@ def __create_inrelease_file(self, release_file_path: Path, gnupghome: str) -> No
         )
         utils.execute_command(command, env=env, input=self.__gpg_key_passphrase)
 
-    def _sign_pkgs(self, indexed_dir: Path) -> None:
+    def __sign_pkgs(self, indexed_dir: Path) -> None:
         gnupghome = TemporaryDirectory(dir=CURRENT_DIR)
         release_file_path = Path(indexed_dir, "Release")
 
@@ -238,7 +240,7 @@ def __init__(
         super().__init__(
             incoming_remote_storage_synchronizer=incoming_remote_storage_synchronizer,
             indexed_remote_storage_synchronizer=indexed_remote_storage_synchronizer,
-            incoming_sub_dir=Path("stable", run_id),
+            incoming_sub_dir=Path("stable", run_id, "deb"),
             dist_dir=Path("stable"),
             cdn=cdn,
             apt_conf_file_path=Path(CURRENT_DIR, "apt_conf", "stable.conf"),
@@ -262,7 +264,7 @@ def __init__(
         super().__init__(
             incoming_remote_storage_synchronizer=incoming_remote_storage_synchronizer,
             indexed_remote_storage_synchronizer=indexed_remote_storage_synchronizer,
-            incoming_sub_dir=Path("nightly", run_id),
+            incoming_sub_dir=Path("nightly", run_id, "deb"),
             dist_dir=Path("nightly"),
             cdn=cdn,
             apt_conf_file_path=Path(CURRENT_DIR, "apt_conf", "nightly.conf"),