feat: Add support for Amazon Linux 2 (issue #118)

.github/workflows/build_docker_slurm.yml

@@ -26,7 +26,7 @@ jobs:
         working-directory: tests/integration/docker-slurm
     strategy:
       matrix:
-        os: ["el7", "el8", "el9", "sl15", "ub20", "ub22"]
+        os: ["amzn2", "el7", "el8", "el9", "sl15", "ub20", "ub22"]
     steps:
       - name: Log in to the container registry
         uses: docker/login-action@v3

.github/workflows/test_and_release.yml

@@ -170,6 +170,38 @@ jobs:
           if-no-files-found: error
           retention-days: 5
 
+  build_amzn2:
+    needs: unit_tests
+    permissions:
+      contents: read
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create Amazon Linux 2 RPM
+        run: build-tools/AmazonLinux_2/build.sh
+      - name: Get package name
+        id: pkg_path
+        run: |
+          echo "AMZN2_RPM=$(
+            ls -1 build-tools/AmazonLinux_2/slurm-mail*.rpm
+          )" >> $GITHUB_ENV
+      - name: Create md5sum for Amazon Linux 2 RPM
+        run: md5sum ${{ env.AMZN2_RPM }} > ${{ env.AMZN2_RPM }}.md5sum
+      - name: Store Amazon Linux 2 RPM
+        uses: actions/upload-artifact@v4
+        with:
+          name: amzn2_pkg
+          path: ${{ env.AMZN2_RPM }}
+          if-no-files-found: error
+          retention-days: 5
+      - name: Store Amazon Linux 2 RPM md5sum
+        uses: actions/upload-artifact@v4
+        with:
+          name: amzn2_pkg_md5sum
+          path: ${{ env.AMZN2_RPM }}.md5sum
+          if-no-files-found: error
+          retention-days: 5
+
   build_rhel8:
     needs: unit_tests
     permissions:
@@ -342,6 +374,31 @@ jobs:
           matrix=$(cat ./supported_slurm_versions.json)
           echo "matrix=${matrix}" >> $GITHUB_OUTPUT
 
+  integration_tests_amzn2:
+    needs: [build_amzn2, generate_slurm_matrix]
+    permissions:
+      contents: read
+      packages: read
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        slurm-version: ${{fromJson(needs.generate_slurm_matrix.outputs.matrix)}}
+    steps:
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: 'amzn2_pkg'
+          path: tests/integration/
+      - name: Run integration tests
+        run: |
+          tests/integration/run.sh -s "${{ matrix.slurm-version }}" -o amzn2 -p
+  
   integration_tests_rhel7:
     needs: [build_rhel7, generate_slurm_matrix]
     permissions:
@@ -496,6 +553,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/v')
     needs: [
       build_tar,
+      integration_tests_amzn2,
       integration_tests_rhel7,
       integration_tests_rhel8,
       integration_tests_rhel9,
@@ -511,6 +569,11 @@ jobs:
       - name: create download directory
         run: |
           mkdir downloads
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: 'amzn2_pkg*'
+          path: downloads
+          merge-multiple: true
       - uses: actions/download-artifact@v4
         with:
           pattern: 'rhel7_pkg*'

CHANGELOG.md

@@ -1,6 +1,13 @@
 Change Log
 ==========
 
+Version 4.14
+------------
+
+Date: 2024-03-14
+
+* Add support for Amazon Linux 2 (issue #118)
+
 Version 4.12
 ------------
 

README.md

@@ -64,7 +64,7 @@ You can also opt to include a number of lines from the end of the job's output f
 
 ## Installation
 
-### RedHat and SUSE Based Operating Systems
+### Amazon Linux, RedHat and SUSE Based Operating Systems
 
 For each release of Slurm-Mail, RPMs for RedHat 7/8/9 and SUSE 15 based operating systems are provided at [releases](https://github.com/neilmunday/slurm-mail/releases).
 
@@ -74,6 +74,12 @@ Once downloaded, install using your appropriate package manager, e.g. for Rocky
 dnf localinstall ./slurm-mail-*.noarch.rpm
 ```
 
+Or for RedHat 7 and Amazon Linux 2:
+
+```bash
+yum localinstall ./slurm-mail-*.noarch.rpm
+```
+
 For other operating systems that use RPM packages you can create a package for your OS like so:
 
 ```bash

build-tools/AmazonLinux_2/Dockerfile

@@ -0,0 +1,7 @@
+FROM amazonlinux:2
+
+RUN yum -y install python3 rpm-build tar
+
+RUN mkdir -p /root/slurm-mail
+
+CMD ["/usr/bin/tail", "-f", "/dev/null"]

build-tools/AmazonLinux_2/build.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+#
+#  This file is part of Slurm-Mail.
+#
+#  Slurm-Mail is a drop in replacement for Slurm's e-mails to give users
+#  much more information about their jobs compared to the standard Slurm
+#  e-mails.
+#
+#   Copyright (C) 2018-2024 Neil Munday ([email protected])
+#
+#  Slurm-Mail is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the
+#  Free Software Foundation, either version 3 of the License, or (at
+#  your option) any later version.
+#
+#  Slurm-Mail is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Slurm-Mail.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+NAME="slurm-mail-builder-al2"
+CONTAINER_NAME="${NAME}-${RANDOM}"
+
+cd $DIR
+source ../common.sh
+
+rm -f ./slurm-mail*.rpm
+
+docker build -t ${NAME}:latest .
+
+tmp_file=`mktemp /tmp/XXXXXX.tar.gz`
+echo "Temporary tar file: $tmp_file"
+tar cvfz $tmp_file ../../*
+
+docker run -h slurm-mail-buildhost -d --name ${CONTAINER_NAME} ${NAME}
+docker cp $tmp_file ${CONTAINER_NAME}:$tmp_file
+rm -f $tmp_file
+docker exec ${CONTAINER_NAME} /bin/bash -c "cd /root/slurm-mail && tar xvf $tmp_file"
+docker exec ${CONTAINER_NAME} /bin/bash -c "/root/slurm-mail/build-tools/build-rpm.sh"
+rpm=`docker exec ${CONTAINER_NAME} /bin/bash -c "ls -1 /root/rpmbuild/RPMS/noarch/slurm-mail*.rpm"`
+docker cp ${CONTAINER_NAME}:$rpm .
+
+echo "Created: "`ls -1 slurm-mail*.rpm`
+
+tidyup ${CONTAINER_NAME}

build-tools/amzn2

@@ -0,0 +1 @@
+AmazonLinux_2

src/slurmmail/__init__.py

@@ -45,5 +45,5 @@
     ' compared to the standard Slurm e-mails.'
 MAINTAINER = 'Neil Munday'
 NAME = 'slurmmail'
-VERSION = '4.12'
+VERSION = '4.13'
 URL = 'https://www.github.com/neilmunday/slurm-mail'

tests/integration/Dockerfile.slurm-mail.amzn2

@@ -0,0 +1,36 @@
+ARG SLURM_VER
+
+FROM ghcr.io/neilmunday/slurm-mail/slurm-amzn2:$SLURM_VER
+
+ARG SMTP_SERVER=localhost
+ARG SMTP_PORT=25
+ARG DISABLE_CRON=0
+ARG SLURM_MAIL_PKG
+
+RUN yum install -y chrony cronie python3 && \
+    pip3 install aiosmtpd && \
+    pip3 install pyyaml && \
+    mkdir -p /root/testing/output && \
+    sed -i "s#MailProg=/usr/bin/mailx#MailProg=/usr/bin/slurm-spool-mail#" /etc/slurm/slurm.conf && \
+    echo "alias ll='ls -l'" >> /root/.bashrc
+
+COPY supervisord.conf /etc/supervisord.conf
+
+COPY mail-server.py /usr/local/sbin/
+
+RUN chmod 0700 /usr/local/sbin/mail-server.py
+
+COPY run-tests.py tests.yml /root/testing/
+
+COPY $SLURM_MAIL_PKG /root/
+
+RUN yum localinstall -y /root/$SLURM_MAIL_PKG && \
+    sed -i "s/includeOutputLines = 0/includeOutputLines = 20/" /etc/slurm-mail/slurm-mail.conf && \
+    sed -i "s/verbose = false/verbose = true/" /etc/slurm-mail/slurm-mail.conf && \
+    sed -i "s/smtpServer = localhost/smtpServer = $SMTP_SERVER/" /etc/slurm-mail/slurm-mail.conf && \
+    sed -i "s/smtpPort = 25/smtpPort = $SMTP_PORT/" /etc/slurm-mail/slurm-mail.conf && \
+    rm -f /root/$SLURM_MAIL_PKG
+
+RUN if [ "$DISABLE_CRON" = 1 ]; then rm /etc/cron.d/slurm-mail; fi
+
+CMD ["tail -f /dev/null"]

tests/integration/docker-slurm/Dockerfile.amzn2

@@ -0,0 +1,57 @@
+FROM amazonlinux:2
+
+LABEL org.opencontainers.image.source="https://github.com/neilmunday/slurm-mail" \
+      org.opencontainers.image.description="An Amazon Linux 2 Slurm container intended for testing Slurm-Mail" \
+      org.opencontainers.image.title="amzn2-slurm" \
+      maintainer="Neil Munday"
+
+ARG SLURM_VER=23.11.4
+
+# download, build, install and clean-up
+RUN yum update -y && \
+    amazon-linux-extras install epel -y && \
+    yum install -y \
+    gcc \
+    mailx \
+    make \
+    mariadb-devel \
+    mariadb-server \
+    munge-devel \
+    pam-devel \
+    perl \
+    perl-ExtUtils-MakeMaker \
+    python3 \
+    readline-devel \
+    rpm-build \
+    supervisor \
+    tini \
+    wget && \
+    wget https://download.schedmd.com/slurm/slurm-${SLURM_VER}.tar.bz2 -O /root/slurm-${SLURM_VER}.tar.bz2 && \
+    rpmbuild -tb /root/slurm-${SLURM_VER}.tar.bz2 && \
+    yum localinstall -y /root/rpmbuild/RPMS/x86_64/slurm-${SLURM_VER}*.amzn2.x86_64.rpm \
+    /root/rpmbuild/RPMS/x86_64/slurm-slurmctld-${SLURM_VER}*.amzn2.x86_64.rpm \
+    /root/rpmbuild/RPMS/x86_64/slurm-slurmd-${SLURM_VER}*.amzn2.x86_64.rpm \
+    /root/rpmbuild/RPMS/x86_64/slurm-slurmdbd-${SLURM_VER}*.amzn2.x86_64.rpm && \
+    yum -y erase gcc mariab-devel make munge-devel pam-devel readline-devel rpm-build wget && \
+    yum clean all && \
+    rm -rf /root/rpmbuild /root/slurm*.tar.bz2 && \
+    groupadd -r slurm && \
+    useradd -r -g slurm -d /var/empty/slurm -m -s /bin/bash slurm && \
+    groupadd test && \
+    useradd -g test -d /home/test -m test && \
+    install -d -o slurm -g slurm /etc/slurm /var/spool/slurm /var/log/slurm
+
+COPY supervisord.conf /etc/
+COPY --chown=slurm slurm.*.conf /etc/slurm/
+COPY --chown=slurm slurmdbd.conf /etc/slurm/
+COPY --chown=root entrypoint.sh /usr/local/sbin/
+
+RUN MAJOR_VER=`echo ${SLURM_VER} | egrep -o "^[0-9]+"` && \
+  mv /etc/slurm/slurm.${MAJOR_VER}.conf /etc/slurm/slurm.conf && \
+  rm -f /etc/slurm/slurm.*.conf && \
+  chmod 600 /etc/slurm/slurm*.conf
+
+RUN /usr/bin/mysql_install_db --user=mysql
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/sbin/entrypoint.sh"]
+CMD ["tail -f /dev/null"]

tests/integration/docker-slurm/entrypoint.sh

@@ -36,6 +36,8 @@ fi
 mysql -e "CREATE DATABASE IF NOT EXISTS slurm_acct_db;" || die "failed to create database"
 if [ -e /etc/redhat-release ] && grep -q 'release 7' /etc/redhat-release; then
   mysql -e "GRANT ALL ON slurm_acct_db.* TO 'slurm'@'localhost' IDENTIFIED BY 'password'" || die "failed to create slurm mysql user"
+elif [ -e /etc/system-release ] && grep -q 'Amazon Linux release 2' /etc/system-release; then
+  mysql -e "GRANT ALL ON slurm_acct_db.* TO 'slurm'@'localhost' IDENTIFIED BY 'password'" || die "failed to create slurm mysql user"
 else
   mysql -e "CREATE USER IF NOT EXISTS 'slurm'@'localhost' identified by 'password';" || die "failed to create slurm mysql user"
 fi

tests/integration/run.sh

@@ -24,7 +24,8 @@
 
 function catch {
   if [ "$1" != "0" ]; then
-    #echo "Error $1 occurred on line $2" 1>&2
+    echo "Error $1 occurred" 1>&2
+    docker container logs $NAME
     tidyup $NAME
   fi
 }
@@ -147,7 +148,7 @@ docker exec $NAME /bin/bash -c \
 if [ $PKG_EXT == ".rpm" ]; then
   if [[ $OS == sl* ]]; then
     docker exec $NAME /bin/bash -c "zypper remove -y slurm-mail"
-  elif [ $OS == "el7" ]; then
+  elif [ $OS == "el7" ] || [ $OS == "amzn2" ]; then
     docker exec $NAME /bin/bash -c "yum erase -y slurm-mail"
   else
     docker exec $NAME /bin/bash -c "dnf erase -y slurm-mail"