From 58e204ea15ad4646807fb338da29c23634bece63 Mon Sep 17 00:00:00 2001 From: oxpa Date: Fri, 19 Apr 2024 22:26:24 +0100 Subject: [PATCH 1/6] Rename CI file and make it work for pull requests only --- .github/workflows/{ci.yml => ci-pull.yml} | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) rename .github/workflows/{ci.yml => ci-pull.yml} (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci-pull.yml similarity index 99% rename from .github/workflows/ci.yml rename to .github/workflows/ci-pull.yml index 4de8a3b60..04a2bd7b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci-pull.yml @@ -1,10 +1,8 @@ name: ci on: + workflow_dispatch: pull_request: - push: - branches: - - master jobs: test: From d4b218b01e477b1fe7eaf96a4e190bc2aaefc8c3 Mon Sep 17 00:00:00 2001 From: oxpa Date: Fri, 19 Apr 2024 22:26:57 +0100 Subject: [PATCH 2/6] Add package lists and a script to install them into workers --- .github/scripts/pkglists/alpine | 78 +++++++ .github/scripts/pkglists/alpine-3.18 | 6 + .github/scripts/pkglists/alpine-3.19 | 20 ++ .github/scripts/pkglists/amazonlinux | 45 ++++ .github/scripts/pkglists/amazonlinux-2023 | 46 ++++ .github/scripts/pkglists/debian | 70 ++++++ .github/scripts/pkglists/debian-11 | 32 +++ .github/scripts/pkglists/debian-12 | 26 +++ .github/scripts/pkglists/rhel | 47 ++++ .github/scripts/pkglists/rhel-8 | 33 +++ .github/scripts/pkglists/rhel-9 | 31 +++ .github/scripts/pkglists/ubuntu | 71 ++++++ .github/scripts/pkglists/ubuntu-22.04 | 28 +++ .github/scripts/pkglists/ubuntu-23.10 | 28 +++ .github/scripts/setup.sh | 260 ++++++++++++++++++++++ 15 files changed, 821 insertions(+) create mode 100644 .github/scripts/pkglists/alpine create mode 100644 .github/scripts/pkglists/alpine-3.18 create mode 100644 .github/scripts/pkglists/alpine-3.19 create mode 100644 .github/scripts/pkglists/amazonlinux create mode 100644 .github/scripts/pkglists/amazonlinux-2023 create mode 100644 .github/scripts/pkglists/debian create mode 100644 .github/scripts/pkglists/debian-11 create mode 100644 .github/scripts/pkglists/debian-12 create mode 100644 .github/scripts/pkglists/rhel create mode 100644 .github/scripts/pkglists/rhel-8 create mode 100644 .github/scripts/pkglists/rhel-9 create mode 100644 .github/scripts/pkglists/ubuntu create mode 100644 .github/scripts/pkglists/ubuntu-22.04 create mode 100644 .github/scripts/pkglists/ubuntu-23.10 create mode 100755 .github/scripts/setup.sh diff --git a/.github/scripts/pkglists/alpine b/.github/scripts/pkglists/alpine new file mode 100644 index 000000000..aea55d020 --- /dev/null +++ b/.github/scripts/pkglists/alpine @@ -0,0 +1,78 @@ +alpine-sdk +automake +boost-dev +bzip2-dev +brotli-dev +cmake +coreutils +curl +c-ares-dev +dhcpcd +e2fsprogs-extra +expect +findutils +flex +g++ +gawk +gd-dev +gdb +geoip-dev +gettext +go +json-c-dev +krb5-dev +libaio-dev +libedit-dev +libmaxminddb-dev +libmemcached-dev +libtool +libxml2-dev +libxslt-dev +linux-headers +linux-pam-dev +luajit-dev +memcached +mercurial +musl-dbg +nghttp2-dev +openssl +openssl-dev +patchelf +perl-dev +procps +py-twisted +python3-dev +re2-dev +rsync +ruby-dev +ruby-etc +ruby-rake +screen +shadow +sudo +tcpdump +tmux +vim +wget +yajl-dev +yaml +yaml-cpp-dev +zsh +nodejs +nodejs-dev +npm +xz +py3-openssl +pcre2-dev +bash +perl-parse-recdescent +libxml2-utils +perl-cryptx +perl-fcgi +perl-io-socket-ssl +py3-pyhamcrest +py3-pytest +musl-dev +cargo +clang +clang-libclang diff --git a/.github/scripts/pkglists/alpine-3.18 b/.github/scripts/pkglists/alpine-3.18 new file mode 100644 index 000000000..20ec176a1 --- /dev/null +++ b/.github/scripts/pkglists/alpine-3.18 @@ -0,0 +1,6 @@ +curl-dev +libssh2-dev +openjdk11 +py3-pip +php82-dev +php82-embed diff --git a/.github/scripts/pkglists/alpine-3.19 b/.github/scripts/pkglists/alpine-3.19 new file mode 100644 index 000000000..1443e344f --- /dev/null +++ b/.github/scripts/pkglists/alpine-3.19 @@ -0,0 +1,20 @@ +curl-dev +libssh2-dev +openjdk11-jre-headless +openjdk17-jre-headless +openjdk18-jre-headless +openjdk20-jre-headless +openjdk21-jre-headless +openjdk11-jdk +openjdk17-jdk +openjdk18-jdk +openjdk20-jdk +openjdk21-jdk +py3-pip +php83-dev +php83-embed +php83 +php82 +php82-dev +php82-embed +coreutils-sha512sum diff --git a/.github/scripts/pkglists/amazonlinux b/.github/scripts/pkglists/amazonlinux new file mode 100644 index 000000000..9000d1e83 --- /dev/null +++ b/.github/scripts/pkglists/amazonlinux @@ -0,0 +1,45 @@ +boost-devel +cmake +c-ares-devel +flex +gcc-c++ +gd-devel +gdb +git +golang +httpd-devel +json-c-devel +libcurl-devel +libedit-devel +libmaxminddb-devel +libtool +libuuid-devel +libxcb-devel +libxml2-devel +libxslt-devel +lua-devel +memcached +mercurial +patchelf +pcre-devel +perl-ExtUtils-Embed +perl-IO-Socket-SSL +perl-JSON +perl-Test-Simple +php-devel +rpm-build +rpmlint +re2-devel +rsync +ruby-devel +rubygem-rake +tcpdump +tcsh +tmux +vim-enhanced +zsh +pcre2-devel +perl-Parse-RecDescent +perl-CryptX +clang +llvm diff --git a/.github/scripts/pkglists/amazonlinux-2023 b/.github/scripts/pkglists/amazonlinux-2023 new file mode 100644 index 000000000..f34a5ef09 --- /dev/null +++ b/.github/scripts/pkglists/amazonlinux-2023 @@ -0,0 +1,46 @@ +bison +brotli-devel +bzip2-devel +desktop-file-utils +elfutils +enchant +gd-progs +gdb-headless +libX11-xcb +libbabeltrace +libimagequant-devel +libtiff-devel +libwebp-devel +lsof +ncurses-c++-libs +net-tools +nmap-ncat +perl-Digest-MD5 +perl-IO-Compress +perl-JSON-PP +protobuf-c-devel +python3-devel +python3-enchant +python3-pip +python3-pytest +rpm-sign +sysstat +yajl-devel +zstd +nodejs +nodejs-npm +python3-Automat +libstdc++-static +openssl-devel +python3-pyOpenSSL +perl-GD +screen +perl-macros +ruby3.2-rubygem-rake +php-embedded +yaml-cpp-devel +perl-FindBin +perl-Sys-Hostname +java-17-amazon-corretto-devel +java-21-amazon-corretto-devel +python3.11-devel diff --git a/.github/scripts/pkglists/debian b/.github/scripts/pkglists/debian new file mode 100644 index 000000000..1eca10804 --- /dev/null +++ b/.github/scripts/pkglists/debian @@ -0,0 +1,70 @@ +automake +bash +build-essential +ca-certificates +curl +debhelper +devscripts +expect +fakeroot +flex +gdb +git +less +libboost-dev +libcache-memcached-fast-perl +libcache-memcached-perl +libcurl4-openssl-dev +libc-ares-dev +libedit-dev +libfcgi-perl +libgd-perl +libgd-dev +libgeoip-dev +libio-socket-ssl-perl +libjson-c-dev +libkrb5-dev +liblua5.1-0-dev +libmemcached-dev +libpcre3-dev +libperl-dev +libre2-dev +libssl-dev +libtool +libyaml-cpp-dev +libxml2-utils +libxslt1-dev +lintian +lsb-release +memcached +mercurial +openssh-server +patchelf +quilt +rake +reprepro +rsync +ruby-dev +screen +sudo +tcpdump +tcsh +time +tmux +udev +unzip +vim +wget +xsltproc +zlib1g-dev +zsh +python3-dev +python3-pip +libffi-dev +libpcre2-dev +libparse-recdescent-perl +libcryptx-perl +dh-exec +clang +llvm +python3-pytest diff --git a/.github/scripts/pkglists/debian-11 b/.github/scripts/pkglists/debian-11 new file mode 100644 index 000000000..5d12ddb9d --- /dev/null +++ b/.github/scripts/pkglists/debian-11 @@ -0,0 +1,32 @@ +apache2-dev +cmake +dh-python +ffmpeg +firefox-esr +gcc-multilib-i686-linux-gnu +g++-multilib-i686-linux-gnu +golang-1.15 +libbrotli-dev +libmaxminddb-dev +libphp-embed +libprotobuf-c-dev +libyajl-dev +libyajl2 +openjdk-11-jdk-headless +openjdk-11-jre-headless +openjdk-17-jdk-headless +openjdk-17-jre-headless +php-dev +python3.9-dev +python3-psutil +ruby-rack +yajl-tools +golang-go +golang +libclang-11-dev +llvm-11-dev +clang-11 +python2-dev +python2.7-dev +python3-dev +uwsgi-plugin-python3 diff --git a/.github/scripts/pkglists/debian-12 b/.github/scripts/pkglists/debian-12 new file mode 100644 index 000000000..e27b5a782 --- /dev/null +++ b/.github/scripts/pkglists/debian-12 @@ -0,0 +1,26 @@ +apache2-dev +cmake +dh-python +ffmpeg +firefox-esr +golang-1.19 +libbrotli-dev +libmaxminddb-dev +libphp-embed +libprotobuf-c-dev +libyajl-dev +libyajl2 +openjdk-17-jdk-headless +openjdk-17-jre-headless +php-dev +python3.11-dev +python3-psutil +ruby-rack +yajl-tools +golang-go +golang +libclang-14-dev +llvm-14-dev +clang-14 +python3-dev +uwsgi-plugin-python3 diff --git a/.github/scripts/pkglists/rhel b/.github/scripts/pkglists/rhel new file mode 100644 index 000000000..bec22fff8 --- /dev/null +++ b/.github/scripts/pkglists/rhel @@ -0,0 +1,47 @@ +boost-devel +cmake +c-ares-devel +flex +gcc-c++ +gd-devel +gdb +git +golang +httpd-devel +json-c-devel +libcurl-devel +libedit-devel +libmaxminddb-devel +libtool +libuuid-devel +libxcb-devel +libxml2-devel +libxslt-devel +lua-devel +memcached +mercurial +patchelf +pcre-devel +perl-ExtUtils-Embed +perl-FCGI +perl-IO-Socket-SSL +perl-JSON +perl-Test-Simple +php-devel +rpm-build +rpmlint +re2-devel +rsync +ruby-devel +rubygem-rake +tcpdump +tcsh +tmux +vim-enhanced +zsh +pcre2-devel +perl-Parse-RecDescent +perl-CryptX +clang +llvm +python3-pytest diff --git a/.github/scripts/pkglists/rhel-8 b/.github/scripts/pkglists/rhel-8 new file mode 100644 index 000000000..76a7608a6 --- /dev/null +++ b/.github/scripts/pkglists/rhel-8 @@ -0,0 +1,33 @@ +brotli-devel +java-1.8.0-openjdk-devel +java-11-openjdk-devel +perl-Digest-MD5 +perl-IO-Compress +perl-JSON-PP +protobuf-c-devel +python2-devel +python3-pip +python36-devel +rpm-sign +selinux-policy-devel +yajl-devel +nodejs +npm +libstdc++-static +openssl-devel +perl-IPC-Cmd +python3-pyOpenSSL +libmemcached-devel +perl-Cache-Memcached +perl-GD +screen +yaml-cpp-devel +php-embedded +redhat-lsb-core +perl-bignum +python38-devel +python39-devel +libacl-devel +libyaml-devel +lmdb-devel +pam-devel diff --git a/.github/scripts/pkglists/rhel-9 b/.github/scripts/pkglists/rhel-9 new file mode 100644 index 000000000..10e333c50 --- /dev/null +++ b/.github/scripts/pkglists/rhel-9 @@ -0,0 +1,31 @@ +brotli-devel +java-1.8.0-openjdk-devel +java-11-openjdk-devel +java-17-openjdk-devel +java-21-openjdk-devel +perl-Digest-MD5 +perl-IO-Compress +perl-JSON-PP +protobuf-c-devel +python3-pip +python3-devel +rpm-sign +selinux-policy-devel +yajl-devel +nodejs +npm +libstdc++-static +openssl-devel +perl-IPC-Cmd +python3-pyOpenSSL +perl-macros +perl-bignum +yaml-cpp-devel +perl +php-embedded +lksctp-tools-devel +libacl-devel +libyaml-devel +lmdb-devel +pam-devel +libunwind-devel diff --git a/.github/scripts/pkglists/ubuntu b/.github/scripts/pkglists/ubuntu new file mode 100644 index 000000000..1d06bddc2 --- /dev/null +++ b/.github/scripts/pkglists/ubuntu @@ -0,0 +1,71 @@ +automake +bash +build-essential +ca-certificates +curl +debhelper +devscripts +expect +fakeroot +flex +gdb +git +less +libboost-dev +libcache-memcached-fast-perl +libcache-memcached-perl +libcurl4-openssl-dev +libc-ares-dev +libedit-dev +libfcgi-perl +libgd-perl +libgd-dev +libgeoip-dev +libio-socket-ssl-perl +libjson-c-dev +libkrb5-dev +liblua5.1-0-dev +libmemcached-dev +libpcre3-dev +libperl-dev +libre2-dev +libssl-dev +libtool +libyaml-cpp-dev +libxml2-utils +libxslt1-dev +lintian +lsb-release +memcached +mercurial +openssh-server +patchelf +quilt +rake +reprepro +rsync +ruby-dev +screen +sudo +tcpdump +tcsh +time +tmux +udev +unzip +vim +wget +xsltproc +zlib1g-dev +zsh +python3-dev +python3-pip +python3-pytest +libffi-dev +libpcre2-dev +libparse-recdescent-perl +libcryptx-perl +dh-exec +clang +llvm +cargo diff --git a/.github/scripts/pkglists/ubuntu-22.04 b/.github/scripts/pkglists/ubuntu-22.04 new file mode 100644 index 000000000..223e9e40c --- /dev/null +++ b/.github/scripts/pkglists/ubuntu-22.04 @@ -0,0 +1,28 @@ +apache2-dev +apt-transport-https +cmake +ffmpeg +golang +golang-1.17 +golang-1.18 +libbrotli-dev +libgd-dev +libmaxminddb-dev +libphp-embed +libprotobuf-c-dev +libyajl-dev +openjdk-11-jdk-headless +openjdk-11-jre-headless +openjdk-17-jdk-headless +openjdk-17-jre-headless +openjdk-18-jdk-headless +openjdk-18-jre-headless +openjdk-21-jre-headless +openjdk-21-jdk-headless +policykit-1 +php-dev +python2.7-dev +python3.10-dev +ruby-rack +yajl-tools +uwsgi-plugin-python3 diff --git a/.github/scripts/pkglists/ubuntu-23.10 b/.github/scripts/pkglists/ubuntu-23.10 new file mode 100644 index 000000000..4dca9d317 --- /dev/null +++ b/.github/scripts/pkglists/ubuntu-23.10 @@ -0,0 +1,28 @@ +apache2-dev +apt-transport-https +cmake +ffmpeg +golang +libbrotli-dev +libgd-dev +libmaxminddb-dev +libphp-embed +libprotobuf-c-dev +libyajl-dev +openjdk-11-jdk-headless +openjdk-11-jre-headless +openjdk-17-jdk-headless +openjdk-17-jre-headless +openjdk-19-jdk-headless +openjdk-19-jre-headless +openjdk-20-jdk-headless +openjdk-20-jre-headless +openjdk-21-jdk-headless +openjdk-21-jre-headless +policykit-1 +php-dev +python3.11-dev +python3.12-dev +ruby-rack +yajl-tools +uwsgi-plugin-python3 diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh new file mode 100755 index 000000000..297811a91 --- /dev/null +++ b/.github/scripts/setup.sh @@ -0,0 +1,260 @@ +#/bin/sh -x + +DIR=`dirname $0` +DISTRO="$1"; shift; + +# package management helpers +PACKAGE_MANAGER_INSTALL_yum="yum --setopt=skip_missing_names_on_install=False install -y" +PACKAGE_MANAGER_REMOVE_yum="yum erase -y" +PACKAGE_DEBUG_SUFFIX_yum="debuginfo" +PACKAGE_DEVEL_SUFFIX_yum="devel" +PACKAGE_MANAGER_INSTALL_apt="apt-get install -y --force-yes -q " +PACKAGE_MANAGER_REMOVE_apt="apt-get remove -y --force-yes -q --purge" +PACKAGE_DEBUG_SUFFIX_apt="dbg" +PACKAGE_DEVEL_SUFFIX_apt="dev" +LANGUAGES_COMMON="go perl php" +PACKAGES_PREREQUISITES="curl git" + +case "$DISTRO" in + amazon-lts) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON wasm" + python_versions="2.7 3.7" + java_versions="1.8" + distroversion="amzn/2" + ;; + amazonlinux-2023) + PACKAGES_PREREQUISITES="git" + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON wasm" + python_versions="3.9 3.11" + java_versions="17" + distroversion="amzn/2023" + ;; + centos-7|centos-74|rhel-7|ol-7) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON" + python_versions="2.7 3.6" + java_versions="1.8 11" + distroversion="centos/7" + ;; + rhel-8|alma-8|rocky-8|ol-8) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON wasm" + python_versions="2.7 3.6 3.8 3.9" + java_versions="1.8 11" + distroversion="centos/8" + ;; + rhel-9|alma-9|rocky-9|ol-9) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON wasm" + python_versions="3.9" + java_versions="1.8 11" + distroversion="centos/9" + ;; + fedora-38) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="3.11" + java_versions="1.8 11" + distroversion="fedora/38" + ;; + fedora-39) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_yum + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_yum + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_yum" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_yum" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="3.12" + java_versions="17" + distroversion="fedora/39" + ;; + ubuntu-18.04) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.6 3.7 3.8" + java_versions="1.8 11" + distroversion="ubuntu/18.04" + ;; + ubuntu-20.04) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.8" + java_versions="11" + distroversion="ubuntu/20.04" + ;; + ubuntu-21.10) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.9 3.10" + java_versions="11 16 17 18" + distroversion="ubuntu/21.10" + ;; + ubuntu-22.04) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.10" + java_versions="11 17 18" + distroversion="ubuntu/22.04" + ;; + ubuntu-22.10) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.10" + java_versions="11 17 18 19" + distroversion="ubuntu/22.10" + ;; + ubuntu-23.04) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="3.11" + java_versions="11 17 18 19 20" + distroversion="ubuntu/23.04" + ;; + ubuntu-23.10) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="3.12" + java_versions="11 17 19 20 21" + distroversion="ubuntu/23.10" + ;; + debian-10) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.7" + java_versions="11" + distroversion="debian/10" + ;; + debian-11) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="2.7 3.9" + java_versions="11" + distroversion="debian/11" + ;; + debian-12) + PACKAGE_MANAGER_INSTALL=$PACKAGE_MANAGER_INSTALL_apt + PACKAGE_MANAGER_REMOVE=$PACKAGE_MANAGER_REMOVE_apt + PACKAGE_DEBUG_SUFFIX="$PACKAGE_DEBUG_SUFFIX_apt" + PACKAGE_DEVEL_SUFFIX="$PACKAGE_DEVEL_SUFFIX_apt" + PACKAGES_PREREQUISITES="curl apt-transport-https lsb-release ca-certificates" + languages="$LANGUAGES_COMMON ruby wasm" + python_versions="3.11" + java_versions="17" + distroversion="debian/12" + ;; + alpine*) + PACKAGE_MANAGER_INSTALL="apk add " + PACKAGE_MANAGER_REMOVE="apk del " + ;; + + *) + echo "Unknown distro: $DISTRO, exiting" + exit 1 + ;; +esac + +x_wait_apt_lock() { + if [ -f "/var/lib/apt/lists/lock" ]; then + while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do + echo "Waiting for apt lists lock to be free" + sleep 5 + done + while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + echo "Waiting for dpkg lock to be free" + sleep 5 + done + fi +} +# try to install the packages for five times +# sleep for five seconds between attempts +x_install_package() +{ + if [ -f "/var/lib/apt/lists/lock" ]; then + x_wait_apt_lock + export DEBIAN_FRONTEND=noninteractive + apt-get update -y --no-allow-insecure-repositories + systemctl stop unattended-upgrades.service || : + systemctl disable unattended-upgrades.service || : + fi + case "$DISTRO" in + alpine*) + apk update + ;; + esac + retry=1 + try=1 + while [ -n "$retry" ] && [ "$try" -le 5 ]; do + retry= + if $PACKAGE_MANAGER_INSTALL $@; then + rc=0 + else + rc=$? + try=$((try + 1)) + retry=1 + x_wait_apt_lock + sleep 1 + continue + fi + done + return "$rc" +} + +family=`echo $DISTRO | cut -f 1 -d -` +PKGS=`cat $DIR/pkglists/$family $DIR/pkglists/$DISTRO` +x_install_package $PACKAGES_PREREQUISITES $PKGS + From acd515c959fdbe59e64bdfc5782a2d500d81cd83 Mon Sep 17 00:00:00 2001 From: oxpa Date: Fri, 19 Apr 2024 22:27:45 +0100 Subject: [PATCH 3/6] Replicate buildbot workflow for commits into master --- .github/workflows/ci-commit.yml | 345 ++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 .github/workflows/ci-commit.yml diff --git a/.github/workflows/ci-commit.yml b/.github/workflows/ci-commit.yml new file mode 100644 index 000000000..e1eb55580 --- /dev/null +++ b/.github/workflows/ci-commit.yml @@ -0,0 +1,345 @@ +name: ci + +on: + workflow_dispatch: + push: + branches: + - master + +jobs: + test: + name: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.build }} + runs-on: ${{ matrix.os }}-${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + - ubuntu-23.10 + - debian-11 + - debian-12 + - alpine-3.19 + - rhel-9 + - amazonlinux-2023 + arch: + - amd64 + - arm64 + build: + # Core + - unit + # Modules + - go-1.21 + - go-1.22 + - java-17 + - java-18 + - java-21 + - node-20 + - node-21 + - perl + - php-8.2 + - php-8.3 + - python-3.9 + - python-3.10 + - python-3.11 + - python-3.12 + - ruby + - wasm + - wasm-wasi-component + + steps: + - uses: actions/checkout@v4 + + # Provides module, language version and testpath from build name + - name: Output build metadata + id: metadata + run: | + if [ "${{ matrix.build }}" = "wasm-wasi-component" ]; then + module="wasm-wasi-component" + else + # Split the build name by '-' into module and version + IFS='-' read -r module version <<< "${{ matrix.build }}" + fi + + testpath="test/test_${module}*" + + # Run all tests for "unit" and "python" + # Python is the default module for tests + if [ "$module" = "unit" ] || [ "$module" = "python" ]; then + testpath="test" + fi + + echo "module=${module}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "testpath=${testpath}" >> "$GITHUB_OUTPUT" + + NJS_VERSION=$(sed -n "s/NJS_VERSION := \(.*\)/\1/p" pkg/contrib/src/njs/version) + echo "njs_version=${NJS_VERSION}" >> "$GITHUB_OUTPUT" + + cat "$GITHUB_OUTPUT" + + - name: Setup requirements + id: setup_reqs + run: | + set -x + sudo .github/scripts/setup.sh ${{ matrix.os }} + + ## + ## njs + ## + + - name: Clone njs repository + uses: actions/checkout@v4 + with: + repository: nginx/njs + ref: '${{ steps.metadata.outputs.njs_version }}' + path: njs + + - name: Make njs + run: | + ./configure --no-libxml2 --no-zlib + make -j4 -k + working-directory: njs + + ## + ## Unit + ## + + - name: Configure unit + run: | + ./configure \ + --tests \ + --openssl \ + --njs \ + --cc-opt="-I njs/src/ -I njs/build" \ + --ld-opt="-L njs/build" + + - name: Make unit + run: | + make -j4 -k || make + + ## + ## Go + ## + + - uses: actions/setup-go@v5 + with: + go-version: '${{ steps.metadata.outputs.version }}' + cache: false + if: steps.metadata.outputs.module == 'go' + + - name: Configure go + run: | + ./configure go --go-path= + if: steps.metadata.outputs.module == 'go' + + - name: Make go + run: | + make go + make go-install + if: steps.metadata.outputs.module == 'go' + + ## + ## Java + ## + - name: Configure java + id: setupjava + run: | + set -x + JVERSION=${{ steps.metadata.outputs.version }} + JC=`find /usr/lib/jvm -name javac | grep $JVERSION | head -1` + JHOME=`dirname "$JC"`; JHOME=`dirname "$JHOME"`; #strips /bin/javac + SUM=`which sha512sum` + if [ "$JHOME" != "." -a -d "$JHOME" ]; then + echo java=true >> $GITHUB_OUTPUT + ./configure java --home=$JHOME --sha512=$SUM + fi + + if: steps.metadata.outputs.module == 'java' + + - name: Make java + run: | + make java + if: steps.metadata.outputs.module == 'java' && steps.setupjava.outputs.java == 'true' + + ## + ## Node + ## + + - uses: actions/setup-node@v4 + with: + node-version: '${{ steps.metadata.outputs.version }}' + if: steps.metadata.outputs.module == 'node' && matrix.os != 'alpine-3.19' + + - name: Install node-gyp + run: | + npm install -g node-gyp + if: steps.metadata.outputs.module == 'node' && matrix.os != 'alpine-3.19' + + - name: Configure node + run: | + ./configure nodejs + if: steps.metadata.outputs.module == 'node' && matrix.os != 'alpine-3.19' + + - name: Make node + run: | + make node-local-install DESTDIR=node + if: steps.metadata.outputs.module == 'node' && matrix.os != 'alpine-3.19' + + ## + ## Perl + ## + + # Uses default Actions VM Perl + # https://github.com/actions/runner-images#available-images + + - name: Configure perl + run: | + ./configure perl + if: steps.metadata.outputs.module == 'perl' + + - name: Make perl + run: | + make perl + if: steps.metadata.outputs.module == 'perl' + + ## + ## PHP + ## + + - name: Configure php + id: setupphp + run: | + + PHPV=`echo ${{ steps.metadata.outputs.version }}` + for V in $PHPV `echo $PHPV | tr -d .` ""; do + PHPC=`which php-config$V || echo -n ` + test -n "$PHPC" && break + done; + if [ "${{ matrix.os }}" = "alpine-3.19" ]; then + V=`echo $PHPV | tr -d .` + sudo ln -s /usr/lib/libphp$V.so /usr/lib/libphp.so + fi ; + if test -n $PHPC && $PHPC --version | grep -qF "$PHPV"; then + ./configure php --config=$PHPC && echo "php=true" > $GITHUB_OUTPUT + else + echo 'php=false' > $GITHUB_OUTPUT + fi + + + if: steps.metadata.outputs.module == 'php' + + - name: Make php + run: | + make php + if: steps.metadata.outputs.module == 'php' && steps.setupphp.outputs.php == 'true' + + ## + ## Python 3 + ## + + - name: Configure python3 + id: setuppython + run: | + set -x + PYTHONV=${{ steps.metadata.outputs.version }} + for V in $PYTHONV `echo $PYTHONV | tr -d .` ""; do + PCONF=`which python$V-config || echo -n` + PYTH=`which python$V || echo -n` + test -n "$PCONF" && break + done + if test -n "$PYTH" && $PYTH --version | grep -qF "$PYTHONV"; then + ./configure python --config=$PCONF && echo 'python=true' >> $GITHUB_OUTPUT + fi + if: steps.metadata.outputs.module == 'python' + + - name: Make python3 + run: | + make python${{ steps.metadata.outputs.version }} + if: steps.metadata.outputs.module == 'python' && steps.setuppython.outputs.python == 'true' + + ## + ## Ruby + ## + + - name: Configure ruby + id: setupruby + run: | + if which ruby; then + gem install --user-install rack # tests need rack + ./configure ruby + echo 'ruby=true' >> $GITHUB_OUTPUT + fi + if: steps.metadata.outputs.module == 'ruby' + + - name: Make ruby + run: | + make ruby + if: steps.metadata.outputs.module == 'ruby' && steps.setupruby.outputs.ruby == 'true' + + ## + ## Rust + ## + + - name: Setup rust + run: | + if ! which cargo; then + curl -sSf https://sh.rustup.rs | sh -s -- -y + fi + if: steps.metadata.outputs.module == 'wasm-wasi-component' || steps.metadata.outputs.module == 'wasm' + + ## + ## Wasm + ## + + - name: Make wasmtime + run: | + which cargo || source $HOME/.cargo/env && which cargo + make -C pkg/contrib .wasmtime + if: steps.metadata.outputs.module == 'wasm' + + - name: Configure wasm + run: | + which cargo || source $HOME/.cargo/env && which cargo + ./configure wasm --include-path=pkg/contrib/wasmtime/crates/c-api/include --lib-path=pkg/contrib/wasmtime/target/release + if: steps.metadata.outputs.module == 'wasm' + + - name: Make wasm + run: | + which cargo || source $HOME/.cargo/env && which cargo + make wasm + if: steps.metadata.outputs.module == 'wasm' + + ## + ## wasm-wasi-component + ## + + - name: Configure wasm-wasi-component + run: | + which cargo || source $HOME/.cargo/env && which cargo + ./configure wasm-wasi-component + if: steps.metadata.outputs.module == 'wasm-wasi-component' + + - name: Make wasm-wasi-component + run: | + which cargo || source $HOME/.cargo/env && which cargo + BINDGEN_EXTRA_CLANG_ARGS="-I../../njs/src -I../../njs/build" \ + make wasm-wasi-component + if: steps.metadata.outputs.module == 'wasm-wasi-component' + + ## + ## Tests + ## + - name: Run ${{ steps.metadata.outputs.module }} tests + run: | + PYTEST=`which pytest || which pytest-3` + # temporary java workaround which need javac in PATH for tests to work + if [ "${{ steps.metadata.outputs.module }}" = "java" ]; then + JVERSION=${{ steps.metadata.outputs.version }} + JC=`find /usr/lib/jvm -name javac | grep $JVERSION | head -1` + test -n "$JC" && export PATH=`dirname $JC`:$PATH || : + fi + $PYTEST --print-log ${{ steps.metadata.outputs.testpath }} + # Skip pytest if wasm build, as there are no tests yet + if: steps.metadata.outputs.module != 'wasm' && + steps.metadata.outputs.module != 'wasm-wasi-component' + From b01c197a7b5509ae7638e1121d1916e729c38844 Mon Sep 17 00:00:00 2001 From: oxpa Date: Fri, 19 Apr 2024 23:14:39 +0100 Subject: [PATCH 4/6] Re-enable GH based workflow for commit as well --- .github/workflows/ci-pull.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-pull.yml b/.github/workflows/ci-pull.yml index 04a2bd7b4..44147c5de 100644 --- a/.github/workflows/ci-pull.yml +++ b/.github/workflows/ci-pull.yml @@ -3,6 +3,9 @@ name: ci on: workflow_dispatch: pull_request: + push: + branches: + - master jobs: test: From eeb8ea8682968a9214912abe9c36c1f4da21d89a Mon Sep 17 00:00:00 2001 From: oxpa Date: Mon, 22 Apr 2024 13:57:18 +0100 Subject: [PATCH 5/6] Update workflow file names --- .../{ci-commit.yml => ci-selfhosted-or-smth-like-that.yml} | 0 .github/workflows/{ci-pull.yml => ci.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci-commit.yml => ci-selfhosted-or-smth-like-that.yml} (100%) rename .github/workflows/{ci-pull.yml => ci.yml} (100%) diff --git a/.github/workflows/ci-commit.yml b/.github/workflows/ci-selfhosted-or-smth-like-that.yml similarity index 100% rename from .github/workflows/ci-commit.yml rename to .github/workflows/ci-selfhosted-or-smth-like-that.yml diff --git a/.github/workflows/ci-pull.yml b/.github/workflows/ci.yml similarity index 100% rename from .github/workflows/ci-pull.yml rename to .github/workflows/ci.yml From b623a9913b26269caf7b6c9a1171c6e5961749d0 Mon Sep 17 00:00:00 2001 From: oxpa Date: Mon, 22 Apr 2024 13:57:34 +0100 Subject: [PATCH 6/6] proper name for a ci file --- .../{ci-selfhosted-or-smth-like-that.yml => ci-selfhosted.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci-selfhosted-or-smth-like-that.yml => ci-selfhosted.yml} (100%) diff --git a/.github/workflows/ci-selfhosted-or-smth-like-that.yml b/.github/workflows/ci-selfhosted.yml similarity index 100% rename from .github/workflows/ci-selfhosted-or-smth-like-that.yml rename to .github/workflows/ci-selfhosted.yml