From 6dc2f5f9f320c956712fb1992379fcc0bf2907d5 Mon Sep 17 00:00:00 2001
From: flg <fabrice@legoff.eu>
Date: Tue, 12 Nov 2024 11:12:06 +0100
Subject: [PATCH] added container to build traccc/SYCL for Nvidia GPU backend
 on Alma9 (#127)

* added container to build traccc/SYCL for Nvidia GPU backend on Alma9

* add container to build matrix

---------

Co-authored-by: Fabrice Le Goff <fabrice.le.goff@cern.ch>
---
 .github/workflows/build-images.yml |  1 +
 alma9_cuda_oneapi/Dockerfile       | 47 ++++++++++++++++++++++++++++++
 alma9_cuda_oneapi/cuda.repo        |  8 +++++
 alma9_cuda_oneapi/oneapi.repo      |  7 +++++
 alma9_cuda_oneapi/setenv.sh        | 37 +++++++++++++++++++++++
 5 files changed, 100 insertions(+)
 create mode 100644 alma9_cuda_oneapi/Dockerfile
 create mode 100644 alma9_cuda_oneapi/cuda.repo
 create mode 100644 alma9_cuda_oneapi/oneapi.repo
 create mode 100755 alma9_cuda_oneapi/setenv.sh

diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 5fa572e..6b6d3b7 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -30,6 +30,7 @@ jobs:
           - ubuntu2404_rocm_oneapi
           - ubuntu2404_oneapi
           - alma9-base
+          - alma9_cuda_oneapi
     steps:
       - uses: actions/checkout@v4
 
diff --git a/alma9_cuda_oneapi/Dockerfile b/alma9_cuda_oneapi/Dockerfile
new file mode 100644
index 0000000..3629cd4
--- /dev/null
+++ b/alma9_cuda_oneapi/Dockerfile
@@ -0,0 +1,47 @@
+# Docker machinery, part of the ACTS project
+#
+# (c) 2024 CERN for the benefit of the ACTS project
+#
+# Mozilla Public License Version 2.0
+
+FROM cern/alma9-base:latest
+
+LABEL description="CERN AlmaLinux 9 with Acts dependencies and CUDA + oneAPI"
+LABEL version="1"
+
+RUN dnf install -y https://linuxsoft.cern.ch/wlcg/el9/x86_64/HEP_OSlibs-9.1.0-2.el9.x86_64.rpm \
+  && dnf -y clean all
+
+# Set up gcc 13.
+RUN dnf install -y gcc-toolset-13
+
+# Install boost development package
+RUN dnf install -y boost boost-devel
+
+# Set up the CUDA repository.
+COPY cuda.repo /etc/yum.repos.d/cuda.repo
+ARG CUDA_VERSION=12-5
+
+# Install CUDA
+RUN dnf install -y cuda-${CUDA_VERSION}.x86_64
+
+# Set up the oneAPI repository.
+COPY oneapi.repo /etc/yum.repos.d/oneapi.repo
+
+# Install oneAPI.
+ARG ONEAPI_VERSION=2024.2
+RUN dnf install -y intel-oneapi-compiler-dpcpp-cpp-${ONEAPI_VERSION}
+
+# Install the CodePlay AMD plugin on top of oneAPI.
+ARG CODEPLAY_PLUGIN_VERSION=2024.2
+RUN curl -SL \
+    "https://developer.codeplay.com/api/v1/products/download?product=oneapi&variant=nvidia&version=${CODEPLAY_PLUGIN_VERSION}" \
+    -o plugin.sh && \
+    sh plugin.sh --install-dir /opt/intel/oneapi --yes && \
+    rm plugin.sh
+
+# By default the environment with ROCm, OneAPI SYCL compiler, and AMD backend plugin is loaded
+ADD --chmod=700 setenv.sh /
+# Make sure it's executable
+RUN chmod +x /setenv.sh
+ENTRYPOINT /setenv.sh
diff --git a/alma9_cuda_oneapi/cuda.repo b/alma9_cuda_oneapi/cuda.repo
new file mode 100644
index 0000000..b8639dd
--- /dev/null
+++ b/alma9_cuda_oneapi/cuda.repo
@@ -0,0 +1,8 @@
+[cuda]
+name=CUDA repository
+baseurl=https://linuxsoft.cern.ch/mirror/developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64
+enabled=1
+gpgcheck=1
+gpgkey=https://linuxsoft.cern.ch/mirror/developer.download.nvidia.com/RPM-GPG-KEY-cuda-D42D0685
+priority=12
+module_hotfixes=1
diff --git a/alma9_cuda_oneapi/oneapi.repo b/alma9_cuda_oneapi/oneapi.repo
new file mode 100644
index 0000000..ec85829
--- /dev/null
+++ b/alma9_cuda_oneapi/oneapi.repo
@@ -0,0 +1,7 @@
+[oneAPI]
+name=Intel oneAPI repository
+baseurl=https://yum.repos.intel.com/oneapi
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
diff --git a/alma9_cuda_oneapi/setenv.sh b/alma9_cuda_oneapi/setenv.sh
new file mode 100755
index 0000000..b273fb3
--- /dev/null
+++ b/alma9_cuda_oneapi/setenv.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+GCCDIR=/opt/rh/gcc-toolset-13/root
+
+# Set up gcc 13.
+source ${GCCDIR}/../enable
+# Set up SYCL compiler, including AMD backend.
+source /opt/intel/oneapi/setvars.sh --include-intel-llvm
+
+# Set up the compilers to use (with CMake).
+export CC="`which clang` --gcc-toolchain=${GCCDIR}"
+export CXX="`which clang++` --gcc-toolchain=${GCCDIR}"
+
+# Set up the compiler and its options for SYCL
+# CERN's Tesla T4 support sm_75
+export SYCLCXX="${CXX} -fsycl"
+export SYCLFLAGS="-fsycl-targets=nvidia_gpu_sm_75 -Xclang -opaque-pointers -Wno-unknown-cuda-version"
+
+# Set up CUDA.
+export NVIDIA_VISIBLE_DEVICES=all
+export NVIDIA_DRIVER_CAPABILITIES=compute,utility
+export PATH=/usr/local/cuda/bin:"$PATH"
+export LD_LIBRARY_PATH=/usr/local/cuda/lib64:"$LD_LIBRARY_PATH"
+export CUDAHOSTCXX="clang++"
+export CUDAFLAGS="-allow-unsupported-compiler"
+
+# Clean
+unset GCCDIR
+
+# For user's convenience:
+export TRACCC_URL="https://github.com/acts-project/traccc"
+echo "git clone $TRACCC_URL"
+echo "cmake -S . -B buildsyclcuda -DTRACCC_BUILD_SYCL=ON -DTRACCC_USE_ROOT=OFF -DCMAKE_INSTALL_PREFIX:PATH=./installed"
+echo "cmake --build buildsyclcuda --target install"
+
+echo 'alias ll="ls -lh"' > ~/.bashrc
+
+bash