Skip to content
.github/workflows/CI-linux.yml

CI: try to enable multiarch CI jobs #3

Sign in to view logs

CI: try to enable multiarch CI jobs

CI: try to enable multiarch CI jobs #3

Workflow file for this run

.github/workflows/CI-linux.yml at f9cf869
on:
workflow_call:
inputs:
os:
required: true
type: string
distro:
required: true
type: string
compiler-family:
required: true
type: string
compiler-version:
required: true
type: string
compiler-CC:
required: true
type: string
compiler-CXX:
required: true
type: string
compiler-GCOV:
type: string
compiler-CLANG_TIDY:
type: string
flavor:
required: true
type: string
enable-sample-based-testing:
type: boolean
default: false
rpuu-cache-key:
type: string
ECO:
type: string
secrets:
SONAR_TOKEN:
SONAR_PROJECT_KEY:
SONAR_ORGANIZATION:
env:
RPUU_DST: ${{ github.workspace }}/raw-camera-samples/raw.pixls.us-unique
COVERAGE_REPORT_PREFIX: rawspeed-coverage # relative to the workspace!
jobs:
linux:
runs-on: ubuntu-latest
env:
SONAR_SCANNER_VERSION: 5.0.1.3006
name: ${{ inputs.distro }}.${{ inputs.compiler-family }}${{ inputs.compiler-version }}.${{ inputs.flavor }}
container:
image: ${{ inputs.distro }}
options: --tmpfs /tmp:exec --tmpfs /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}:exec
steps:
- name: Enable multiarch Docker container support
timeout-minutes: 1
container:
image: multiarch/qemu-user-static
options: --rm --privileged --reset
- name: Configure APT
timeout-minutes: 1
run: |
set -xe
tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io > /dev/null <<EOT
force-unsafe-io
EOT
tee /etc/apt/apt.conf.d/tmpfs > /dev/null <<EOT
Dir::Cache::Archives "/tmp/apt/archives";
APT::ExtractTemplates::TempDir "/tmp/apt/temp";
EOT
mkdir -p /tmp/apt/archives
tee /etc/apt/apt.conf.d/80retry > /dev/null <<EOT
Acquire::Retries "10";
EOT
tee /etc/apt/apt.conf.d/80recommends > /dev/null <<EOT
APT::Install-Recommends "false";
EOT
tee /etc/apt/apt.conf.d/80suggests > /dev/null <<EOT
APT::Install-Suggests "false";
EOT
tee /etc/apt/apt.conf.d/80forceyes > /dev/null <<EOT
APT::Get::Assume-Yes "true";
EOT
tee /etc/apt/apt.conf.d/80fixmissing > /dev/null <<EOT
APT::Get::Fix-Missing "true";
EOT
rm -rf /etc/apt/sources.list*
if [ "${{ inputs.distro }}" = "debian:trixie-slim" ]; then
tee /etc/apt/sources.list > /dev/null <<EOT
deb http://debian-archive.trafficmanager.net/debian trixie main
deb http://debian-archive.trafficmanager.net/debian trixie-updates main
deb http://debian-archive.trafficmanager.net/debian-security trixie-security main
deb http://debian-archive.trafficmanager.net/debian trixie-backports main
EOT
elif [ "${{ inputs.distro }}" = "debian:bookworm-slim" ]; then
tee /etc/apt/sources.list > /dev/null <<EOT
deb http://debian-archive.trafficmanager.net/debian bookworm main
deb http://debian-archive.trafficmanager.net/debian bookworm-updates main
deb http://debian-archive.trafficmanager.net/debian-security bookworm-security main
deb http://debian-archive.trafficmanager.net/debian bookworm-backports main
EOT
fi
- name: Update base packages
timeout-minutes: 1
env:
DEBIAN_FRONTEND: noninteractive
COMPILER_FAMILY: ${{ inputs.compiler-family }}
FLAVOR: ${{ inputs.flavor }}
run: |
set -xe
rm -rf /var/lib/apt/lists/*
apt-get update
apt-get install eatmydata
eatmydata apt-get upgrade
- name: Install necessary packages
timeout-minutes: 1
env:
DEBIAN_FRONTEND: noninteractive
COMPILER_FAMILY: ${{ inputs.compiler-family }}
FLAVOR: ${{ inputs.flavor }}
ENABLE_SAMPLE_BASED_TESTING: ${{ inputs.enable-sample-based-testing }}
run: |
set -xe
eatmydata apt-get install ca-certificates \
cmake \
git \
googletest \
libjpeg-dev \
libpugixml-dev \
libxml2-utils \
ninja-build \
zlib1g-dev
if [ "$COMPILER_FAMILY" = "GNU" ]; then
eatmydata apt-get install g++-${{ inputs.compiler-version }} \
gcc \
gcc-${{ inputs.compiler-version }}
fi
if [ "$COMPILER_FAMILY" = "LLVM" ]; then
eatmydata apt-get install clang-${{ inputs.compiler-version }} \
libomp-${{ inputs.compiler-version }}-dev
fi
if [ "$ENABLE_SAMPLE_BASED_TESTING" = "true" ]; then
eatmydata apt-get install zstd
fi
if [ "$FLAVOR" = "ClangTidy" ]; then
eatmydata apt-get install clang-format-${{ inputs.compiler-version }} clang-tidy-${{ inputs.compiler-version }}
eatmydata apt-get install curl gpg
echo 'deb https://mirror.aardsoft.fi/opensuse/repositories/graphics:/darktable:/master/Debian_Testing/ /' | tee /etc/apt/sources.list
curl -fsSL https://download.opensuse.org/repositories/graphics:darktable:master/Debian_Testing/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/graphics_darktable_master.gpg > /dev/null
apt-get update
eatmydata apt-get install rawspeed-clang-tidy-module
fi
if [ "$FLAVOR" = "ClangStaticAnalysis" ] || [ "$FLAVOR" = "ClangCTUStaticAnalysis" ]; then
eatmydata apt-get install clang-tools-${{ inputs.compiler-version }} \
curl \
libxml2-dev \
libxslt1-dev \
make \
python3-dev \
python3-lxml \
python3-pip-whl \
python3-portalocker \
python3-psutil \
python3-venv \
python3-virtualenv \
python3-wheel \
python3-yaml \
xz-utils
fi
eatmydata apt-get clean
dpkg-divert --add --rename --divert /usr/bin/ld.original /usr/bin/ld
if [ "$FLAVOR" = "ClangStaticAnalysis" ] || [ "$FLAVOR" = "ClangCTUStaticAnalysis" ]; then
ln -s /usr/bin/${{ inputs.compiler-CC }} /usr/local/bin/clang
ln -s /usr/bin/${{ inputs.compiler-CXX }} /usr/local/bin/clang++
ln -s /usr/bin/${{ inputs.compiler-CC }} /usr/local/bin/gcc
ln -s /usr/bin/${{ inputs.compiler-CXX }} /usr/local/bin/g++
ln -s /usr/bin/${{ inputs.compiler-CC }} /usr/local/bin/x86_64-linux-gnu-gcc
ln -s /usr/bin/${{ inputs.compiler-CXX }} /usr/local/bin/x86_64-linux-gnu-g++
fi
ln -s /usr/bin/ld.gold /usr/bin/ld
- name: Configure git
timeout-minutes: 1
run: |
set -xe
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Fetch/Checkout RawSpeed git repo
timeout-minutes: 1
uses: actions/checkout@v4
with:
path: 'rawspeed'
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Cache raw.pixls.us masterset
timeout-minutes: 1
if: inputs.enable-sample-based-testing
uses: actions/cache/restore@v4
with:
path: ${{ env.RPUU_DST }}
key: ${{ inputs.rpuu-cache-key }}
fail-on-cache-miss: true
- name: Fetch/Checkout CodeChecker git repo (for clang static analysis)
timeout-minutes: 1
if: inputs.flavor == 'ClangStaticAnalysis' || inputs.flavor == 'ClangCTUStaticAnalysis'
uses: actions/checkout@v4
with:
repository: 'Ericsson/codechecker'
path: 'codechecker'
fetch-depth: '1'
- name: Install CodeChecker (for clang static analysis)
timeout-minutes: 1
if: inputs.flavor == 'ClangStaticAnalysis' || inputs.flavor == 'ClangCTUStaticAnalysis'
env:
CC: ${{ inputs.compiler-CC }}
CXX: ${{ inputs.compiler-CXX }}
run: |
set -xe
cd codechecker
make venv
. $PWD/venv/bin/activate
BUILD_LOGGER_64_BIT_ONLY=YES BUILD_UI_DIST=NO make package
export PATH="$PWD/build/CodeChecker/bin:$PATH"
- name: Initialize CodeQL (for CodeQL static analysis)
timeout-minutes: 1
if: inputs.flavor == 'CodeQLAnalysis'
uses: github/codeql-action/init@v2
with:
languages: cpp
- name: Set up JDK 11 (for SonarCloud static analysis)
timeout-minutes: 1
if: inputs.flavor == 'SonarCloudStaticAnalysis' && github.repository == 'darktable-org/rawspeed' && github.event_name != 'pull_request' && github.ref_type == 'branch' && (github.ref_name == 'develop' || github.ref_name == 'stable')
uses: actions/setup-java@v4
with:
distribution: zulu
java-version: 21
- name: Download and set up sonar-scanner (for SonarCloud static analysis)
timeout-minutes: 1
if: inputs.flavor == 'SonarCloudStaticAnalysis' && github.repository == 'darktable-org/rawspeed' && github.event_name != 'pull_request' && github.ref_type == 'branch' && (github.ref_name == 'develop' || github.ref_name == 'stable')
env:
SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
run: |
set -xe
rm -rf /var/lib/apt/lists/*
eatmydata apt-get update
eatmydata apt-get install curl unzip
eatmydata apt-get clean
mkdir -p $HOME/.sonar
curl -sSLo $HOME/.sonar/sonar-scanner.zip ${{ env.SONAR_SCANNER_DOWNLOAD_URL }}
unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/
echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> $GITHUB_PATH
- name: Verify that the source code is properly formatted
timeout-minutes: 1
if: inputs.flavor == 'ClangTidy'
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
run: |
set -xe
cd "$SRC_DIR"
git ls-tree -r --name-only HEAD | grep -vf .clang-format-ignore | xargs clang-format-${{ inputs.compiler-version }} --verbose --dry-run -Werror
- name: Configure
timeout-minutes: 1
env:
CC: ${{ inputs.compiler-CC }}
CXX: ${{ inputs.compiler-CXX }}
CLANG_TIDY: ${{ inputs.compiler-CLANG_TIDY }}
GCOV: ${{ inputs.compiler-GCOV }}
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
RPUU_DST: ${{ env.RPUU_DST }}
ECO: ${{ inputs.ECO }} -DALLOW_DOWNLOADING_GOOGLETEST=ON -DALLOW_DOWNLOADING_GOOGLEBENCHMARK=ON -DUSE_CLANG_TIDY=${{ inputs.flavor == 'ClangTidy' }} -DRAWSPEED_ENABLE_SAMPLE_BASED_TESTING=${{ inputs.enable-sample-based-testing }}
FLAVOR: ${{ inputs.flavor }}
TARGET: configure
run: |
set -xe
cmake -E make_directory "${BUILD_DIR}"
cmake -E make_directory "${INSTALL_PREFIX}"
export ECO="${ECO} -DRAWSPEED_REFERENCE_SAMPLE_ARCHIVE=${RPUU_DST}"
"${SRC_DIR}/.ci/ci-script.sh"
- name: Build
timeout-minutes: ${{ inputs.flavor != 'ClangTidy' && (inputs.flavor != 'CodeQLAnalysis' && 7 || 12) || 25 }}
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: build
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
- name: Test (unit tests)
timeout-minutes: 1
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: test
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
- name: Collect source coverage data (unit tests)
timeout-minutes: 1
if: inputs.flavor == 'Coverage'
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}/linux
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: coverage
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
mkdir -p "${COVERAGE_REPORT_PREFIX}"
mv "${BUILD_DIR}/gcov-reports-unittest" "${COVERAGE_REPORT_PREFIX}/unittests"
- name: Test (benchmarks)
timeout-minutes: 1
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: test_benchmarks
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
- name: Collect source coverage data (benchmarks)
timeout-minutes: 1
if: inputs.flavor == 'Coverage'
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}/linux
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: coverage_benchmarks
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
mkdir -p "${COVERAGE_REPORT_PREFIX}"
mv "${BUILD_DIR}/gcov-reports-benchmarks" "${COVERAGE_REPORT_PREFIX}/benchmarks"
- name: Test (integration)
timeout-minutes: ${{ inputs.flavor != 'Coverage' && 1 || 31 }}
if: inputs.enable-sample-based-testing
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: test_integration
OMP_NUM_THREADS: 1
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
- name: Collect source coverage data (integration)
timeout-minutes: 1
if: inputs.flavor == 'Coverage' && inputs.enable-sample-based-testing
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}/linux
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: coverage_integration
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
mkdir -p "${COVERAGE_REPORT_PREFIX}"
mv "${BUILD_DIR}/gcov-reports-rsa" "${COVERAGE_REPORT_PREFIX}/integration"
- if: inputs.flavor == 'Coverage'
timeout-minutes: 1
name: Cache code coverage (linux reports)
uses: actions/cache/save@v4
with:
path: ${{ env.COVERAGE_REPORT_PREFIX }}/linux # relative path!
key: coverage-${{ github.sha }}-linux
enableCrossOsArchive: true
- name: Install
timeout-minutes: 1
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
FLAVOR: ${{ inputs.flavor }}
TARGET: install
run: |
set -xe
"${SRC_DIR}/.ci/ci-script.sh"
- name: Perform clang static analysis (plain mode)
timeout-minutes: 25
if: inputs.flavor == 'ClangStaticAnalysis'
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
CC: ${{ inputs.compiler-CC }}
CXX: ${{ inputs.compiler-CXX }}
run: |
set -xe
. "$PWD/codechecker/venv/bin/activate"
export PATH="$PWD/codechecker/build/CodeChecker/bin:$PATH"
set +e
CodeChecker check --logfile "${BUILD_DIR}/compile_commands.json" --analyzers clangsa --jobs "$(nproc --all)" --file "${SRC_DIR}/*" --disable deadcode.DeadStores -o "${BUILD_DIR}/codechecker_results"
bugs=$?
CodeChecker parse -e html "${BUILD_DIR}/codechecker_results" -o "${BUILD_DIR}/codechecker_report"
set -xe
tar -cvJ -f "$GITHUB_WORKSPACE/codechecker_report.tar.xz" -C "${BUILD_DIR}" "codechecker_report"
[ $bugs -eq 0 ] || exit 1
- name: Upload results of clang static analysis (plain mode)
timeout-minutes: 1
if: failure()
uses: actions/upload-artifact@v4
with:
name: codechecker_report.tar.xz
path: ${{ github.workspace }}/codechecker_report.tar.xz
- name: Perform clang static analysis (CTU mode)
timeout-minutes: 25
if: inputs.flavor == 'ClangCTUStaticAnalysis'
env:
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
CC: ${{ inputs.compiler-CC }}
CXX: ${{ inputs.compiler-CXX }}
run: |
set -xe
. "$PWD/codechecker/venv/bin/activate"
export PATH="$PWD/codechecker/build/CodeChecker/bin:$PATH"
set +e
CodeChecker check --ctu --ctu-reanalyze-on-failure --logfile "${BUILD_DIR}/compile_commands.json" --analyzers clangsa --jobs "$(nproc --all)" --file "${SRC_DIR}/*" --disable deadcode.DeadStores -o "${BUILD_DIR}/codechecker_ctu_results"
bugs=$?
CodeChecker parse -e html "${BUILD_DIR}/codechecker_ctu_results" -o "${BUILD_DIR}/codechecker_ctu_report"
set -xe
tar -cvJ -f "$GITHUB_WORKSPACE/codechecker_ctu_report.tar.xz" -C "${BUILD_DIR}" "codechecker_ctu_report"
[ $bugs -eq 0 ] || exit 1
- name: Upload results of clang static analysis (CTU mode)
timeout-minutes: 1
uses: actions/upload-artifact@v4
if: failure()
with:
name: codechecker_ctu_report.tar.xz
path: ${{ github.workspace }}/codechecker_ctu_report.tar.xz
- name: Perform CodeQL static analysis
timeout-minutes: 11
if: inputs.flavor == 'CodeQLAnalysis'
uses: github/codeql-action/analyze@v2
- name: Perform SonarCloud static analysis
timeout-minutes: 6
if: inputs.flavor == 'SonarCloudStaticAnalysis'
env:
SONAR_SERVER_URL: "https://sonarcloud.io"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
SRC_DIR: ${{ github.workspace }}/rawspeed
BUILD_DIR: ${{ github.workspace }}/rawspeed-build
COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}
run: |
set -xe
cd "${SRC_DIR}"
sonar-scanner --define project.settings="${SRC_DIR}/.ci/sonar-project.properties" --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.projectKey="${{ secrets.SONAR_PROJECT_KEY }}" --define sonar.organization="${{ secrets.SONAR_ORGANIZATION }}" --define sonar.cfamily.compile-commands="${BUILD_DIR}/compile_commands.json" --define sonar.cfamily.analysisCache.mode=fs --define sonar.cfamily.analysisCache.path="${BUILD_DIR}/sonar-scanner-cache"