unix-conda-build.yml

# Script for building CCTBX on linux and macOS
#
# Variables:
#   CONDA: Linux, MacOSX
#   OS: linux-64, osx-64
#   PYTHON_VERSION: py27, py36, py37, py38, py39, py310
#   MODULES: <modules artifact name>
#
# Optional Parameters:
#   distribution: centos, ubuntu
#   version: [6, 10]

steps:

# download components
- checkout: none

- task: DownloadPipelineArtifact@2
  inputs:
    source: 'current'
    artifact: $(MODULES)
    path: $(Pipeline.Workspace)
  displayName: Download modules tarball

- script: |
    set -xe
    cd $(Pipeline.Workspace)
    tar -xf modules.tar
  displayName: Extract modules tarball

- script: |
    set -xe
    cd $(Pipeline.Workspace)/modules
    wget "https://github.com/SCons/scons/archive/3.1.1.zip"
    unzip 3.1.1.zip
    mv scons-3.1.1 scons
  displayName: Download SCons for Python 2
  condition: eq(variables['PYTHON_VERSION'], 'py27')

- script: |
    set -xe
    cd $(Pipeline.Workspace)/modules
    rm -fr boost
  displayName: Remove subset of Boost sources
  condition: ne(variables['PYTHON_VERSION'], 'py27')

- script: |
    set -xe
    cd $(Pipeline.Workspace)/modules/dxtbx
    git remote set-url origin https://github.com/dials/dxtbx.git
    git fetch origin
    git checkout dials-2.2
  displayName: Use dials-2.2 branch for Python 2
  condition: eq(variables['PYTHON_VERSION'], 'py27')

- script: |
    set -xe
    cd $(Pipeline.Workspace)/modules/cctbx_project/scitbx
    sed -i .bak 's/"$D\/suffixtree/#"$D\/suffixtree/g' run_tests.py
  displayName: Skip scitbx/suffixtree/test/tst_single.py for -O1 on macOS
  condition: and(eq(variables['MODULES'], 'modules1'), eq(variables['OS'], 'osx-64'))

- script: |
    set -xe
    cd $(Pipeline.Workspace)/modules/cctbx_project/scitbx/math/tests
    sed -i .back 's/  test_B_matrix/# test_B_matrix/g' tst_error_propagation_matrix_inverse.py
  displayName: Skip test for -O0 on macOS
  condition: and(eq(variables['MODULES'], 'modules0'), eq(variables['OS'], 'osx-64'))

- script: |
    set -xe
    cd $(Pipeline.Workspace)
    ln -s modules/cctbx_project/libtbx/auto_build/bootstrap.py
  displayName: Link bootstrap.py

# install miniforge
- task: DownloadPipelineArtifact@2
  inputs:
    source: 'specific'
    project: '$(resources.pipeline.data_cache.projectID)'
    pipeline: '$(resources.pipeline.data_cache.pipelineID)'
    artifact: 'miniforge'
    path: $(Pipeline.Workspace)/miniforge
  displayName: Download miniforge

- script: |
    set -xe
    bash $(Pipeline.Workspace)/miniforge/Miniforge3-$(CONDA)-x86_64.sh -b -u -p $(Pipeline.Workspace)/miniforge
  displayName: Install miniforge

- script: |
    set -xe
    source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
    conda update --all -y -n base
  displayName: Update miniforge

# create conda environment
# fresh downloads for "Update build cache" and "Full" pipelines
- task: DownloadPipelineArtifact@2
  inputs:
    source: 'current'
    artifact: $(PYTHON_VERSION)_$(OS)
    path: $(Pipeline.Workspace)/channel
  displayName: Download conda packages
  condition: >
    or(eq(variables['Build.DefinitionName'], 'Update build cache'),
       eq(variables['Build.DefinitionName'], 'Full'))

- task: DownloadPipelineArtifact@2
  inputs:
    source: 'specific'
    project: '$(resources.pipeline.build_cache.projectID)'
    pipeline: '$(resources.pipeline.build_cache.pipelineID)'
    allowPartiallySucceededBuilds: true
    allowFailedBuilds: true
    artifact: $(PYTHON_VERSION)_$(OS)
    path: $(Pipeline.Workspace)/channel
  displayName: Download cached conda packages
  condition: >
    and(ne(variables['Build.DefinitionName'], 'Update build cache'),
        ne(variables['Build.DefinitionName'], 'Full'))

- script: |
    set -xe
    source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
    conda install -y conda-build
    cd $(Pipeline.Workspace)/channel
    conda index .
    echo @EXPLICIT > env.txt
    for filename in `cat filenames.txt`; do
      echo file://$(Pipeline.Workspace)/channel/${filename} >> env.txt
    done
  displayName: Build local channel

- script: |
    set -xe
    source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
    if [ -d $(Pipeline.Workspace)/miniforge/envs/$(PYTHON_VERSION) ]; then
      conda remove  -y --all -n $(PYTHON_VERSION)
    fi
    conda create -y --offline -n $(PYTHON_VERSION) --file $(Pipeline.Workspace)/channel/env.txt
    conda activate $(PYTHON_VERSION)
    pip install junit-xml
  displayName: Create conda environment
  retryCountOnTaskFailure: 3

# build
- script: |
    set -xe
    source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
    conda activate $(PYTHON_VERSION)
    cd $(Pipeline.Workspace)
    python bootstrap.py build \
      --builder=cctbx \
      --use-conda=${CONDA_PREFIX} \
      --config-flags="--cxxstd=c++14" \
      --nproc=4
    cd build
    source setpaths.sh
    libtbx.configure \
      cma_es \
      fable \
      rstbx \
      spotfinder \
      cbflib_adaptbx \
      phenix_regression \
      phenix_examples
    libtbx.scons -j 4
    libtbx.scons -j 4
  displayName: Configure and Build
  condition: >
    and(succeeded(),
        or(ne(variables['PYTHON_VERSION'], 'py27'),
           and(eq(variables['PYTHON_VERSION'], 'py27'),
               and(ne('${{ parameters.version[0] }}', 22),
                   ne('${{ parameters.version[0] }}', 9)))))

- script: |
    set -xe
    cd $(Pipeline.Workspace)
    if [ -f "./build/config.log" ]; then
      cd build
      cat config.log
    fi
  displayName: Debug info
  condition: succeededOrFailed()

# test
- script: |
    set -xe
    source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
    conda activate $(PYTHON_VERSION)
    cd $(Pipeline.Workspace)
    source ./build/setpaths.sh
    rm -fr tests
    mkdir tests
    cd tests
    export PYTHONDEVMODE=1
    export PYTHONTRACEMALLOC=1
    libtbx.run_tests_parallel \
      module=annlib_adaptbx \
      module=boost_adaptbx \
      module=cbflib_adaptbx \
      module=cctbx \
      module=cctbx_website \
      module=cma_es \
      module=fable \
      module=gltbx \
      module=iotbx \
      module=libtbx \
      module=rstbx \
      module=scitbx \
      module=crys3d \
      module=smtbx \
      module=spotfinder \
      nproc=4
  failOnStderr: false
  displayName: Test
  condition: >
    and(succeeded(),
        or(ne(variables['PYTHON_VERSION'], 'py27'),
           and(eq(variables['PYTHON_VERSION'], 'py27'),
               and(ne('${{ parameters.version[0] }}', 22),
                   ne('${{ parameters.version[0] }}', 9)))))
  retryCountOnTaskFailure: 3

# store tests directory as an artifact
- script: |
    set -xe
    cd $(Pipeline.Workspace)
    mkdir -p tests
    tar -Jcf tests_$(job_name).tar.xz ./tests
    touch ./tests/output.xml
  displayName: Save test directory
  condition: succeededOrFailed()
  continueOnError: true

- publish: $(Pipeline.Workspace)/tests_$(job_name).tar.xz
  artifact: tests_$(job_name)_$(MODULES)
  condition: succeededOrFailed()
  continueOnError: true

# publish test results
- task: PublishTestResults@2
  condition: succeededOrFailed()
  inputs:
    testResultsFormat: 'JUnit'
    testResultsFiles: '$(Pipeline.Workspace)/tests/output.xml'