azure-pypi-pipeline.yml

name: PyPI_Packaging

# Trigger this build whenever the dev or release branches are updated,
# or on pull requests that affect PyPI-related files.
#
# Ideally we'd run this pipeline for all pull requests, but doing so consumes
# our limited number of slots on Azure and almost always just duplicates the
# build done in the main pipeline.
trigger:
    branches:
        include:
        - dev
        - release

pr:
    branches:
        include:
        - dev
        - release
    paths:
        include:
        - azure-pypi-pipeline.yml
        - build_scripts/pypi/*.py
        - build_scripts/pypi/package_files/*

# Azure does not support an empty string as a default argument, so we use a
# special 'none' string for post_release_tag.
parameters:
- name: post_release_tag
  displayName: Post Release Version Tag
  type: string
  default: 'none'

variables:
- name: post_release_tag_arg
  ${{ if eq(parameters.post_release_tag, 'none') }}:
    value: ''
  ${{ else }}:
    value: ${{ format('--post-release-tag {0}', parameters.post_release_tag) }}

stages:
- stage: Build_and_Package
  jobs:

  - job: Linux
    strategy:
      matrix:
        Python36:
          PYTHON_INTERPRETER: /opt/python/cp36-cp36m/bin/python
          PYTHON_TAG: cp36
        Python37:
          PYTHON_INTERPRETER: /opt/python/cp37-cp37m/bin/python
          PYTHON_TAG: cp37
        Python38:
          PYTHON_INTERPRETER: /opt/python/cp38-cp38/bin/python
          PYTHON_TAG: cp38
        Python39:
          PYTHON_INTERPRETER: /opt/python/cp39-cp39/bin/python
          PYTHON_TAG: cp39
        Python310:
          PYTHON_INTERPRETER: /opt/python/cp310-cp310/bin/python
          PYTHON_TAG: cp310
    timeoutInMinutes: 90
    pool:
      vmImage: Ubuntu-20.04
    steps:
    - bash: |
        docker build -t manylinuxwithcmake build_scripts/pypi/docker
        docker run --name usdmanylinux --rm -id -v $(Build.SourcesDirectory):/opt/USD -v /home/vsts/dist:/opt/USD-dist manylinuxwithcmake
      displayName: 'Creating docker build environment'
    - bash: |
        # Terrible, terrible hack. The manylinux Docker image used to build the
        # Python wheel does not include the corresponding Python shared library
        # to link against. https://peps.python.org/pep-0513/#libpythonx-y-so-1
        # describes why this is so. However, the FindPython CMake module used
        # by USD's build system requires that the library exists and will error
        # out otherwise, even though we explicitly avoid linking against Python
        # via the PXR_PY_UNDEFINED_DYNAMIC_LOOKUP flag.
        # 
        # To work around this, we create a dummy file for the library using
        # the same logic as build_usd.py to determine where the library should
        # exist (see GetPythonInfo). FindPython will see that the library exists
        # and allow the build to continue. The file is 100% bogus, but the
        # PXR_PY_UNDEFINED_DYNAMIC_LOOKUP flag will ensure that we never try to
        # link against this library anyway, so it doesn't matter.
        docker exec usdmanylinux $(PYTHON_INTERPRETER) -c "import pathlib,sysconfig; pathlib.Path(sysconfig.get_config_var('LIBDIR'), sysconfig.get_config_var('LDLIBRARY')).touch()"
        docker exec usdmanylinux $(PYTHON_INTERPRETER)  build_scripts/build_usd.py --build-args USD,"-DPXR_PY_UNDEFINED_DYNAMIC_LOOKUP=ON -DPXR_BUILD_USD_TOOLS=OFF -DPXR_INSTALL_LOCATION=../pxr/pluginfo" --no-materialx --no-imaging --no-examples --no-tutorials --build /opt/USD/gen/build --src /opt/USD/gen/src /opt/USD/inst -v
      displayName: 'Building USD'
    - bash: |
        docker exec usdmanylinux mkdir ./packaging
        docker exec usdmanylinux cp -R /opt/USD/inst ./packaging
        docker exec usdmanylinux sh -c 'cp build_scripts/pypi/package_files/* ./packaging'
        docker exec usdmanylinux sh -c 'cp LICENSE.txt ./packaging'
      displayName: "Creating packaging directory"
    - bash: |
        docker exec -w /opt/USD/packaging usdmanylinux $(PYTHON_INTERPRETER) setup.py $(post_release_tag_arg) bdist_wheel --python-tag ${PYTHON_TAG}
      displayName: 'Running setup.py'
    - bash: |
        docker exec usdmanylinux /bin/bash -c 'PYTHONPATH=/opt/USD/packaging/pypi/lib/python LD_LIBRARY_PATH=/opt/USD/packaging/pypi/lib:$LD_LIBRARY_PATH auditwheel repair packaging/dist/*.whl'
      displayName: 'Running auditwheel repair (moves .so files into package)'
    - bash: |
        WHEEL_PACKAGE_NAME=`docker exec usdmanylinux ls wheelhouse`
        docker exec usdmanylinux $(PYTHON_INTERPRETER) build_scripts/pypi/updatePluginfos.py "wheelhouse/$WHEEL_PACKAGE_NAME" "/opt/USD-dist/$WHEEL_PACKAGE_NAME"
      displayName: 'Updating pluginfo paths'
    - bash: |
        docker stop usdmanylinux
      displayName: 'Stopping docker container'
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: dist-linux-$(PYTHON_TAG)
        targetPath: /home/vsts/dist

  - job: Windows
    strategy:
      matrix:
        Python36:
          PYTHON_VERSION_SPEC: 3.6
          PYTHON_TAG: cp36
        Python37:
          PYTHON_VERSION_SPEC: 3.7
          PYTHON_TAG: cp37
        Python38:
          PYTHON_VERSION_SPEC: 3.8
          PYTHON_TAG: cp38
        Python39:
          PYTHON_VERSION_SPEC: 3.9
          PYTHON_TAG: cp39
        Python310:
          PYTHON_VERSION_SPEC: 3.10
          PYTHON_TAG: cp310
    timeoutInMinutes: 90
    pool:
      vmImage: 'windows-2019'
    steps:
    - task: UsePythonVersion@0
      inputs:
        versionSpec: $(PYTHON_VERSION_SPEC)
        addToPath: true
    - script: |
        call C:\"Program Files (x86)"\"Microsoft Visual Studio"\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
        set BOOST_ROOT=
        python --version
        python build_scripts/build_usd.py --build-args USD,"-DPXR_ENABLE_PRECOMPILED_HEADERS=OFF -DPXR_PY_UNDEFINED_DYNAMIC_LOOKUP=ON -DPXR_BUILD_USD_TOOLS=OFF -DPXR_INSTALL_LOCATION=../pxr/pluginfo" --no-materialx --no-imaging --no-examples --no-tutorials --build %HOME%/USDgen/build --src %HOME%/USDgen/src %HOME%/USDinst -v
      displayName: 'Building USD'
    - script: |
        dir
        mkdir D:\packaging
        xcopy /E /I %HOME%\USDinst D:\packaging\inst
        copy build_scripts\pypi\package_files\* D:\packaging
        copy LICENSE.txt D:\packaging
        dir D:\packaging
        dir D:\packaging\inst
      displayName: "Creating packaging directory"
    - script: |
        python -m pip install wheel
      displayName: 'Installing python packages'
    - script: |
        cd D:\packaging
        dir
        python setup.py $(post_release_tag_arg) bdist_wheel --python-tag $(PYTHON_TAG) --plat-name win_amd64
      displayName: 'Running setup.py'
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: dist-windows-$(PYTHON_TAG)
        targetPath: D:\packaging\dist

  - job: Mac
    strategy:
      matrix:
        Python36:
          PYTHON_VERSION_SPEC: 3.6
          PYTHON_INTERPRETER: python3.6
          PYTHON_TAG: cp36
        Python37:
          PYTHON_VERSION_SPEC: 3.7
          PYTHON_INTERPRETER: python3.7
          PYTHON_TAG: cp37
        Python38:
          PYTHON_VERSION_SPEC: 3.8
          PYTHON_INTERPRETER: python3.8
          PYTHON_TAG: cp38
        Python39:
          PYTHON_VERSION_SPEC: 3.9
          PYTHON_INTERPRETER: python3.9
          PYTHON_TAG: cp39
        Python310:
          PYTHON_VERSION_SPEC: 3.10
          PYTHON_INTERPRETER: python3.10
          PYTHON_TAG: cp310
    timeoutInMinutes: 180
    pool:
      vmImage: 'macOS-11'
    steps:
    - task: UsePythonVersion@0
      inputs:
        versionSpec: $(PYTHON_VERSION_SPEC)
        addToPath: true
    - script: |
        sudo xcode-select -s /Applications/Xcode_13.2.app/Contents/Developer
        $(PYTHON_INTERPRETER) build_scripts/build_usd.py --build-args USD,"-DPXR_PY_UNDEFINED_DYNAMIC_LOOKUP=ON -DPXR_BUILD_USD_TOOLS=OFF -DPXR_INSTALL_LOCATION=../pluginfo" --no-materialx --no-imaging --no-examples --no-tutorials --generator Xcode --build-target universal --build $HOME/USDgen/build --src $HOME/USDgen/src $HOME/USDinst -v
      displayName: 'Building USD'
    - bash: |
        $(PYTHON_INTERPRETER) -m pip install delocate wheel
      displayName: 'Installing python packages'
    - bash: |
        mkdir ./packaging
        mkdir ./packaging/inst
        cp -R $HOME/USDinst/* ./packaging/inst
        cp build_scripts/pypi/package_files/* ./packaging
        cp LICENSE.txt ./packaging
      displayName: "Creating packaging directory"
    - bash: |
        cd ./packaging
        $(PYTHON_INTERPRETER) setup.py $(post_release_tag_arg) bdist_wheel --python-tag ${PYTHON_TAG} --plat-name macosx_10_9_universal2
      displayName: 'Running setup.py'
    - bash: |
        delocate-wheel -v -w dist-delocated packaging/dist/*
      displayName: 'Running delocate (moves shared lib files into package)'
    - bash: |
        WHEEL_PACKAGE_NAME=`ls ./packaging/dist`
        mkdir -p ./dist
        $(PYTHON_INTERPRETER)  build_scripts/pypi/updatePluginfos.py "./dist-delocated/$WHEEL_PACKAGE_NAME" "./dist/$WHEEL_PACKAGE_NAME"
      displayName: 'Updating pluginfo paths'
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: dist-mac-$(PYTHON_TAG)
        targetPath: ./dist


- stage: Deliver
  dependsOn: Build_and_Package
  jobs:
  - job: Collect_Packages
    timeoutInMinutes: 15
    pool:
      vmImage: Ubuntu-20.04
    steps:
    - task: DownloadPipelineArtifact@2
      displayName: 'Downloading all artifacts'
      inputs:
        source: current
        # Not specifying artifact names so we get all of them
        downloadPath: '$(Pipeline.Workspace)'
    - bash: |
        cd $(Pipeline.Workspace)
        mkdir dist-final
        cp dist*/*.whl dist-final
      displayName: 'Collecting whl packages'
    - task: PublishPipelineArtifact@0
      displayName: 'Publishing in a single download'
      inputs:
        artifactName: dist
        targetPath: '$(Pipeline.Workspace)/dist-final'


# The matrix below is verbose. There is a way to do this with a more n x m
# syntax using a foreach loop kind of construct. For now, opting for the below
# because it seems easier to read. An example of the other approach can be
# found here:
# https://github.com/nedrebo/parameterized-azure-jobs
- stage: Test_Packages
  dependsOn: Deliver
  jobs:
  - job: Test_Install
    strategy:
      matrix:
        Linux_Python36:
          PYTHON_VERSION_SPEC: 3.6
          IMAGE: 'Ubuntu-20.04'
          PYTHON_INTERPRETER: python3
        Linux_Python37:
          PYTHON_VERSION_SPEC: 3.7
          IMAGE: 'Ubuntu-20.04'
          PYTHON_INTERPRETER: python3
        Linux_Python38:
          PYTHON_VERSION_SPEC: 3.8
          IMAGE: 'Ubuntu-20.04'
          PYTHON_INTERPRETER: python3
        Linux_Python39:
          PYTHON_VERSION_SPEC: 3.9
          IMAGE: 'Ubuntu-20.04'
          PYTHON_INTERPRETER: python3
        Linux_Python310:
          PYTHON_VERSION_SPEC: 3.10
          IMAGE: 'Ubuntu-20.04'
          PYTHON_INTERPRETER: python3
        Windows_Python36:
          PYTHON_VERSION_SPEC: 3.6
          IMAGE: 'windows-2019'
          PYTHON_INTERPRETER: python
        Windows_Python37:
          PYTHON_VERSION_SPEC: 3.7
          IMAGE: 'windows-2019'
          PYTHON_INTERPRETER: python
        Windows_Python38:
          PYTHON_VERSION_SPEC: 3.8
          IMAGE: 'windows-2019'
          PYTHON_INTERPRETER: python
        Windows_Python39:
          PYTHON_VERSION_SPEC: 3.9
          IMAGE: 'windows-2019'
          PYTHON_INTERPRETER: python
        Windows_Python310:
          PYTHON_VERSION_SPEC: 3.10
          IMAGE: 'windows-2019'
          PYTHON_INTERPRETER: python
        Mac_Python36:
          PYTHON_VERSION_SPEC: 3.6
          IMAGE: 'macOS-11'
          PYTHON_INTERPRETER: python3
        Mac_Python37:
          PYTHON_VERSION_SPEC: 3.7
          IMAGE: 'macOS-11'
          PYTHON_INTERPRETER: python3
        Mac_Python38:
          PYTHON_VERSION_SPEC: 3.8
          IMAGE: 'macOS-11'
          PYTHON_INTERPRETER: python3
        Mac_Python39:
          PYTHON_VERSION_SPEC: 3.9
          IMAGE: 'macOS-11'
          PYTHON_INTERPRETER: python3
        Mac_Python310:
          PYTHON_VERSION_SPEC: 3.10
          IMAGE: 'macOS-11'
          PYTHON_INTERPRETER: python3
    timeoutInMinutes: 10
    pool:
      vmImage: '$(IMAGE)'
    steps:
    - task: UsePythonVersion@0
      inputs:
        versionSpec: $(PYTHON_VERSION_SPEC)
        addToPath: true
    - task: DownloadPipelineArtifact@2
      displayName: 'Downloading dist package'
      inputs:
        source: current
        artifact: dist
        downloadPath: '$(Pipeline.Workspace)'
    - script: |
        $(PYTHON_INTERPRETER) --version
        $(PYTHON_INTERPRETER) -m pip install pytest
        $(PYTHON_INTERPRETER) -m pip install --no-index --find-links=file://$(Pipeline.Workspace) usd-core
        py.test --junitxml TEST-usdinstall.xml build_scripts/pypi/test.py
      displayName: "Testing fresh pip install"
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testRunTitle: 'Test results for $(IMAGE) Python $(PYTHON_VERSION_SPEC)'