unittest.yml

# This is a basic workflow to help you get started with Actions

name: CI

# Controls when the action will run.
on:
  # Triggers the workflow on push or pull request events but only for the specified branches
  push:
    branches:
      - master
    tags:
      - "v*"
  pull_request:
    branches:
      - master

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  unittest:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    strategy:
      matrix:
        config: [pyspark-2.4, tf-1.15, pyarrow-3.0, pyarrow-4.0, latest]
        include:
        - config: pyspark-2.4
          PYARROW_VERSION: "2.0.0"
          NUMPY_VERSION: "1.19.1"
          TF_VERSION: "1.15.5"
          PYSPARK_VERSION: "2.4.4"
          ARROW_PRE_0_15_IPC_FORMAT: 1
          PY: "3.7"
        - config: tf-1.15
          PYARROW_VERSION: "2.0.0"
          NUMPY_VERSION: "1.19.1"
          TF_VERSION: "1.15.5"
          PYSPARK_VERSION: "3.0.0"
          ARROW_PRE_0_15_IPC_FORMAT: 0
          PY: "3.7"
        - config: pyarrow-3.0
          PYARROW_VERSION: "3.0.0"
          NUMPY_VERSION: "1.19.1"
          TF_VERSION: "2.5.0"
          PYSPARK_VERSION: "3.0.0"
          ARROW_PRE_0_15_IPC_FORMAT: 0
          PY: "3.7"
        - config: pyarrow-4.0
          PYARROW_VERSION: "4.0.0"
          NUMPY_VERSION: "1.19.1"
          TF_VERSION: "2.5.0"
          PYSPARK_VERSION: "3.0.0"
          ARROW_PRE_0_15_IPC_FORMAT: 0
          PY: "3.7"
        - config: latest
          PYARROW_VERSION: "6.0.1"
          NUMPY_VERSION: "1.21.5"
          TF_VERSION: "2.8.0"
          PYSPARK_VERSION: "3.0.0"
          ARROW_PRE_0_15_IPC_FORMAT: "0"
          PY: "3.9"

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2

      # Prepare
      - name: prepare
        run: |
          export CI_IMAGE=selitvin/petastorm_ci_auto:ci-image-06-29-2021
          docker pull $CI_IMAGE
          docker images
          pip install -U codecov
          docker run -v `pwd`:/petastorm --name petastorm_ci $CI_IMAGE /bin/sh -c "sleep 3600" &

      # Run unit tests
      - name: build and run unit tests
        run: |
          sleep 30
          export PYARROW_VERSION=${{matrix.PYARROW_VERSION}}
          export NUMPY_VERSION=${{matrix.NUMPY_VERSION}}
          export TF_VERSION=${{matrix.TF_VERSION}}
          export PY=${{matrix.PY}}
          export PYSPARK_VERSION=${{matrix.PYSPARK_VERSION}}
          export ARROW_PRE_0_15_IPC_FORMAT=${{matrix.ARROW_PRE_0_15_IPC_FORMAT}}
          export RUN="docker exec -e ARROW_PRE_0_15_IPC_FORMAT=$ARROW_PRE_0_15_IPC_FORMAT petastorm_ci bash /run_in_venv.sh ${PY}"
          export PYTEST="pytest --timeout=360 -v --color=yes --cov=./ --cov-report xml:coverage.xml"
          $RUN pip install -U pip setuptools
          $RUN pip install -e /petastorm/[test,tf,torch,docs,opencv]
          $RUN pip install --upgrade numpy==$NUMPY_VERSION
          $RUN pip install -U pyarrow==${PYARROW_VERSION} tensorflow==${TF_VERSION} pyspark==${PYSPARK_VERSION}
          $RUN pip list
          $RUN mypy petastorm
          $RUN flake8 . --count --show-source --statistics
          $RUN flake8 . --count --exit-zero --max-complexity=20 --statistics
          $RUN pylint --rcfile=.pylintrc petastorm examples -f parseable -r n
          $RUN ulimit -c unlimited -S
          $RUN bash -c "cd /petastorm/docs/autodoc && pwd && make html"
          $RUN $PYTEST -m "forked" --forked -Y \
          --ignore=examples/mnist/tests/test_pytorch_mnist.py \
          --ignore=petastorm/tests/test_pytorch_utils.py \
          --ignore=petastorm/tests/test_pytorch_dataloader.py \
          --ignore=petastorm/tests/test_tf_autograph.py \
          petastorm examples
          $RUN $PYTEST -m "not forked" -Y  --cov-append \
          --ignore=examples/mnist/tests/test_pytorch_mnist.py \
          --ignore=petastorm/tests/test_pytorch_utils.py \
          --ignore=petastorm/tests/test_pytorch_dataloader.py \
          --ignore=petastorm/tests/test_tf_autograph.py \
          petastorm examples
          $RUN $PYTEST  --cov-append \
          examples/mnist/tests/test_pytorch_mnist.py \
          petastorm/tests/test_pytorch_dataloader.py \
          petastorm/tests/test_pytorch_utils.py
          $RUN $PYTEST -Y --cov-append petastorm/tests/test_tf_autograph.py

      - name: codecov
        run: codecov --required

  draft_release:
    needs: unittest
    # Only come with a tag
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    name: Create Release
    runs-on: ubuntu-latest
    steps:
      - name: Create release file
        run: |
          export VERSION_NUMBER=$(echo "${{ github.ref }}" | sed -nre 's/^[^0-9]*(([0-9]+\.)*[0-9]+).*/\1/p' | sed -e 's/\.//g')
          echo "Version link appendix: ${VERSION_NUMBER}"
          echo "https://github.com/uber/petastorm/blob/master/docs/release-notes.rst#release-${VERSION_NUMBER}" > /tmp/release.md
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Create Release
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
        with:
          tag_name: ${{ github.ref }}
          release_name: Release ${{ github.ref }}
          body_path: /tmp/release.md
          draft: false
          prerelease: ${{ contains(github.ref, 'rc') }} # having "rc" in version number means prerelease

  deploy:
    needs: draft_release
    # Only come with a tag
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Before Deploy
        run: |
          pip install wheel
          python setup.py bdist_wheel

      - name: Publish package
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}