python.yaml

name: Python CI

on:
  push:
    branches:
      - '*'
  pull_request:
    types: ['opened', 'reopened', 'synchronize']

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ "ubuntu-latest" ]
        python-version: [ "3.10" ]

    steps:
      - name: Check out code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Set up Python environment
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: flake8 Lint
        uses: py-actions/flake8@v2
        with:
          path: "python/xoscar"
          args: "--config python/setup.cfg"
      - name: black
        uses: psf/black@stable
        with:
          src: "python/xoscar"
          options: "--check"
      - uses: isort/isort-action@master
        with:
          sortPaths: "python/xoscar"
          configuration: "--check-only --diff --sp python/setup.cfg"
      - name: mypy
        run: pip install mypy && cd python && mypy xoscar
      - name: codespell
        run: pip install codespell && cd python && codespell xoscar

      - name: clang-format
        uses: jidicula/clang-format-action@v4.11.0
        with:
          clang-format-version: '14'
          fallback-style: 'LLVM'
          check-path: 'cpp'
      - name: cpplint
        run: |
          pip install cpplint
          cpplint --recursive cpp
      - name: cmake-format
        run: |
          pip install cmakelang[YAML]
          find . -name "CMakeLists.txt" -not -path "*third_party/*" | xargs cmake-format -c .cmake-format.yaml --check

  build_test_job:
    if: github.repository == 'xorbitsai/xoscar'
    runs-on: ${{ matrix.os }}
    needs: lint
    env:
      CONDA_ENV: xoscar-test
      SELF_HOST_PYTHON: /root/miniconda3/envs/xoscar-test/bin/python
      SELF_HOST_CONDA: /root/miniconda3/condabin/conda
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-13", "windows-latest"]
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        module: ["xoscar"]
        exclude:
          - { os: macos-13, python-version: 3.10}
          - { os: macos-13, python-version: 3.11}
          - { os: windows-latest, python-version: 3.10}
          - { os: windows-latest, python-version: 3.11}
        include:
          - { os: self-hosted, module: gpu, python-version: 3.11}
          - { os: ubuntu-20.04, module: doc-build, python-version: 3.9}

    steps:
    - name: Check out code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        submodules: recursive
    - name: Add msbuild to PATH
      if: ${{ matrix.os == 'windows-latest'}}
      uses: microsoft/setup-msbuild@v2
    - name: Set up conda ${{ matrix.python-version }}
      uses: conda-incubator/setup-miniconda@v3
      if: ${{ matrix.module != 'gpu' }}
      with:
        python-version: ${{ matrix.python-version }}
        activate-environment: ${{ env.CONDA_ENV }}

    # Fix "version `GLIBCXX_3.4.30' not found (required by xoscar_store.cpython-311-x86_64-linux-gnu.so)" issue
    - name: Install libstdcxx-ng
      if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') }}
      run: |
        conda install -c conda-forge libstdcxx-ng

    # Important for python == 3.12
    - name: Update pip and setuptools
      if: ${{ matrix.python-version == '3.12' }}
      run: |
        python -m pip install -U pip setuptools

    - name: Install dependencies
      env:
        MODULE: ${{ matrix.module }}
      if: ${{ matrix.module != 'gpu' }}
      run: |
        pip install numpy scipy cython coverage flaky
        pip install -e ".[dev,extra]"
      working-directory: ./python

    - name: Install ucx dependencies
      if: ${{ matrix.os == 'ubuntu-latest' }}
      run: |
        # ucx-py move to ucxx and ucxx-cu12 can be run on CPU
        # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py
        pip install ucxx-cu12

    - name: Install fury
      if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') }}
      run: |
        pip install pyfury

    - name: Install on GPU
      if: ${{ matrix.module == 'gpu' }}
      run: |
        source activate ${{ env.CONDA_ENV }}
        conda install -y conda-forge::nccl=2.22.3
        pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12
        pip install ucxx-cu12 cloudpickle psutil tblib uvloop packaging "numpy<2.0.0" scipy cython coverage flaky
        python setup.py clean --all
        pip install -e ./
      working-directory: ./python

    - name: Build doc
      if: ${{ matrix.module == 'doc-build' }}
      run: |
        make html
      working-directory: ./doc

    - name: Test with pytest
      if: ${{ matrix.module != 'doc-build'}}
      env:
        MODULE: ${{ matrix.module }}
      run: |
        if [[ "$MODULE" == "xoscar" ]]; then
          pytest --timeout=1500 \
            -W ignore::PendingDeprecationWarning \
            --cov-config=setup.cfg --cov-report=xml --cov=xoscar xoscar --capture=no
        else
          source activate ${{ env.CONDA_ENV }}
          pytest -m cuda --cov-config=setup.cfg --cov-report=xml --cov=xoscar --capture=no
        fi
      working-directory: ./python

    - name: Report coverage data
      uses: codecov/codecov-action@v4
      with:
        working-directory: ./python
        flags: unittests