main.yml

name: Main
on: [push, pull_request]
jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        PYTHON_VERSION: ["3.8", "3.10"]
        JOBLIB_VERSION: ["1.2.0", "1.3.0"]
        PIN_MODE: [false, true]
        PYSPARK_VERSION: ["3.0.3", "3.1.3", "3.2.3", "3.3.2", "3.4.0"]
        exclude:
          - PYSPARK_VERSION: "3.0.3"
            PIN_MODE: true
          - PYSPARK_VERSION: "3.1.3"
            PIN_MODE: true
    name: Run test on pyspark ${{ matrix.PYSPARK_VERSION }}, pin_mode ${{ matrix.PIN_MODE }}, python ${{ matrix.PYTHON_VERSION }}, joblib ${{ matrix.JOBLIB_VERSION }}
    steps:
      - uses: actions/checkout@v3
      - name: Setup python ${{ matrix.PYTHON_VERSION }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.PYTHON_VERSION }}
          architecture: x64
      - name: Install python packages
        run: |
          pip install joblib==${{ matrix.JOBLIB_VERSION }} scikit-learn>=0.23.1 pytest pylint pyspark==${{ matrix.PYSPARK_VERSION }}
      - name: Run pylint
        run: |
          ./run-pylint.sh
      - name: Run test suites
        run: |
          PYSPARK_PIN_THREAD=${{ matrix.PIN_MODE }} ./run-tests.sh