Skip to content

Integrate with dask>=2023.7.1, enable fugue pytest support, remove python 3.7 support, change pandas dtype system #50

Integrate with dask>=2023.7.1, enable fugue pytest support, remove python 3.7 support, change pandas dtype system

Integrate with dask>=2023.7.1, enable fugue pytest support, remove python 3.7 support, change pandas dtype system #50

Workflow file for this run

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Spark Tests
on:
push:
branches: [ master ]
paths-ignore:
- 'docs/**'
- '**.md'
pull_request:
branches: [ master ]
paths-ignore:
- 'docs/**'
- '**.md'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test_combinations:
name: Spark ${{ matrix.spark-version }} Pandas ${{ matrix.pandas-version }}
runs-on: ubuntu-latest
strategy:
matrix:
spark-version: ["3.1.1","3.4.0"]
pandas-version: ["1.5.3","2.0.1"]
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
uses: actions/setup-python@v1
with:
python-version: 3.9
- name: Install dependencies
run: make devenv
- name: Install Spark ${{ matrix.spark-version }}
run: pip install "pyspark==${{ matrix.spark-version }}"
- name: Install Pandas ${{ matrix.pandas-version }}
run: pip install "pandas==${{ matrix.pandas-version }}"
- name: Downgrade Ibis
if: matrix.spark-version < '3.4.0'
run: pip install "ibis-framework<5"
- name: Test
run: make testspark
test_connect:
name: Spark Connect
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
uses: actions/setup-python@v1
with:
python-version: 3.9
- name: Install dependencies
run: make devenv
- name: Setup Spark
run: make sparkconnect
- name: Test
run: make testsparkconnect