From 3c239534f9e9ea6132cafab23472734c41b79872 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Tue, 28 May 2024 09:22:24 -0400 Subject: [PATCH] Prepare for 2024.5.0 release (#1337) * Prepare for 2024.5.0 release * Override style check's default Rust toolchain * Drop mlflow from GPU CI deps * Add python 3.11 GPU CI job * Bump docker-py to 7.1.0 to get around requests breakage * Don't check dtype for scalar timestamps GPU tests --- .github/workflows/style.yml | 5 ++ Cargo.lock | 2 +- Cargo.toml | 2 +- continuous_integration/environment-3.10.yaml | 3 +- continuous_integration/environment-3.11.yaml | 3 +- continuous_integration/environment-3.12.yaml | 3 +- continuous_integration/environment-3.9.yaml | 3 +- continuous_integration/gpuci/axis.yaml | 1 + .../gpuci/environment-3.10.yaml | 1 - .../gpuci/environment-3.11.yaml | 53 +++++++++++++++++++ .../gpuci/environment-3.9.yaml | 1 - tests/integration/test_rex.py | 4 +- 12 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 continuous_integration/gpuci/environment-3.11.yaml diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index cd83bffc3..2661e39e5 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -15,6 +15,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: 1.72 + components: clippy + default: true - uses: actions-rs/toolchain@v1 with: toolchain: nightly diff --git a/Cargo.lock b/Cargo.lock index 91a50787e..3f5d28f40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -754,7 +754,7 @@ dependencies = [ [[package]] name = "dask-sql" -version = "2024.3.0" +version = "2024.5.0" dependencies = [ "async-trait", "datafusion-python", diff --git a/Cargo.toml b/Cargo.toml index f362039f6..97a5c716e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dask-sql" repository = "https://github.com/dask-contrib/dask-sql" -version = "2024.3.0" +version = "2024.5.0" description = "Bindings for DataFusion used by Dask-SQL" readme = "README.md" license = "Apache-2.0" diff --git a/continuous_integration/environment-3.10.yaml b/continuous_integration/environment-3.10.yaml index 6ee044f42..a05c88665 100644 --- a/continuous_integration/environment-3.10.yaml +++ b/continuous_integration/environment-3.10.yaml @@ -5,6 +5,7 @@ dependencies: - c-compiler - dask>=2024.4.1 - dask-expr>=1.0.11 +- docker-py>=7.1.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 @@ -12,7 +13,7 @@ dependencies: - jsonschema - lightgbm - maturin>=1.3,<1.4 -- mlflow>=2.9 +- mlflow>=2.10 - mock - numpy>=1.22.4 - pandas>=2 diff --git a/continuous_integration/environment-3.11.yaml b/continuous_integration/environment-3.11.yaml index 9edaddbbd..653e5b185 100644 --- a/continuous_integration/environment-3.11.yaml +++ b/continuous_integration/environment-3.11.yaml @@ -5,6 +5,7 @@ dependencies: - c-compiler - dask>=2024.4.1 - dask-expr>=1.0.11 +- docker-py>=7.1.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 @@ -12,7 +13,7 @@ dependencies: - jsonschema - lightgbm - maturin>=1.3,<1.4 -- mlflow>=2.9 +- mlflow>=2.10 - mock - numpy>=1.22.4 - pandas>=2 diff --git a/continuous_integration/environment-3.12.yaml b/continuous_integration/environment-3.12.yaml index 657e18507..9c4f446f0 100644 --- a/continuous_integration/environment-3.12.yaml +++ b/continuous_integration/environment-3.12.yaml @@ -5,6 +5,7 @@ dependencies: - c-compiler - dask>=2024.4.1 - dask-expr>=1.0.11 +- docker-py>=7.1.0 - fastapi>=0.92.0 - fugue>=0.7.3 - httpx>=0.24.1 @@ -13,7 +14,7 @@ dependencies: - lightgbm - maturin>=1.3,<1.4 # TODO: add once mlflow 3.12 builds are available -# - mlflow>=2.9 +# - mlflow>=2.10 - mock - numpy>=1.22.4 - pandas>=2 diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml index 88eee32b5..7b5353c5d 100644 --- a/continuous_integration/environment-3.9.yaml +++ b/continuous_integration/environment-3.9.yaml @@ -5,6 +5,7 @@ dependencies: - c-compiler - dask=2024.4.1 - dask-expr=1.0.11 +- docker-py>=7.1.0 - fastapi=0.92.0 - fugue=0.7.3 - httpx=0.24.1 @@ -12,7 +13,7 @@ dependencies: - jsonschema - lightgbm - maturin=1.3 -- mlflow=2.9 +- mlflow=2.10 - mock - numpy=1.22.4 - pandas=2 diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml index 267eb5a24..f7dff8116 100644 --- a/continuous_integration/gpuci/axis.yaml +++ b/continuous_integration/gpuci/axis.yaml @@ -1,6 +1,7 @@ PYTHON_VER: - "3.9" - "3.10" +- "3.11" CUDA_VER: - "11.8.0" diff --git a/continuous_integration/gpuci/environment-3.10.yaml b/continuous_integration/gpuci/environment-3.10.yaml index 2371144e7..752c9f14f 100644 --- a/continuous_integration/gpuci/environment-3.10.yaml +++ b/continuous_integration/gpuci/environment-3.10.yaml @@ -18,7 +18,6 @@ dependencies: - jsonschema - lightgbm - maturin>=1.3,<1.4 -- mlflow>=2.9 - mock - numpy>=1.22.4 - pandas>=2 diff --git a/continuous_integration/gpuci/environment-3.11.yaml b/continuous_integration/gpuci/environment-3.11.yaml new file mode 100644 index 000000000..ad1fa43d6 --- /dev/null +++ b/continuous_integration/gpuci/environment-3.11.yaml @@ -0,0 +1,53 @@ +name: dask-sql +channels: +- rapidsai +- rapidsai-nightly +- dask/label/dev +- conda-forge +- nvidia +- nodefaults +dependencies: +- c-compiler +- zlib +- dask>=2024.4.1 +- dask-expr>=1.0.11 +- fastapi>=0.92.0 +- fugue>=0.7.3 +- httpx>=0.24.1 +- intake>=0.6.0 +- jsonschema +- lightgbm +- maturin>=1.3,<1.4 +- mock +- numpy>=1.22.4 +- pandas>=2 +- pre-commit +- prompt_toolkit>=3.0.8 +- psycopg2 +- pyarrow>=14.0.1 +- pygments>=2.7.1 +- pyhive +- pytest-cov +- pytest-rerunfailures +- pytest-xdist +- pytest +- python=3.11 +- py-xgboost>=2.0.3 +- scikit-learn>=1.0.0 +- sphinx +- sqlalchemy +- tpot>=0.12.0 +# FIXME: https://github.com/fugue-project/fugue/issues/526 +- triad<0.9.2 +- tzlocal>=2.1 +- uvicorn>=0.14 +# GPU-specific requirements +- cudatoolkit=11.8 +- cudf=24.06 +- cuml=24.06 +- dask-cudf=24.06 +- dask-cuda=24.06 +- ucx-proc=*=gpu +- ucx-py=0.38 +- xgboost=*=rapidsai_py* +- libxgboost=*=rapidsai_h* diff --git a/continuous_integration/gpuci/environment-3.9.yaml b/continuous_integration/gpuci/environment-3.9.yaml index cb54b2ac7..f70473a08 100644 --- a/continuous_integration/gpuci/environment-3.9.yaml +++ b/continuous_integration/gpuci/environment-3.9.yaml @@ -18,7 +18,6 @@ dependencies: - jsonschema - lightgbm - maturin>=1.3,<1.4 -- mlflow>=2.9 - mock - numpy>=1.22.4 - pandas>=2 diff --git a/tests/integration/test_rex.py b/tests/integration/test_rex.py index b02607f61..67e2d137b 100644 --- a/tests/integration/test_rex.py +++ b/tests/integration/test_rex.py @@ -1144,7 +1144,7 @@ def test_scalar_timestamps(c, gpu): } ) df1 = c.sql("SELECT CEIL(to_timestamp(d) TO DAY) AS ceil_to_day FROM df") - assert_eq(df1, expected_df) + assert_eq(df1, expected_df, check_dtype=(not gpu)) df2 = c.sql("SELECT CEIL(CAST(d AS TIMESTAMP) TO DAY) AS ceil_to_day FROM df") assert_eq(df2, expected_df) @@ -1154,7 +1154,7 @@ def test_scalar_timestamps(c, gpu): } ) df1 = c.sql("SELECT FLOOR(to_timestamp(d) TO DAY) AS floor_to_day FROM df") - assert_eq(df1, expected_df) + assert_eq(df1, expected_df, check_dtype=(not gpu)) df2 = c.sql("SELECT FLOOR(CAST(d AS TIMESTAMP) TO DAY) AS floor_to_day FROM df") assert_eq(df2, expected_df)